General work on inputhook and the docs.

* Re-wrote the Sphinx docs on GUI integration.
* Improved docstrings of inputhook module.
* Added current_gui function to inputhook.
* Fixed lots of small bugs in the docs.

IPython/core/magic.py

@@ -1,3585 +1,3587 b''
 # -*- coding: utf-8 -*-
 """Magic functions for InteractiveShell.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 # Python standard modules
 import __builtin__
 import bdb
 import inspect
 import os
 import pdb
 import pydoc
 import sys
 import re
 import tempfile
 import time
 import cPickle as pickle
 import textwrap
 from cStringIO import StringIO
 from getopt import getopt,GetoptError
 from pprint import pprint, pformat
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile,pstats
     except ImportError:
         profile = pstats = None
 
 # Homebrewed
 import IPython
 from IPython.utils import wildcard
 from IPython.core import debugger, oinspect
 from IPython.core.fakemodule import FakeModule
 from IPython.external.Itpl import Itpl, itpl, printpl,itplns
 from IPython.utils.PyColorize import Parser
 from IPython.utils.ipstruct import Struct
 from IPython.core.macro import Macro
 from IPython.utils.genutils import *
 from IPython.utils import platutils
 import IPython.utils.generics
 from IPython.core import ipapi
 from IPython.core.ipapi import UsageError
 from IPython.testing import decorators as testdec
 
 #***************************************************************************
 # Utility functions
 def on_off(tag):
     """Return an ON/OFF string for a 1/0 input. Simple utility function."""
     return ['OFF','ON'][tag]
 
 class Bunch: pass
 
 def compress_dhist(dh):
     head, tail = dh[:-10], dh[-10:]
 
     newhead = []
     done = set()
     for h in head:
         if h in done:
             continue
         newhead.append(h)
         done.add(h)
 
     return newhead + tail        
         
 
 #***************************************************************************
 # Main class implementing Magic functionality
 class Magic:
     """Magic functions for InteractiveShell.
 
     Shell functions which can be reached as %function_name. All magic
     functions should accept a string, which they can parse for their own
     needs. This can make some functions easier to type, eg `%cd ../`
     vs. `%cd("../")`
 
     ALL definitions MUST begin with the prefix magic_. The user won't need it
     at the command line, but it is is needed in the definition. """
 
     # class globals
     auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
                    'Automagic is ON, % prefix NOT needed for magic functions.']
 
     #......................................................................
     # some utility functions
 
     def __init__(self,shell):
         
         self.options_table = {}
         if profile is None:
             self.magic_prun = self.profile_missing_notice
         self.shell = shell
 
         # namespace for holding state we may need
         self._magic_state = Bunch()
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
 
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
 
     def lsmagic(self):
         """Return a list of currently available magic functions.
 
         Gives a list of the bare names after mangling (['ls','cd', ...], not
         ['magic_ls','magic_cd',...]"""
 
         # FIXME. This needs a cleanup, in the way the magics list is built.
         
         # magics in class definition
         class_magic = lambda fn: fn.startswith('magic_') and \
                       callable(Magic.__dict__[fn])
         # in instance namespace (run-time user additions)
         inst_magic =  lambda fn: fn.startswith('magic_') and \
                      callable(self.__dict__[fn])
         # and bound magics by user (so they can access self):
         inst_bound_magic =  lambda fn: fn.startswith('magic_') and \
                            callable(self.__class__.__dict__[fn])
         magics = filter(class_magic,Magic.__dict__.keys()) + \
                  filter(inst_magic,self.__dict__.keys()) + \
                  filter(inst_bound_magic,self.__class__.__dict__.keys())
         out = []
         for fn in set(magics):
             out.append(fn.replace('magic_','',1))
         out.sort()
         return out
     
     def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
           - slices: the set of slices is given as a list of strings (like
           ['1','4:8','9'], since this function is for use by magic functions
           which get their arguments as strings.
 
         Optional inputs:
 
           - raw(False): by default, the processed input is used.  If this is
           true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
             hist = self.shell.input_hist_raw
         else:
             hist = self.shell.input_hist
 
         cmds = []
         for chunk in slices:
             if ':' in chunk:
                 ini,fin = map(int,chunk.split(':'))
             elif '-' in chunk:
                 ini,fin = map(int,chunk.split('-'))
                 fin += 1
             else:
                 ini = int(chunk)
                 fin = ini+1
             cmds.append(hist[ini:fin])
         return cmds
         
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         
         oname = oname.strip()
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.shell.user_ns),
                            ('IPython internal', self.shell.internal_ns),
                            ('Python builtin', __builtin__.__dict__),
                            ('Alias', self.shell.alias_table),
                            ]
             alias_ns = self.shell.alias_table
 
         # initialize results to 'null'
         found = 0; obj = None;  ospace = None;  ds = None;
         ismagic = 0; isalias = 0; parent = None
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = 1
                     ospace = nsname
                     if ns == alias_ns:
                         isalias = 1
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             if oname.startswith(self.shell.ESC_MAGIC):
                 oname = oname[1:]
             obj = getattr(self,'magic_'+oname,None)
             if obj is not None:
                 found = 1
                 ospace = 'IPython internal'
                 ismagic = 1
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = 1
             ospace = 'Interactive'
             
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
     
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
         print 'Error in arguments:'
         print OInspect.getdoc(func)
 
     def format_latex(self,strng):
         """Format a string for latex inclusion."""
 
         # Characters that need to be escaped for latex:
         escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
         # Magic command names as headers:
         cmd_name_re = re.compile(r'^(%s.*?):' % self.shell.ESC_MAGIC,
                                  re.MULTILINE)
         # Magic commands 
         cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % self.shell.ESC_MAGIC,
                             re.MULTILINE)
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
 
         # The "\n" symbol
         newline_re = re.compile(r'\\n')
 
         # Now build the string for output:
         #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
         strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
                                 strng)
         strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
         strng = par_re.sub(r'\\\\',strng)
         strng = escape_re.sub(r'\\\1',strng)
         strng = newline_re.sub(r'\\textbackslash{}n',strng)
         return strng
 
     def format_screen(self,strng):
         """Format a string for screen printing.
 
         This removes some latex-type format codes."""
         # Paragraph continue
         par_re = re.compile(r'\\$',re.MULTILINE)
         strng = par_re.sub('',strng)
         return strng
 
     def parse_options(self,arg_str,opt_str,*long_opts,**kw):
         """Parse options passed to an argument string.
 
         The interface is similar to that of getopt(), but it returns back a
         Struct with the options as keys and the stripped argument string still
         as a string.
 
         arg_str is quoted as a true sys.argv vector by using shlex.split.
         This allows us to easily expand variables, glob files, quote
         arguments, etc.
 
         Options:
           -mode: default 'string'. If given as 'list', the argument string is
           returned as a list (split on whitespace) instead of a string.
 
           -list_all: put all option values in lists. Normally only options
           appearing more than once are put in a list.
 
           -posix (True): whether to split the input line in POSIX mode or not,
           as per the conventions outlined in the shlex module from the
           standard library."""
  
         # inject default options at the beginning of the input line
         caller = sys._getframe(1).f_code.co_name.replace('magic_','')
         arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
         
         mode = kw.get('mode','string')
         if mode not in ['string','list']:
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
         posix = kw.get('posix',True)
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
         args = arg_str.split()
         if len(args) >= 1:
             # If the list of inputs only has 0 or 1 thing in it, there's no
             # need to look for options
             argv = arg_split(arg_str,posix)
             # Do regular option processing
             try:
                 opts,args = getopt(argv,opt_str,*long_opts)
             except GetoptError,e:
                 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, 
                                         " ".join(long_opts)))
             for o,a in opts:
                 if o.startswith('--'):
                     o = o[2:]
                 else:
                     o = o[1:]
                 try:
                     odict[o].append(a)
                 except AttributeError:
                     odict[o] = [odict[o],a]
                 except KeyError:
                     if list_all:
                         odict[o] = [a]
                     else:
                         odict[o] = a
 
         # Prepare opts,args for return
         opts = Struct(odict)
         if mode == 'string':
             args = ' '.join(args)
 
         return opts,args
     
     #......................................................................
     # And now the actual magic functions
 
     # Functions for IPython shell work (vars,funcs, config, etc)
     def magic_lsmagic(self, parameter_s = ''):
         """List currently available magic functions."""
         mesc = self.shell.ESC_MAGIC
         print 'Available magic functions:\n'+mesc+\
               ('  '+mesc).join(self.lsmagic())
         print '\n' + Magic.auto_status[self.shell.rc.automagic]
         return None
         
     def magic_magic(self, parameter_s = ''):
         """Print information about the magic function system.
         
         Supported formats: -latex, -brief, -rest        
         """
 
         mode = ''
         try:
             if parameter_s.split()[0] == '-latex':
                 mode = 'latex'
             if parameter_s.split()[0] == '-brief':
                 mode = 'brief'
             if parameter_s.split()[0] == '-rest':
                 mode = 'rest'
                 rest_docs = []
         except:
             pass
 
         magic_docs = []
         for fname in self.lsmagic():
             mname = 'magic_' + fname
             for space in (Magic,self,self.__class__):
                 try:
                     fn = space.__dict__[mname]
                 except KeyError:
                     pass
                 else:
                     break
             if mode == 'brief':
                 # only first line
                 if fn.__doc__:                    
                     fndoc = fn.__doc__.split('\n',1)[0]
                 else:
                     fndoc = 'No documentation'
             else:
                 if fn.__doc__:
                     fndoc = fn.__doc__.rstrip()       
                 else:
                     fndoc = 'No documentation'
                 
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
                                                     fname,fndoc))
                 
             else:
                 magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC,
                                                     fname,fndoc))
                 
         magic_docs = ''.join(magic_docs)
 
         if mode == 'rest':
             return "".join(rest_docs)
         
         if mode == 'latex':
             print self.format_latex(magic_docs)
             return
         else:
             magic_docs = self.format_screen(magic_docs)
         if mode == 'brief':
             return magic_docs
         
         outmsg = """
 IPython's 'magic' functions
 ===========================
 
 The magic function system provides a series of functions which allow you to
 control the behavior of IPython itself, plus a lot of system-type
 features. All these functions are prefixed with a % character, but parameters
 are given without parentheses or quotes.
 
 NOTE: If you have 'automagic' enabled (via the command line option or with the
 %automagic function), you don't need to type in the % explicitly.  By default,
 IPython ships with automagic on, so you should only rarely need the % escape.
 
 Example: typing '%cd mydir' (without the quotes) changes you working directory
 to 'mydir', if it exists.
 
 You can define your own magic functions to extend the system. See the supplied
 ipythonrc and example-magic.py files for details (in your ipython
 configuration directory, typically $HOME/.ipython/).
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like:
 
   execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
 
 will define %pf as a new name for %profile.
 
 You can also call magics in code using the ipmagic() function, which IPython
 automatically adds to the builtin namespace.  Type 'ipmagic?' for details.
 
 For a list of the available magic functions, use %lsmagic. For a description
 of any of them, type %magic_name?, e.g. '%cd?'.
 
 Currently the magic system has the following functions:\n"""
 
         mesc = self.shell.ESC_MAGIC
         outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
                   "\n\n%s%s\n\n%s" % (outmsg,
                                      magic_docs,mesc,mesc,
                                      ('  '+mesc).join(self.lsmagic()),
                                      Magic.auto_status[self.shell.rc.automagic] ) )
 
         page(outmsg,screen_lines=self.shell.rc.screen_length)
   
 
     def magic_autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
 
 
     def magic_automagic(self, parameter_s = ''):
         """Make magic functions callable without having to type the initial %.
 
         Without argumentsl toggles on/off (when off, you must call it as
         %automagic, of course).  With arguments it sets the value, and you can
         use any of (case insensitive):
 
          - on,1,True: to activate
 
          - off,0,False: to deactivate.
 
         Note that magic functions have lowest priority, so if there's a
         variable whose name collides with that of a magic fn, automagic won't
         work for that function (you get the variable instead). However, if you
         delete the variable (del var), the previously shadowed magic function
         becomes visible to automagic again."""
 
         rc = self.shell.rc
         arg = parameter_s.lower()
         if parameter_s in ('on','1','true'):
             rc.automagic = True
         elif parameter_s in ('off','0','false'):
             rc.automagic = False
         else:
             rc.automagic = not rc.automagic
         print '\n' + Magic.auto_status[rc.automagic]
 
     @testdec.skip_doctest
     def magic_autocall(self, parameter_s = ''):
         """Make functions callable without having to type parentheses.
 
         Usage:
 
            %autocall [mode]
 
         The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
         value is toggled on and off (remembering the previous state).
 
         In more detail, these values mean:
 
         0 -> fully disabled
 
         1 -> active, but do not apply if there are no arguments on the line.
 
         In this mode, you get:
 
         In [1]: callable
         Out[1]: <built-in function callable>
 
         In [2]: callable 'hello'
         ------> callable('hello')
         Out[2]: False
 
         2 -> Active always.  Even if no arguments are present, the callable
         object is called:
 
         In [2]: float
         ------> float()
         Out[2]: 0.0
 
         Note that even with autocall off, you can still use '/' at the start of
         a line to treat the first argument on the command line as a function
         and add parentheses to it:
 
         In [8]: /str 43
         ------> str(43)
         Out[8]: '43'
 
         # all-random (note for auto-testing)
         """
 
         rc = self.shell.rc
 
         if parameter_s:
             arg = int(parameter_s)
         else:
             arg = 'toggle'
 
         if not arg in (0,1,2,'toggle'):
             error('Valid modes: (0->Off, 1->Smart, 2->Full')
             return
 
         if arg in (0,1,2):
             rc.autocall = arg
         else: # toggle
             if rc.autocall:
                 self._magic_state.autocall_save = rc.autocall
                 rc.autocall = 0
             else:
                 try:
                     rc.autocall = self._magic_state.autocall_save
                 except AttributeError:
                     rc.autocall = self._magic_state.autocall_save = 1
 
         print "Automatic calling is:",['OFF','Smart','Full'][rc.autocall]
 
     def magic_system_verbose(self, parameter_s = ''):
         """Set verbose printing of system calls.
 
         If called without an argument, act as a toggle"""
 
         if parameter_s:
             val = bool(eval(parameter_s))
         else:
             val = None
             
         self.shell.rc_set_toggle('system_verbose',val)
         print "System verbose printing is:",\
               ['OFF','ON'][self.shell.rc.system_verbose]
 
 
     def magic_page(self, parameter_s=''):
         """Pretty print the object and display it through a pager.
         
         %page [options] OBJECT
 
         If no object is given, use _ (last output).
         
         Options:
 
           -r: page str(object), don't pretty-print it."""
 
         # After a function contributed by Olivier Aubert, slightly modified.
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'r')
         raw = 'r' in opts
 
         oname = args and args or '_'
         info = self._ofind(oname)
         if info['found']:
             txt = (raw and str or pformat)( info['obj'] )
             page(txt)
         else:
             print 'Object `%s` not found' % oname
 
     def magic_profile(self, parameter_s=''):
         """Print your currently active IPyhton profile."""
         if self.shell.rc.profile:
             printpl('Current IPython profile: $self.shell.rc.profile.')
         else:
             print 'No profile active.'
 
     def magic_pinfo(self, parameter_s='', namespaces=None):
         """Provide detailed information about an object.
     
         '%pinfo object' is just a synonym for object? or ?object."""
     
         #print 'pinfo par: <%s>' % parameter_s  # dbg
     
         
         # detail_level: 0 -> obj? , 1 -> obj??
         detail_level = 0
         # We need to detect if we got called as 'pinfo pinfo foo', which can
         # happen if the user types 'pinfo foo?' at the cmd line.
         pinfo,qmark1,oname,qmark2 = \
                re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
         if pinfo or qmark1 or qmark2:
             detail_level = 1
         if "*" in oname:
             self.magic_psearch(oname)
         else:
             self._inspect('pinfo', oname, detail_level=detail_level,
                           namespaces=namespaces)
 
     def magic_pdef(self, parameter_s='', namespaces=None):
         """Print the definition header for any callable object.
 
         If the object is a class, print the constructor information."""
         self._inspect('pdef',parameter_s, namespaces)
 
     def magic_pdoc(self, parameter_s='', namespaces=None):
         """Print the docstring for an object.
 
         If the given object is a class, it will print both the class and the
         constructor docstrings."""
         self._inspect('pdoc',parameter_s, namespaces)
 
     def magic_psource(self, parameter_s='', namespaces=None):
         """Print (or run through pager) the source code for an object."""
         self._inspect('psource',parameter_s, namespaces)
 
     def magic_pfile(self, parameter_s=''):
         """Print (or run through pager) the file where an object is defined.
 
         The file opens at the line where the object definition begins. IPython
         will honor the environment variable PAGER if set, and otherwise will
         do its best to print the file in a convenient form.
 
         If the given argument is not an object currently defined, IPython will
         try to interpret it as a filename (automatically adding a .py extension
         if needed). You can thus use %pfile as a syntax highlighting code
         viewer."""
 
         # first interpret argument as an object name
         out = self._inspect('pfile',parameter_s)
         # if not, try the input as a filename
         if out == 'not found':
             try:
                 filename = get_py_filename(parameter_s)
             except IOError,msg:
                 print msg
                 return
             page(self.shell.inspector.format(file(filename).read()))
 
     def _inspect(self,meth,oname,namespaces=None,**kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
 
         #oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         try:
             oname = oname.strip().encode('ascii')
             #print '2- oname: <%r>' % oname  # dbg
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return 'not found'
             
         info = Struct(self._ofind(oname, namespaces))
         
         if info.found:
             try:
                 IPython.utils.generics.inspect_object(info.obj)
                 return
             except ipapi.TryNext:
                 pass
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__') 
                     # The object belongs to a class instance. 
                     try: 
                         target = getattr(target, path[-1])
                         # The class defines the object. 
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
                         
             pmethod = getattr(self.shell.inspector,meth)
             formatter = info.ismagic and self.format_screen or None
             if meth == 'pdoc':
                 pmethod(info.obj,oname,formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj,oname,formatter,info,**kw)
             else:
                 pmethod(info.obj,oname)
         else:
             print 'Object `%s` not found.' % oname
             return 'not found'  # so callers can take other action
         
     def magic_psearch(self, parameter_s=''):
         """Search for object in namespaces by wildcard.
 
         %psearch [options] PATTERN [OBJECT TYPE]
 
         Note: ? can be used as a synonym for %psearch, at the beginning or at
         the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
         rest of the command line must be unchanged (options come first), so
         for example the following forms are equivalent
 
         %psearch -i a* function
         -i a* function?
         ?-i a* function
 
         Arguments:
         
           PATTERN
 
           where PATTERN is a string containing * as a wildcard similar to its
           use in a shell.  The pattern is matched in all namespaces on the
           search path. By default objects starting with a single _ are not
           matched, many IPython generated objects have a single
           underscore. The default is case insensitive matching. Matching is
           also done on the attributes of objects and not only on the objects
           in a module.
 
           [OBJECT TYPE]
 
           Is the name of a python type from the types module. The name is
           given in lowercase without the ending type, ex. StringType is
           written string. By adding a type here only objects matching the
           given type are matched. Using all here makes the pattern match all
           types (this is the default).
 
         Options:
 
           -a: makes the pattern match even objects whose names start with a
           single underscore.  These names are normally ommitted from the
           search.
 
           -i/-c: make the pattern case insensitive/sensitive.  If neither of
           these options is given, the default is read from your ipythonrc
           file.  The option name which sets this value is
           'wildcards_case_sensitive'.  If this option is not specified in your
           ipythonrc file, IPython's internal default is to do a case sensitive
           search.
 
           -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
           specifiy can be searched in any of the following namespaces:
           'builtin', 'user', 'user_global','internal', 'alias', where
           'builtin' and 'user' are the search defaults.  Note that you should
           not use quotes when specifying namespaces.
 
           'Builtin' contains the python module builtin, 'user' contains all
           user data, 'alias' only contain the shell aliases and no python
           objects, 'internal' contains objects used by IPython.  The
           'user_global' namespace is only used by embedded IPython instances,
           and it contains module-level globals.  You can add namespaces to the
           search with -s or exclude them with -e (these options can be given
           more than once).
     
         Examples:
        
         %psearch a*            -> objects beginning with an a
         %psearch -e builtin a* -> objects NOT in the builtin space starting in a
         %psearch a* function   -> all functions beginning with an a
         %psearch re.e*         -> objects beginning with an e in module re
         %psearch r*.e*         -> objects that start with e in modules starting in r
         %psearch r*.* string   -> all strings in modules beginning with r
 
         Case sensitve search:
        
         %psearch -c a*         list all object beginning with lower case a
 
         Show objects beginning with a single _:
        
         %psearch -a _*         list objects beginning with a single underscore"""
         try:
             parameter_s = parameter_s.encode('ascii')
         except UnicodeEncodeError:
             print 'Python identifiers can only contain ascii characters.'
             return
 
         # default namespaces to be searched
         def_search = ['user','builtin']
 
         # Process options/args
         opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
         opt = opts.get
         shell = self.shell
         psearch = shell.inspector.psearch
 
         # select case options
         if opts.has_key('i'):
             ignore_case = True
         elif opts.has_key('c'):
             ignore_case = False
         else:
             ignore_case = not shell.rc.wildcards_case_sensitive
 
         # Build list of namespaces to search from user options
         def_search.extend(opt('s',[]))
         ns_exclude = ns_exclude=opt('e',[])
         ns_search = [nm for nm in def_search if nm not in ns_exclude]
         
         # Call the actual search
         try:
             psearch(args,shell.ns_table,ns_search,
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
 
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
         If arguments are given, only variables of types matching these
         arguments are returned."""
 
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_config_ns = self.shell.user_config_ns
         out = []
         typelist = parameter_s.split()
 
         for i in user_ns:
             if not (i.startswith('_') or i.startswith('_i')) \
                    and not (i in internal_ns or i in user_config_ns):
                 if typelist:
                     if type(user_ns[i]).__name__ in typelist:
                         out.append(i)
                 else:
                     out.append(i)
         out.sort()
         return out
         
     def magic_who(self, parameter_s=''):
         """Print all interactive variables, with some minimal formatting.
 
         If any arguments are given, only variables whose type matches one of
         these are printed.  For example:
 
           %who function str
 
         will only list functions and strings, excluding all other types of
         variables.  To find the proper type names, simply use type(var) at a
         command line to see how python prints type names.  For example:
 
           In [1]: type('hello')\\
           Out[1]: <type 'str'>
 
         indicates that the type name for strings is 'str'.
 
         %who always excludes executed names loaded through your configuration
         file and things which are internal to IPython.
 
         This is deliberate, as typically you may load many modules and the
         purpose of %who is to show you only what you've manually defined."""
 
         varlist = self.magic_who_ls(parameter_s)
         if not varlist:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
         count = 0
         for i in varlist:
             print i+'\t',
             count += 1
             if count > 8:
                 count = 0
                 print
         print
 
     def magic_whos(self, parameter_s=''):
         """Like %who, but gives some extra information about each variable.
 
         The same type filtering of %who can be applied here.
 
         For all variables, the type is printed. Additionally it prints:
         
           - For {},[],(): their length.
 
           - For numpy and Numeric arrays, a summary with shape, number of
           elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
           too long."""
         
         varnames = self.magic_who_ls(parameter_s)
         if not varnames:
             if parameter_s:
                 print 'No variables match your requested type.'
             else:
                 print 'Interactive namespace is empty.'
             return
 
         # if we have variables, move on...
 
         # for these types, show len() instead of data:
         seq_types = [types.DictType,types.ListType,types.TupleType]
 
         # for numpy/Numeric arrays, display summary info
         try:
             import numpy
         except ImportError:
             ndarray_type = None
         else:
             ndarray_type = numpy.ndarray.__name__
         try:
             import Numeric
         except ImportError:
             array_type = None
         else:
             array_type = Numeric.ArrayType.__name__
 
         # Find all variable names and types so we can figure out column sizes
         def get_vars(i):
             return self.shell.user_ns[i]
         
         # some types are well known and can be shorter
         abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
         def type_name(v):
             tn = type(v).__name__
             return abbrevs.get(tn,tn)
             
         varlist = map(get_vars,varnames)
 
         typelist = []
         for vv in varlist:
             tt = type_name(vv)
 
             if tt=='instance':
                 typelist.append( abbrevs.get(str(vv.__class__),
                                              str(vv.__class__)))
             else:
                 typelist.append(tt)
 
         # column labels and # of spaces as separator
         varlabel = 'Variable'
         typelabel = 'Type'
         datalabel = 'Data/Info'
         colsep = 3
         # variable format strings
         vformat    = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
         vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
         aformat    = "%s: %s elems, type `%s`, %s bytes"
         # find the size of the columns to format the output nicely
         varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
         typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
         # table header
         print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
               ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
         # and the table itself
         kb = 1024
         Mb = 1048576  # kb**2
         for vname,var,vtype in zip(varnames,varlist,typelist):
             print itpl(vformat),
             if vtype in seq_types:
                 print len(var)
             elif vtype in [array_type,ndarray_type]:
                 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
                 if vtype==ndarray_type:
                     # numpy
                     vsize  = var.size
                     vbytes = vsize*var.itemsize
                     vdtype = var.dtype
                 else:
                     # Numeric
                     vsize  = Numeric.size(var)
                     vbytes = vsize*var.itemsize()
                     vdtype = var.typecode()
                     
                 if vbytes < 100000:
                     print aformat % (vshape,vsize,vdtype,vbytes)
                 else:
                     print aformat % (vshape,vsize,vdtype,vbytes),
                     if vbytes < Mb:
                         print '(%s kb)' % (vbytes/kb,)
                     else:
                         print '(%s Mb)' % (vbytes/Mb,)
             else:
                 try:
                     vstr = str(var)
                 except UnicodeEncodeError:
                     vstr = unicode(var).encode(sys.getdefaultencoding(),
                                                'backslashreplace')
                 vstr = vstr.replace('\n','\\n')
                 if len(vstr) < 50:
                     print vstr
                 else:
                     printpl(vfmt_short)
                 
     def magic_reset(self, parameter_s=''):
         """Resets the namespace by removing all names defined by the user.
 
         Input/Output history are left around in case you need them.
 
         Parameters
         ----------
           -y : force reset without asking for confirmation.
 
         Examples
         --------
         In [6]: a = 1
 
         In [7]: a
         Out[7]: 1
 
         In [8]: 'a' in _ip.user_ns
         Out[8]: True
 
         In [9]: %reset -f
 
         In [10]: 'a' in _ip.user_ns
         Out[10]: False
         """
 
         if parameter_s == '-f':
             ans = True
         else:
             ans = self.shell.ask_yes_no(
                 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
         if not ans:
             print 'Nothing done.'
             return
         user_ns = self.shell.user_ns
         for i in self.magic_who_ls():
             del(user_ns[i])
             
         # Also flush the private list of module references kept for script
         # execution protection
         self.shell.clear_main_mod_cache()
 
     def magic_logstart(self,parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call:
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           '_ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
         
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         rc = self.shell.rc
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipytohn remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = rc.opts.get('logfile','')  
         if logfname:
             logfname = os.path.expanduser(logfname)
         rc.opts.logfile = logfname
         loghead = self.shell.loghead_tpl % (rc.opts,rc.args)
         try:
             started  = logger.logstart(logfname,loghead,logmode,
                                        log_output,timestamp,log_raw_input)
         except:
             rc.opts.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.input_hist_raw
             else:
                 input_hist = self.shell.input_hist
                 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
                 logger.log_write(input_hist[1:])
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
                 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     def magic_logstop(self,parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
         self.logger.logstop()
 
     def magic_logoff(self,parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
         
     def magic_logon(self,parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
         
         self.shell.logger.switch_log(1)
     
     def magic_logstate(self,parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()
         
     def magic_pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your ipythonrc
         configuration file (the variable is called 'pdb').
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print 'Automatic pdb calling has been turned',on_off(new_pdb)
 
     def magic_debug(self, parameter_s=''):
         """Activate the interactive debugger in post-mortem mode.
 
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         
         self.shell.debugger(force=True)
 
     @testdec.skip_doctest
     def magic_prun(self, parameter_s ='',user_mode=1,
                    opts=None,arg_lst=None,prog_ns=None):
 
         """Run a statement through the python code profiler.
 
         Usage:
           %prun [options] statement
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>: you can place restrictions on what or how much of the
         profile gets printed. The limit value can be:
 
           * A string: only information for function names containing this string
           is printed.
 
           * An integer: only these many lines are printed.
 
           * A float (between 0 and 1): this fraction of the report is printed
           (for example, use a limit of 0.4 to see the topmost 40% only).
 
         You can combine several limits with repeated use of the option. For
         example, '-l __init__ -l 5' will print only the topmost 5 lines of
         information about class constructors.
 
         -r: return the pstats.Stats object generated by the profiling. This
         object has all the information about the profile in it, and you can
         later use it for further analysis or in other functions.
 
        -s <key>: sort profile by given key. You can provide more than one key
         by using the option several times: '-s key1 -s key2 -s key3...'. The
         default sorting key is 'time'.
 
         The following is copied verbatim from the profile documentation
         referenced below:
 
         When more than one key is provided, additional keys are used as
         secondary criteria when the there is equality in all keys selected
         before them.
         
         Abbreviations can be used for any key names, as long as the
         abbreviation is unambiguous.  The following are the keys currently
         defined:
 
                 Valid Arg       Meaning
                   "calls"      call count
                   "cumulative" cumulative time
                   "file"       file name
                   "module"     file name
                   "pcalls"     primitive call count
                   "line"       line number
                   "name"       function name
                   "nfl"        name/file/line
                   "stdname"    standard name
                   "time"       internal time
 
         Note that all sorts on statistics are in descending order (placing
         most time consuming items first), where as name, file, and line number
         searches are in ascending order (i.e., alphabetical). The subtle
         distinction between "nfl" and "stdname" is that the standard name is a
         sort of the name as printed, which means that the embedded line
         numbers get compared in an odd way.  For example, lines 3, 20, and 40
         would (if the file names were the same) appear in the string order
         "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
         line numbers.  In fact, sort_stats("nfl") is the same as
         sort_stats("name", "file", "line").
 
         -T <filename>: save profile results as shown on screen to a text
         file. The profile is still shown on screen.
 
         -D <filename>: save (via dump_stats) profile statistics to given
         filename. This data is in a format understod by the pstats module, and
         is generated by a call to the dump_stats() method of profile
         objects. The profile is still shown on screen.
 
         If you want to run complete programs under the profiler's control, use
         '%run -p [prof_opts] filename.py [args to program]' where prof_opts
         contains profiler specific options as described here.
         
         You can read the complete documentation for the profile module with::
         
           In [1]: import profile; profile.help()
         """
 
         opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
         # protect user quote marks
         parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
         
         if user_mode:  # regular user call
             opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
                                               list_all=1)
             namespace = self.shell.user_ns
         else:  # called to run a program by %run -p
             try:
                 filename = get_py_filename(arg_lst[0])
             except IOError,msg:
                 error(msg)
                 return
 
             arg_str = 'execfile(filename,prog_ns)'
             namespace = locals()
 
         opts.merge(opts_def)
         
         prof = profile.Profile()
         try:
             prof = prof.runctx(arg_str,namespace,namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
                     
         # Trap output.
         stdout_trap = StringIO()
 
         if hasattr(stats,'stream'):
             # In newer versions of python, the stats object has a 'stream'
             # attribute to write into.
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         else:
             # For older versions, we manually redirect stdout during printing
             sys_stdout = sys.stdout
             try:
                 sys.stdout = stdout_trap
                 stats.print_stats(*lims)
             finally:
                 sys.stdout = sys_stdout
                 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         page(output,screen_lines=self.shell.rc.screen_length)
         print sys_exit,
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print '\n*** Profile stats marshalled to file',\
                   `dump_file`+'.',sys_exit
         if text_file:
             pfile = file(text_file,'w')
             pfile.write(output)
             pfile.close()
             print '\n*** Profile printout saved to text file',\
                   `text_file`+'.',sys_exit
 
         if opts.has_key('r'):
             return stats
         else:
             return None
 
     @testdec.skip_doctest
     def magic_run(self, parameter_s ='',runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage:\\
           %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
         
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt:\\
           $ python file args\\
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         __name__=='__main__' and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Options:
         
         -n: __name__ is NOT set to '__main__', but to the running file's name
         without extension (as python does under import).  This allows running
         scripts and reloading the definitions in them without calling code
         protected by an ' if __name__ == "__main__" ' clause.
 
         -i: run the file in IPython's namespace instead of an empty one. This
         is useful if you are experimenting with code written in a text editor
         which depends on variables defined interactively.
 
         -e: ignore sys.exit() calls or SystemExit exceptions in the script
         being run.  This is particularly useful if IPython is being used to
         run unittests, which always exit with a sys.exit() call.  In such
         cases you are interested in the output of the test results, not in
         seeing a traceback of the unittest module.
 
         -t: print timing information at the end of the run.  IPython will give
         you an estimated CPU time consumption for your script, which under
         Unix uses the resource module to avoid the wraparound problems of
         time.clock().  Under Unix, an estimate of time spent on system tasks
         is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional -N<N> option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py):
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):\\
               User  :    0.19597 s.\\
               System:        0.0 s.\\
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):\\
             Total runs performed: 5\\
               Times :      Total       Per run\\
               User  :   0.910862 s,  0.1821724 s.\\
               System:        0.0 s,        0.0 s.
 
         -d: run your program under the control of pdb, the Python debugger.
         This allows you to execute your program step by step, watch variables,
         etc.  Internally, what IPython does is similar to calling:
         
           pdb.run('execfile("YOURFILENAME")')
 
         with a breakpoint set on line 1 of your file.  You can change the line
         number for this automatic breakpoint to be <N> by using the -bN option
         (where N must be an integer).  For example:
 
           %run -d -b40 myscript
 
         will set the first breakpoint at line 40 in myscript.py.  Note that
         the first breakpoint must be set on a line which actually does
         something (not a comment or docstring) for it to stop execution.
 
         When the pdb debugger starts, you will see a (Pdb) prompt.  You must
         first enter 'c' (without qoutes) to start execution up to the first
         breakpoint.
 
         Entering 'help' gives information about the use of the debugger.  You
         can easily see pdb's full documentation with "import pdb;pdb.help()"
         at a prompt.
 
         -p: run program under the control of the Python profiler module (which
         prints a detailed report of execution times, function calls, etc).
 
         You can pass other options after -p which affect the behavior of the
         profiler itself. See the docs for %prun for details.
 
         In this mode, the program's variables do NOT propagate back to the
         IPython interactive namespace (because they remain in the namespace
         where the profiler executes them).
 
         Internally this triggers a call to %prun, see its documentation for
         details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
         """
 
         # get arguments and set sys.argv for program to be run.
         opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
                                           mode='list',list_all=1)
 
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print '\n%run:\n',oinspect.getdoc(self.magic_run)
             return
         except IOError,msg:
             error(msg)
             return
 
         if filename.lower().endswith('.ipy'):
             self.api.runlines(open(filename).read())
             return
         
         # Control the response to exit() calls made by the script being run
         exit_ignore = opts.has_key('e')
         
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
         sys.argv = [filename]+ arg_lst[1:]  # put in the proper filename
 
         if opts.has_key('i'):
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.new_main_mod(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             main_mod = self.shell.new_main_mod()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
         prog_ns['__file__'] = filename
 
         # pickle fix.  See iplib for an explanation.  But we need to make sure
         # that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
         
         stats = None
         try:
             self.shell.savehist()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
             else:
                 if opts.has_key('d'):
                     deb = debugger.Pdb(self.shell.rc.colors)
                     # reset Breakpoint state, which is moronically kept
                     # in a class
                     bdb.Breakpoint.next = 1
                     bdb.Breakpoint.bplist = {}
                     bdb.Breakpoint.bpbynumber = [None]
                     # Set an initial breakpoint to stop execution
                     maxtries = 10
                     bp = int(opts.get('b',[1])[0])
                     checkline = deb.checkline(filename,bp)
                     if not checkline:
                         for bp in range(bp+1,bp+maxtries+1):
                             if deb.checkline(filename,bp):
                                 break
                         else:
                             msg = ("\nI failed to find a valid line to set "
                                    "a breakpoint\n"
                                    "after trying up to line: %s.\n"
                                    "Please set a valid breakpoint manually "
                                    "with the -b option." % bp)
                             error(msg)
                             return
                     # if we find a good linenumber, set the breakpoint
                     deb.do_break('%s:%s' % (filename,bp))
                     # Start file run
                     print "NOTE: Enter 'c' at the",
                     print "%s prompt to start your script." % deb.prompt
                     try:
                         deb.run('execfile("%s")' % filename,prog_ns)
                         
                     except:
                         etype, value, tb = sys.exc_info()
                         # Skip three frames in the traceback: the %run one,
                         # one inside bdb.py, and the command-line typed by the
                         # user (run by exec in pdb itself).
                         self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
                 else:
                     if runner is None:
                         runner = self.shell.safe_execfile
                     if opts.has_key('t'):
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         if nruns == 1:
                             t0 = clock2()
                             runner(filename,prog_ns,prog_ns,
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
                         else:
                             runs = range(nruns)
                             t0 = clock2()
                             for nr in runs:
                                 runner(filename,prog_ns,prog_ns,
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
                             t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
                             print "  User  : %10s s, %10s s." % (t_usr,t_usr/nruns)
                             print "  System: %10s s, %10s s." % (t_sys,t_sys/nruns)
                             
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
 
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
                     # The shell MUST hold a reference to prog_ns so after %run
                     # exits, the python deletion mechanism doesn't zero it out
                     # (leaving dangling references).
                     self.shell.cache_main_mod(prog_ns,filename)
                     # update IPython interactive namespace
 
                     # Some forms of read errors on the file may mean the
                     # __name__ key was never set; using pop we don't have to
                     # worry about a possible KeyError.
                     prog_ns.pop('__name__', None)
 
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = __builtin__
             
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
             self.shell.reloadhist()
                 
         return stats
 
     def magic_runlog(self, parameter_s =''):
         """Run files as logs.
 
         Usage:\\
           %runlog file1 file2 ...
 
         Run the named files (treating them as log files) in sequence inside
         the interpreter, and return to the prompt.  This is much slower than
         %run because each line is executed in a try/except block, but it
         allows running files with syntax errors in them.
 
         Normally IPython will guess when a file is one of its own logfiles, so
         you can typically use %run even for logs. This shorthand allows you to
         force any file to be treated as a log file."""
 
         for f in parameter_s.split():
             self.shell.safe_execfile(f,self.shell.user_ns,
                                      self.shell.user_ns,islog=1)
 
     @testdec.skip_doctest
     def magic_timeit(self, parameter_s =''):
         """Time execution of a Python statement or expression
 
         Usage:\\
           %timeit [-n<N> -r<R> [-t|-c]] statement
 
         Time execution of a Python statement or expression using the timeit
         module.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen. 
         
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
         
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
         
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         
         Examples:
 
           In [1]: %timeit pass
           10000000 loops, best of 3: 53.3 ns per loop
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, best of 3: 184 ns per loop
 
           In [4]: %timeit -r 4 u == None
           1000000 loops, best of 4: 242 ns per loop
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loops, best of 3: 2 s per loop
           
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         import timeit
         import math
 
         # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
         # certain terminals.  Until we figure out a robust way of
         # auto-detecting if the terminal can deal with it, use plain 'us' for
         # microseconds.  I am really NOT happy about disabling the proper
         # 'micro' prefix, but crashing is worse... If anyone knows what the
         # right solution for this is, I'm all ears...
         #
         # Note: using
         #
         # s = u'\xb5'
         # s.encode(sys.getdefaultencoding())
         #
         # is not sufficient, as I've seen terminals where that fails but
         # print s
         #
         # succeeds
         #
         # See bug: https://bugs.launchpad.net/ipython/+bug/348466
         
         #units = [u"s", u"ms",u'\xb5',"ns"]
         units = [u"s", u"ms",u'us',"ns"]
         
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
                                         posix=False)
         if stmt == "":
             return
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = timeit.Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
 
         src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
                                  'setup': "pass"}
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         t0 = clock()
         code = compile(src, "<magic-timeit>", "exec")
         tc = clock()-t0
         
         ns = {}
         exec code in self.shell.user_ns, ns
         timer.inner = ns["inner"]
         
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
         
         best = min(timer.repeat(repeat, number)) / number
 
         if best > 0.0:
             order = min(-int(math.floor(math.log10(best)) // 3), 3)
         else:
             order = 3
         print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
                                                           precision,
                                                           best * scaling[order],
                                                           units[order])
         if tc > tc_min:
             print "Compiler time: %.2f s" % tc
 
     @testdec.skip_doctest
     def magic_time(self,parameter_s = ''):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
 
         This function provides very basic timing functionality.  In Python
         2.3, the timeit module offers more control and sophistication, so this
         could be rewritten to use it (patches welcome).
         
         Some examples:
 
           In [1]: time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
         
         # fail immediately if the given expression can't be compiled
         
         expr = self.shell.prefilter(parameter_s,False)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
         
         try:
             mode = 'eval'
             t0 = clock()
             code = compile(expr,'<timed eval>',mode)
             tc = clock()-t0
         except SyntaxError:
             mode = 'exec'
             t0 = clock()
             code = compile(expr,'<timed exec>',mode)
             tc = clock()-t0
         # skew measurement as little as possible
         glob = self.shell.user_ns
         clk = clock2
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clk()
             out = eval(code,glob)
             end = clk()
         else:
             st = clk()
             exec code in glob
             end = clk()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
               (cpu_user,cpu_sys,cpu_tot)
         print "Wall time: %.2f s" % wall_time
         if tc > tc_min:
             print "Compiler : %.2f s" % tc
         return out
 
     @testdec.skip_doctest
     def magic_macro(self,parameter_s = ''):
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The notation for indicating number ranges is: n1-n2 means 'use line
         numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
         using the lines numbered 5,6 and 7.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (%hist prints it):
         
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with:
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with:
         
           'print macro_name'.
 
         For one-off cases which DON'T contain magic function calls in them you
         can obtain similar results by explicitly executing slices from your
         input history with:
 
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         if not args:
             macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
             macs.sort()
             return macs
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name,ranges = args[0], args[1:]
         
         #print 'rng',ranges  # dbg
         lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.user_ns.update({name:macro})
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
         print 'Macro contents:'
         print macro,
 
     def magic_save(self,parameter_s = ''):
         """Save a set of lines to a given filename.
 
         Usage:\\
           %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
         
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
         filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
             ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
         cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
     def _edit_macro(self,mname,macro):
         """open an editor with the macro data in a file"""
         filename = self.shell.mktempfile(macro.value)
         self.shell.hooks.editor(filename)
 
         # and make a new macro object, to replace the old one
         mfile = open(filename)
         mvalue = mfile.read()
         mfile.close()
         self.shell.user_ns[mname] = Macro(mvalue)
 
     def magic_ed(self,parameter_s=''):
         """Alias to %edit."""
         return self.magic_edit(parameter_s)
 
     @testdec.skip_doctest
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs IPython's editor hook.  The default version of this hook is
         set to call the __IPYTHON__.rc.editor command.  This is read from your
         environment variable $EDITOR.  If this isn't found, it will default to
         vi under Linux/Unix and to notepad under Windows.  See the end of this
         docstring for how to change the editor hook.
 
         You can also set the value of this editor via the command line option
         '-editor' or in your ipythonrc file. This is useful if you wish to use
         specifically for IPython an editor different from your typical default
         (and for Windows users who typically don't set environment variables).
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
         
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
         
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
         
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
         
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
         
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
         
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
 
 
         Changing the default editor hook:
 
         If you wish to write your own editor hook, you can put it in a
         configuration file which you load at startup time.  The default hook
         is defined in the IPython.core.hooks module, and you can use that as a
         starting example for further modifications.  That file also has
         general instructions on how to set a new hook for use once you've
         defined it."""
         
         # FIXME: This function has become a convoluted mess.  It needs a
         # ground-up rewrite with clean, simple logic.
 
         def make_filename(arg):
             "Make a filename from the given args"
             try:
                 filename = get_py_filename(arg)
             except IOError:
                 if args.endswith('.py'):
                     filename = arg
                 else:
                     filename = None
             return filename
 
         # custom exceptions
         class DataIsObject(Exception): pass
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
         opts_p = opts.has_key('p')
         opts_r = opts.has_key('r')
         
         # Default line number value
         lineno = opts.get('n',None)
 
         if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
             
         # use last_call to remember the state of the previous call, but don't
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.outputcache.prompt_count
             if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
         use_temp = 1
 
         if re.match(r'\d',args):
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
             use_temp = 0
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
                 data = eval(args,self.shell.user_ns)
                 if not type(data) in StringTypes:
                     raise DataIsObject
 
             except (NameError,SyntaxError):
                 # given argument is not a variable, try as a filename
                 filename = make_filename(args)
                 if filename is None:
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
 
                 data = ''
                 use_temp = 0
             except DataIsObject:
 
                 # macros have a special edit function
                 if isinstance(data,Macro):
                     self._edit_macro(args,data)
                     return
                                 
                 # For objects, try to edit the file where they are defined
                 try:
                     filename = inspect.getabsfile(data)
                     if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
                         # class created by %edit? Try to find source
                         # by looking for method definitions instead, the
                         # __module__ in those classes is FakeModule.
                         attrs = [getattr(data, aname) for aname in dir(data)]
                         for attr in attrs:
                             if not inspect.ismethod(attr):
                                 continue
                             filename = inspect.getabsfile(attr)
                             if filename and 'fakemodule' not in filename.lower():
                                 # change the attribute to be the edit target instead
                                 data = attr 
                                 break
                     
                     datafile = 1
                 except TypeError:
                     filename = make_filename(args)
                     datafile = 1
                     warn('Could not find file where `%s` is defined.\n'
                          'Opening a file named `%s`' % (args,filename))
                 # Now, make sure we can actually read the source (if it was in
                 # a temp file it's gone by now).
                 if datafile:
                     try:
                         if lineno is None:
                             lineno = inspect.getsourcelines(data)[1]
                     except IOError:
                         filename = make_filename(args)
                         if filename is None:
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
                 use_temp = 0
         else:
             data = ''
 
         if use_temp:
             filename = self.shell.mktempfile(data)
             print 'IPython will make a temporary file named:',filename
 
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
         try:
             self.shell.hooks.editor(filename,lineno)
         except ipapi.TryNext:
             warn('Could not open editor')
             return
         
         # XXX TODO: should this be generalized for all string vars?
         # For now, this is special-cased to blocks created by cpaste
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
         
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
             if opts_r:
                 self.shell.runlines(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
         
                                                      
         if use_temp:
             try:
                 return open(filename).read()
             except IOError,msg:
                 if msg.filename == filename:
                     warn('File not found. Did you forget to save?')
                     return
                 else:
                     self.shell.showtraceback()
 
     def magic_xmode(self,parameter_s = ''):
         """Switch modes for the exception handlers.
 
         Valid modes: Plain, Context and Verbose.
 
         If called without arguments, acts as a toggle."""
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)
             print 'Exception reporting mode:',shell.InteractiveTB.mode
         except:
             xmode_switch_err('user')
 
         # threaded shells use a special handler in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_mode(mode=new_mode)
             except:
                 xmode_switch_err('threaded')
             
     def magic_colors(self,parameter_s = ''):
         """Switch color scheme for prompts, info system and exception handlers.
 
         Currently implemented schemes: NoColor, Linux, LightBG.
 
         Color scheme names are not case-sensitive."""
 
         def color_switch_err(name):
             warn('Error changing %s color schemes.\n%s' %
                  (name,sys.exc_info()[1]))
             
         
         new_scheme = parameter_s.strip()
         if not new_scheme:
             raise UsageError(
                 "%colors: you must specify a color scheme. See '%colors?'")
             return
         # local shortcut
         shell = self.shell
 
         import IPython.utils.rlineimpl as readline
 
         if not readline.have_readline and sys.platform == "win32":
             msg = """\
 Proper color support under MS Windows requires the pyreadline library.
 You can find it at:
 http://ipython.scipy.org/moin/PyReadline/Intro
 Gary's readline needs the ctypes module, from:
 http://starship.python.net/crew/theller/ctypes
 (Note that ctypes is already part of Python versions 2.5 and newer).
 
 Defaulting color scheme to 'NoColor'"""
             new_scheme = 'NoColor'
             warn(msg)
         
         # readline option is 0
         if not shell.has_readline:
             new_scheme = 'NoColor'
             
         # Set prompt colors
         try:
             shell.outputcache.set_colors(new_scheme)
         except:
             color_switch_err('prompt')
         else:
             shell.rc.colors = \
                        shell.outputcache.color_table.active_scheme_name
         # Set exception colors
         try:
             shell.InteractiveTB.set_colors(scheme = new_scheme)
             shell.SyntaxTB.set_colors(scheme = new_scheme)
         except:
             color_switch_err('exception')
 
         # threaded shells use a verbose traceback in sys.excepthook
         if shell.isthreaded:
             try:
                 shell.sys_excepthook.set_colors(scheme=new_scheme)
             except:
                 color_switch_err('system exception handler')
         
         # Set info (for 'object?') colors
         if shell.rc.color_info:
             try:
                 shell.inspector.set_active_scheme(new_scheme)
             except:
                 color_switch_err('object inspector')
         else:
             shell.inspector.set_active_scheme('NoColor')
                 
     def magic_color_info(self,parameter_s = ''):
         """Toggle color_info.
 
         The color_info configuration parameter controls whether colors are
         used for displaying object details (by things like %psource, %pfile or
         the '?' system). This function toggles this value with each call.
 
         Note that unless you have a fairly recent pager (less works better
         than more) in your system, using colored object information displays
         will not work properly. Test it and see."""
         
         self.shell.rc.color_info = 1 - self.shell.rc.color_info
         self.magic_colors(self.shell.rc.colors)
         print 'Object introspection functions have now coloring:',
         print ['OFF','ON'][self.shell.rc.color_info]
 
     def magic_Pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         
         self.shell.rc.pprint = 1 - self.shell.rc.pprint
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.rc.pprint]
         
     def magic_exit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so.
 
         You can configure whether IPython asks for confirmation upon exit by
         setting the confirm_exit flag in the ipythonrc file."""
 
         self.shell.exit()
 
     def magic_quit(self, parameter_s=''):
         """Exit IPython, confirming if configured to do so (like %exit)"""
 
         self.shell.exit()
         
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
         self.shell.ask_exit()
 
     #......................................................................
     # Functions to implement unix shell-type things
 
     @testdec.skip_doctest
     def magic_alias(self, parameter_s = ''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example:
 
           In [2]: alias all echo "Input in brackets: <%l>"
           In [3]: all hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter):
         
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !! 
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by IPython:
 
         In [6]: alias show echo
         In [7]: PATH='A Python string'
         In [8]: show $PATH
         A Python string
         In [9]: show $$PATH
         /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             stored = self.db.get('stored_aliases', {} )
             atab = self.shell.alias_table
             aliases = atab.keys()
             aliases.sort()
             res = []
             showlast = []
             for alias in aliases:
                 special = False
                 try:
                     tgt = atab[alias][1]
                 except (TypeError, AttributeError):
                     # unsubscriptable? probably a callable
                     tgt = atab[alias]
                     special = True
                 # 'interesting' aliases
                 if (alias in stored or
                     special or 
                     alias.lower() != os.path.splitext(tgt)[0].lower() or
                     ' ' in tgt):
                     showlast.append((alias, tgt))
                 else:
                     res.append((alias, tgt ))                
             
             # show most interesting aliases last
             res.extend(showlast)
             print "Total number of aliases:",len(aliases)
             return res
         try:
             alias,cmd = par.split(None,1)
         except:
             print oinspect.getdoc(self.magic_alias)
         else:
             nargs = cmd.count('%s')
             if nargs>0 and cmd.find('%l')>=0:
                 error('The %s and %l specifiers are mutually exclusive '
                       'in alias definitions.')
             else:  # all looks OK
                 self.shell.alias_table[alias] = (nargs,cmd)
                 self.shell.alias_table_validate(verbose=0)
     # end magic_alias
 
     def magic_unalias(self, parameter_s = ''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         if aname in self.shell.alias_table:
             del self.shell.alias_table[aname]
         stored = self.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
             del stored[aname]
             self.db['stored_aliases'] = stored
             
 
     def magic_rehashx(self, parameter_s = ''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match agains a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
         
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         
         
         ip = self.api
 
         # for the benefit of module completer in ipy_completers.py
         del ip.db['rootmodules']
         
         path = [os.path.abspath(os.path.expanduser(p)) for p in 
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
         
         alias_table = self.shell.alias_table
         syscmdlist = []
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
 
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
         savedir = os.getcwd()
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff) and ff not in self.shell.no_alias:
                             # each entry in the alias table must be (N,name),
                             # where N is the number of positional arguments of the
                             # alias.                            
                             # Dots will be removed from alias names, since ipython
                             # assumes names with dots to be python code
                             alias_table[ff.replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             else:
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in self.shell.no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                             alias_table[base.lower().replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             # Make sure the alias table doesn't contain keywords or builtins
             self.shell.alias_table_validate()
             # Call again init_auto_alias() so we get 'rm -i' and other
             # modified aliases since %rehashx will probably clobber them
             
             # no, we don't want them. if %rehashx clobbers them, good,
             # we'll probably get better versions
             # self.shell.init_auto_alias()
             db = ip.db
             db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
         
     def magic_pwd(self, parameter_s = ''):
         """Return the current working directory path."""
         return os.getcwd()
 
     def magic_cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
             
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names. 
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
         
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'."""
 
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
         oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print 'The requested directory does not exist in history.'
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
             
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
                 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
             
             if ps is None:
                 print "No matching entry in directory history"
                 return
             else:
                 opts = {}
                 
             
         else:
             #turn all non-space-escaping backslashes to slashes, 
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)            
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or opts.has_key('b'):
                 bkms = self.db.get('bookmarks', {})
             
                 if bkms.has_key(ps):
                     target = bkms[ps]
                     print '(bookmark:%s) -> %s' % (ps,target)
                     ps = target
                 else:
                     if opts.has_key('b'):
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
             
         # at this point ps should point to the target dir
         if ps:
             try:                
                 os.chdir(os.path.expanduser(ps))
                 if self.shell.rc.term_title:
                     #print 'set term title:',self.shell.rc.term_title  # dbg
                     platutils.set_term_title('IPy ' + abbrev_cwd())
             except OSError:
                 print sys.exc_info()[1]
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
         else:
             os.chdir(self.shell.home_dir)
             if self.shell.rc.term_title:
                 platutils.set_term_title("IPy ~")
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
             
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print self.shell.user_ns['_dh'][-1]
 
 
     def magic_env(self, parameter_s=''):
         """List environment variables."""
         
         return os.environ.data
 
     def magic_pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
         
         Usage:\\
           %pushd ['dirname']
         """
         
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(parameter_s)
         cwd = os.getcwd().replace(self.home_dir,'~')
         if tgt:
             self.magic_cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.magic_dirs()
 
     def magic_popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.magic_cd(top)
         print "popd ->",top
 
     def magic_dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     def magic_dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
         
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
         
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(Magic.magic_dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
             else:
                 self.arg_err(Magic.magic_dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         nlprint(dh,
                 header = 'Directory history (kept in _dh)',
                 start=ini,stop=fin)
 
     @testdec.skip_doctest
     def magic_sc(self, parameter_s=''):
         """Shell capture - execute a shell command and capture its output.
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
         
         You should use the form 'var = !command' instead. Example:
          
          "%sc -l myfiles = ls ~" should now be written as
             
          "myfiles = !ls ~"
          
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
         
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example:
 
         # all-random
         
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similiarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents:
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for ouptut capture have
         the following special attributes:
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s,'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=',1)
             var = var.strip()
             # But the the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=',1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         out,err = self.shell.getoutputerror(cmd)
         if err:
             print >> Term.cerr,err
         if opts.has_key('l'):
             out = SList(out.split('\n'))
         else:
             out = LSString(out)
         if opts.has_key('v'):
             print '%s ==\n%s' % (var,pformat(out))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     def magic_sx(self, parameter_s=''):
         """Shell execute - run a shell command and capture its output.
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while:
           !ls
         causes ipython to simply issue system('ls'), typing
           !!ls
         is a shorthand equivalent to:
           %sx ls
         
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
 
         if parameter_s:
             out,err = self.shell.getoutputerror(parameter_s)
             if err:
                 print >> Term.cerr,err
             return SList(out.split('\n'))
 
     def magic_bg(self, parameter_s=''):
         """Run a job in the background, in a separate thread.
 
         For example,
 
           %bg myfunc(x,y,z=1)
 
         will execute 'myfunc(x,y,z=1)' in a background thread.  As soon as the
         execution starts, a message will be printed indicating the job
         number.  If your job number is 5, you can use
 
           myvar = jobs.result(5)  or  myvar = jobs[5].result
 
         to assign this result to variable 'myvar'.
 
         IPython has a job manager, accessible via the 'jobs' object.  You can
         type jobs? to get more information about it, and use jobs.<TAB> to see
         its attributes.  All attributes not starting with an underscore are
         meant for public use.
 
         In particular, look at the jobs.new() method, which is used to create
         new jobs.  This magic %bg function is just a convenience wrapper
         around jobs.new(), for expression-based jobs.  If you want to create a
         new job with an explicit function object and arguments, you must call
         jobs.new() directly.
 
         The jobs.new docstring also describes in detail several important
         caveats associated with a thread-based model for background job
         execution.  Type jobs.new? for details.
 
         You can check the status of all jobs with jobs.status().
 
         The jobs variable is set by IPython into the Python builtin namespace.
         If you ever declare a variable named 'jobs', you will shadow this
         name.  You can either delete your global jobs variable to regain
         access to the job manager, or make a new name and assign it manually
         to the manager (stored in IPython's namespace).  For example, to
         assign the job manager to the Jobs name, use:
 
           Jobs = __builtins__.jobs"""
         
         self.shell.jobs.new(parameter_s,self.shell.user_ns)
         
     def magic_r(self, parameter_s=''):
         """Repeat previous input.
 
         Note: Consider using the more powerfull %rep instead!
         
         If given an argument, repeats the previous command which starts with
         the same string, otherwise it just repeats the previous input.
 
         Shell escaped commands (with ! as first character) are not recognized
         by this system, only pure python code and magic commands.
         """
 
         start = parameter_s.strip()
         esc_magic = self.shell.ESC_MAGIC
         # Identify magic commands even if automagic is on (which means
         # the in-memory version is different from that typed by the user).
         if self.shell.rc.automagic:
             start_magic = esc_magic+start
         else:
             start_magic = start
         # Look through the input history in reverse
         for n in range(len(self.shell.input_hist)-2,0,-1):
             input = self.shell.input_hist[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
                 self.shell.runlines(input)
                 return
         print 'No previous input matching `%s` found.' % start
 
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with:
           %cd -b <name>
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif opts.has_key('r'):
             bkms = {}
         elif opts.has_key('l'):
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len,bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print 'Current bookmarks:'
             for bk in bks:
                 print fmt % (bk,bkms[bk])
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting. """
         
         try:
             filename = get_py_filename(parameter_s)
             cont = file_read(filename)
         except IOError:
             try:
                 cont = eval(parameter_s,self.user_ns)
             except NameError:
                 cont = None
         if cont is None:
             print "Error: no such file or variable"
             return
             
         page(self.shell.pycolorize(cont),
              screen_lines=self.shell.rc.screen_length)
 
     def _rerun_pasted(self):
         """ Rerun a previously pasted command.
         """
         b = self.user_ns.get('pasted_block', None)
         if b is None:
             raise UsageError('No previous pasted block available')
         print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
         exec b in self.user_ns
 
     def _get_pasted_lines(self, sentinel):
         """ Yield pasted lines until the user enters the given sentinel value.
         """
         from IPython.core import iplib
         print "Pasting code; enter '%s' alone on the line to stop." % sentinel
         while True:
             l = iplib.raw_input_original(':')
             if l == sentinel:
                 return
             else:
                 yield l
 
     def _strip_pasted_lines_for_code(self, raw_lines):
         """ Strip non-code parts of a sequence of lines to return a block of
         code.
         """
         # Regular expressions that declare text we strip from the input:
         strip_re =  [r'^\s*In \[\d+\]:', # IPython input prompt
                      r'^\s*(\s?>)+', # Python input prompt
                      r'^\s*\.{3,}', # Continuation prompts
                      r'^\++',
                      ]
 
         strip_from_start = map(re.compile,strip_re)
         
         lines = []
         for l in raw_lines:
             for pat in strip_from_start: 
                 l = pat.sub('',l)
             lines.append(l)
                          
         block = "\n".join(lines) + '\n'
         #print "block:\n",block
         return block
 
     def _execute_block(self, block, par):
         """ Execute a block, or store it in a variable, per the user's request.
         """
         if not par:
             b = textwrap.dedent(block)
             self.user_ns['pasted_block'] = b
             exec b in self.user_ns
         else:
             self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par
 
     def magic_cpaste(self, parameter_s=''):
         """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         You must terminate the block with '--' (two minus-signs) alone on the
         line. You can also provide your own sentinel with '%paste -s %%' ('%%' 
         is the new sentinel for this operation)
         
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
         
         '%cpaste -r' re-executes the block previously entered by cpaste.
         
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         paste: automatically pull code from clipboard.
         """
         
         opts,args = self.parse_options(parameter_s,'rs:',mode='string')
         par = args.strip()
         if opts.has_key('r'):
             self._rerun_pasted()
             return
         
         sentinel = opts.get('s','--')
 
         block = self._strip_pasted_lines_for_code(
             self._get_pasted_lines(sentinel))
 
         self._execute_block(block, par)
 
     def magic_paste(self, parameter_s=''):
         """Allows you to paste & execute a pre-formatted code block from clipboard.
         
         The text is pulled directly from the clipboard without user
         intervention and printed back on the screen before execution (unless
         the -q flag is given to force quiet mode).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
         
         You can also pass a variable name as an argument, e.g. '%paste foo'.
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
 
         Options
         -------
         
           -r: re-executes the block previously entered by cpaste.
 
           -q: quiet mode: do not echo the pasted text back to the terminal.
         
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='string')
         par = args.strip()
         if opts.has_key('r'):
             self._rerun_pasted()
             return
 
         text = self.shell.hooks.clipboard_get()
         block = self._strip_pasted_lines_for_code(text.splitlines())
 
         # By default, echo back to terminal unless quiet mode is requested
         if not opts.has_key('q'):
             write = self.shell.write
             write(block)
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
             
         self._execute_block(block, par)
 
     def magic_quickref(self,arg):
         """ Show a quick reference sheet """
         import IPython.core.usage
         qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
         
         page(qr)
         
     def magic_upgrade(self,arg):
         """ Upgrade your IPython installation
         
         This will copy the config files that don't yet exist in your 
         ipython dir from the system config dir. Use this after upgrading 
         IPython if you don't wish to delete your .ipython dir.
 
         Call with -nolegacy to get rid of ipythonrc* files (recommended for
         new users)
 
         """
         ip = self.getapi()
         ipinstallation = path(IPython.__file__).dirname()
         upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'utils' / 'upgradedir.py')
         src_config = ipinstallation / 'config' / 'userconfig'
         userdir = path(ip.options.ipythondir)
         cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
         print ">",cmd
         shell(cmd)
         if arg == '-nolegacy':
             legacy = userdir.files('ipythonrc*')
             print "Nuking legacy files:",legacy
             
             [p.remove() for p in legacy]
             suffix = (sys.platform == 'win32' and '.ini' or '')
             (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
 
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode allows you to toggle the prompt behavior between normal
         IPython prompts and ones that are as similar to the default IPython
         interpreter as possible.
 
         It also supports the pasting of code snippets that have leading '>>>'
         and '...' prompts in them.  This means that you can paste doctests from
         files or docstrings (even if they have leading whitespace), and the
         code will execute correctly.  You can then use '%history -tn' to see
         the translated history without line numbers; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         # XXX - Fix this to have cleaner activate/deactivate calls.
         from IPython.extensions import InterpreterPasteInput as ipaste
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         oc = shell.outputcache
         rc = shell.rc
         meta = shell.meta
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = meta.setdefault('doctest_mode',Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode',False)
         save_dstore('rc_pprint',rc.pprint)
         save_dstore('xmode',shell.InteractiveTB.mode)
         save_dstore('rc_separate_out',rc.separate_out)
         save_dstore('rc_separate_out2',rc.separate_out2)
         save_dstore('rc_prompts_pad_left',rc.prompts_pad_left)
         save_dstore('rc_separate_in',rc.separate_in)
 
         if mode == False:
             # turn on
             ipaste.activate_prefilter()
 
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
 
             # Prompt separators like plain python
             oc.input_sep = oc.prompt1.sep = ''
             oc.output_sep = ''
             oc.output_sep2 = ''
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                                   oc.prompt_out.pad_left = False
 
             rc.pprint = False
 
             shell.magic_xmode('Plain')
 
         else:
             # turn off
             ipaste.deactivate_prefilter()
 
             oc.prompt1.p_template = rc.prompt_in1
             oc.prompt2.p_template = rc.prompt_in2
             oc.prompt_out.p_template = rc.prompt_out
 
             oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in
 
             oc.output_sep = dstore.rc_separate_out
             oc.output_sep2 = dstore.rc_separate_out2
 
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
             rc.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform
         dstore.mode = bool(1-int(mode))
         print 'Doctest mode is:',
         print ['OFF','ON'][dstore.mode]
 
     def magic_gui(self, parameter_s=''):
         """Enable or disable IPython GUI event loop integration.
 
+        %gui [-a] [GUINAME]
+
         This magic replaces IPython's threaded shells that were activated
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
         are supports:  wxPython, PyQt4, PyGTK, and Tk::
 
             %gui wx    # enable wxPython event loop integration
             %gui qt4   # enable PyQt4 event loop integration
             %gui gtk   # enable PyGTK event loop integration
             %gui tk    # enable Tk event loop integration
             %gui       # disable all event loop integration
 
         WARNING:  after any of these has been called you can simply create
         an application object, but DO NOT start the event loop yourself, as
         we have already handled that.
 
         If you want us to create an appropriate application object add the
         "-a" flag to your command::
 
             %gui -a wx
 
         This is highly recommended for most users.
         """
         from IPython.lib import inputhook
         if "-a" in parameter_s:
             app = True
         else:
             app = False
         if not parameter_s:
             inputhook.clear_inputhook()
         elif 'wx' in parameter_s:
             return inputhook.enable_wx(app)
         elif 'qt4' in parameter_s:
             return inputhook.enable_qt4(app)
         elif 'gtk' in parameter_s:
             return inputhook.enable_gtk(app)
         elif 'tk' in parameter_s:
             return inputhook.enable_tk(app)
 
 
 # end Magic

IPython/lib/inputhook.py

@@ -1,181 +1,226 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 Inputhook management for GUI event loop integration.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import ctypes
 import sys
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class InputHookManager(object):
-    """Manage PyOS_InputHook for different GUI toolkits."""
+    """Manage PyOS_InputHook for different GUI toolkits.
+
+    This class installs various hooks under ``PyOSInputHook`` to handle
+    GUI event loop integration.
+    """
     
     def __init__(self):
         self.PYFUNC = ctypes.PYFUNCTYPE(ctypes.c_int)
         self._reset()
 
     def _reset(self):
         self._callback_pyfunctype = None
         self._callback = None
         self._installed = False
+        self._current_gui = None
 
     def get_pyos_inputhook(self):
         """Return the current PyOS_InputHook as a ctypes.c_void_p.
         """
         return ctypes.c_void_p.in_dll(ctypes.pythonapi,"PyOS_InputHook")
 
     def get_pyos_inputhook_as_func(self):
         """Return the current PyOS_InputHook as a ctypes.PYFUNCYPE.
         """
         return self.PYFUNC.in_dll(ctypes.pythonapi,"PyOS_InputHook")
 
     def set_inputhook(self, callback):
         """Set PyOS_InputHook to callback and return the previous one.
         """
         self._callback = callback
         self._callback_pyfunctype = self.PYFUNC(callback)
         pyos_inputhook_ptr = self.get_pyos_inputhook()
         original = self.get_pyos_inputhook_as_func()
         pyos_inputhook_ptr.value = \
             ctypes.cast(self._callback_pyfunctype, ctypes.c_void_p).value
         self._installed = True
         return original
 
     def clear_inputhook(self):
         """Set PyOS_InputHook to NULL and return the previous one.
         """
         pyos_inputhook_ptr = self.get_pyos_inputhook()
         original = self.get_pyos_inputhook_as_func()
         pyos_inputhook_ptr.value = ctypes.c_void_p(None).value
         self._reset()
         return original
 
     def enable_wx(self, app=False):
         """Enable event loop integration with wxPython.
         
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for wxPython, which allows
         the wxPython to integrate with terminal based applications like
         IPython.
         
         Once this has been called, you can use wx interactively by doing::
         
             >>> import wx
             >>> app = wx.App(redirect=False, clearSigInt=False)
         
         Both options this constructor are important for things to work
         properly in an interactive context.
         
         But, *don't start the event loop*.  That is handled automatically by
         PyOS_InputHook.
         """
         from IPython.lib.inputhookwx import inputhook_wx
         self.set_inputhook(inputhook_wx)
+        self._current_gui = 'wx'
         if app:
             import wx
             app = wx.App(redirect=False, clearSigInt=False)
             return app
 
     def disable_wx(self):
         """Disable event loop integration with wxPython.
         
         This merely sets PyOS_InputHook to NULL.
         """
         self.clear_inputhook()
 
     def enable_qt4(self, app=False):
         """Enable event loop integration with PyQt4.
         
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for wxPython, which allows
         the PyQt4 to integrate with terminal based applications like
         IPython.
         
         Once this has been called, you can simply create a QApplication and
         use it.  But, *don't start the event loop*.  That is handled
         automatically by PyOS_InputHook.
         """
         from PyQt4 import QtCore
         # PyQt4 has had this since 4.3.1.  In version 4.2, PyOS_InputHook
         # was set when QtCore was imported, but if it ever got removed,
         # you couldn't reset it.  For earlier versions we can
         # probably implement a ctypes version.
         try:
             QtCore.pyqtRestoreInputHook()
         except AttributeError:
             pass
+        self._current_gui = 'qt4'
         if app:
             from PyQt4 import QtGui
             app = QtGui.QApplication(sys.argv)
             return app
 
     def disable_qt4(self):
         """Disable event loop integration with PyQt4.
         
         This merely sets PyOS_InputHook to NULL.
         """
         self.clear_inputhook()
 
     def enable_gtk(self, app=False):
         """Enable event loop integration with PyGTK.
         
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
         This methods sets the PyOS_InputHook for PyGTK, which allows
         the PyGTK to integrate with terminal based applications like
         IPython.
         
         Once this has been called, you can simple create PyGTK objects and
         use them.  But, *don't start the event loop*.  That is handled
         automatically by PyOS_InputHook.
         """
         import gtk
         try:
             gtk.set_interactive(True)
+            self._current_gui = 'gtk'
         except AttributeError:
             # For older versions of gtk, use our own ctypes version
             from IPython.lib.inputhookgtk import inputhook_gtk
             add_inputhook(inputhook_gtk)
 
     def disable_gtk(self):
         """Disable event loop integration with PyGTK.
         
         This merely sets PyOS_InputHook to NULL.
         """
         self.clear_inputhook()
 
     def enable_tk(self, app=False):
-        # Creating a Tkinter.Tk object sets PyOS_InputHook()
-        pass
+        """Enable event loop integration with Tk.
+
+        Parameters
+        ----------
+        app : bool
+            Create a running application object or not.
+
+        Notes
+        -----
+        Currently this is a no-op as creating a :class:`Tkinter.Tk` object 
+        sets ``PyOS_InputHook``.
+        """
+        self._current_gui = 'tk'
 
     def disable_tk(self):
         """Disable event loop integration with Tkinter.
         
         This merely sets PyOS_InputHook to NULL.
         """
         self.clear_inputhook()
 
+    def current_gui(self):
+        """Return a string indicating the currently active GUI or None."""
+        return self._current_gui
+
 inputhook_manager = InputHookManager()
 
 enable_wx = inputhook_manager.enable_wx
 disable_wx = inputhook_manager.disable_wx
 enable_qt4 = inputhook_manager.enable_qt4
 disable_qt4 = inputhook_manager.disable_qt4
 enable_gtk = inputhook_manager.enable_gtk
 disable_gtk = inputhook_manager.disable_gtk
 enable_tk = inputhook_manager.enable_tk
 disable_tk = inputhook_manager.disable_tk
 clear_inputhook = inputhook_manager.clear_inputhook
 set_inputhook = inputhook_manager.set_inputhook
+current_gui = inputhook_manager.current_gui
  No newline at end of file

docs/autogen_api.py

@@ -1,34 +1,36 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.txt')
     docwriter.package_skip_patterns += [r'\.fixes$',
                                         r'\.externals$',
                                         r'\.extensions',
                                         r'\.kernel.config',
                                         r'\.attic',
+                                        r'\.quarantine',
+                                        r'\.deathrow'
                                         ]
-    docwriter.module_skip_patterns += [ r'\.FakeModule',
+    docwriter.module_skip_patterns += [ r'\.core.fakemodule',
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',
                                         r'\.frontend.process.winprocess',
                                         ]
     docwriter.write_api_docs(outdir)
     docwriter.write_index(outdir, 'gen',
                           relative_to = pjoin('source','api')
                           )
     print '%d files written' % len(docwriter.written_modules)

docs/source/development/overview.txt

@@ -1,515 +1,517 b''
 .. _development:
 
 ==============================
 IPython development guidelines
 ==============================
 
 
 Overview
 ========
 
 This document describes IPython from the perspective of developers. Most
 importantly, it gives information for people who want to contribute to the
 development of IPython. So if you want to help out, read on!
 
 How to contribute to IPython
 ============================
 
 IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_.
 This makes it easy for people to contribute to the development of IPython.
 There are several ways in which you can join in.
 
 If you have a small change that you want to send to the team, you can edit your
 bazaar checkout of IPython (see below) in-place, and ask bazaar for the
 differences::
 
     $ cd /path/to/your/copy/of/ipython
     $ bzr diff > my_fixes.diff
 
 This produces a patch file with your fixes, which we can apply to the source
 tree.  This file should then be attached to a ticket in our `bug tracker
 <https://bugs.launchpad.net/ipython>`_, indicating what it does.
 
 This model of creating small, self-contained patches works very well and there
 are open source projects that do their entire development this way.  However,
 in IPython we have found that for tracking larger changes, making use of
 bazaar's full capabilities in conjunction with Launchpad's code hosting
 services makes for a much better experience.
 
 Making your own branch of IPython allows you to refine your changes over time,
 track the development of the main team, and propose your own full version of
 the code for others to use and review, with a minimum amount of fuss.  The next
 parts of this document will explain how to do this.
 
 Install Bazaar and create a Launchpad account
 ---------------------------------------------
 
 First make sure you have installed Bazaar (see their `website
 <http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about
 you, try the following::
 
     $ bzr whoami
     Joe Coder <[e-mail]>
 
 This should display your name and email. Next, you will want to create an
 account on the `Launchpad website <http://www.launchpad.net>`_ and setup your
 ssh keys. For more information of setting up your ssh keys, see `this link
 <https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_.
 
 Get the main IPython branch from Launchpad
 ------------------------------------------
 
 Now, you can get a copy of the main IPython development branch (we call this
 the "trunk")::
 
     $ bzr branch lp:ipython
 
 Create a working branch
 -----------------------
 
 When working on IPython, you won't actually make edits directly to the
 :file:`lp:ipython` branch. Instead, you will create a separate branch for your
 changes. For now, let's assume you want to do your work in a branch named
 "ipython-mybranch". Create this branch by doing::
 
     $ bzr branch ipython ipython-mybranch
 
 When you actually create a branch, you will want to give it a name that
 reflects the nature of the work that you will be doing in it, like
 "install-docs-update".
 
 Make edits in your working branch
 ---------------------------------
 
 Now you are ready to actually make edits in your :file:`ipython-mybranch`
 branch. Before doing this, it is helpful to install this branch so you can
 test your changes as you work. This is easiest if you have setuptools
 installed. Then, just do::
 
     $ cd ipython-mybranch
     $ python setupegg.py develop
 
 Now, make some changes. After a while, you will want to commit your changes.
 This let's Bazaar know that you like the changes you have made and gives you
 an opportunity to keep a nice record of what you have done. This looks like
 this::
 
     $ ...do work in ipython-mybranch...
     $ bzr commit -m "the commit message goes here"
 
 Please note that since we now don't use an old-style linear ChangeLog (that
 tends to cause problems with distributed version control systems), you should
 ensure that your log messages are reasonably detailed.  Use a docstring-like
 approach in the commit messages (including the second line being left
 *blank*)::
 
   Single line summary of  changes being committed.
 
   * more details when warranted ...
   * including crediting outside contributors if they sent the
     code/bug/idea!
 
 As you work, you will repeat this edit/commit cycle many times. If you work on
 your branch for a long time, you will also want to get the latest changes from
 the :file:`lp:ipython` branch. This can be done with the following sequence of
 commands::
 
     $ ls
     ipython
     ipython-mybranch
     
     $ cd ipython
     $ bzr pull
     $ cd ../ipython-mybranch
     $ bzr merge ../ipython
     $ bzr commit -m "Merging changes from trunk"
 
 Along the way, you should also run the IPython test suite.  You can do this
 using the :command:`iptest` command (which is basically a customized version of
 :command:`nosetests`)::
 
     $ cd
     $ iptest
 
 The :command:`iptest` command will also pick up and run any tests you have
-written.  See :ref:`_devel_testing` for further details on the testing system.
+written. See :ref:`testing documentation <devel_testing>` for further details
+on the testing system.
 
 
 Post your branch and request a code review
 ------------------------------------------
 
 Once you are done with your edits, you should post your branch on Launchpad so
 that other IPython developers can review the changes and help you merge your
 changes into the main development branch. To post your branch on Launchpad,
 do::
 
     $ cd ipython-mybranch
     $ bzr push lp:~yourusername/ipython/ipython-mybranch
 
 Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you
 should see your branch under the "Code" tab. If you click on your branch, you
 can provide a short description of the branch as well as mark its status. Most
 importantly, you should click the link that reads "Propose for merging into
 another branch". What does this do?
 
 This let's the other IPython developers know that your branch is ready to be
 reviewed and merged into the main development branch. During this review
 process, other developers will give you feedback and help you get your code
 ready to be merged. What types of things will we be looking for:
 
 * All code is documented.
 * All code has tests.
 * The entire IPython test suite passes.
 
 Once your changes have been reviewed and approved, someone will merge them
 into the main development branch.
 
 
 Some notes for core developers when merging third-party contributions
 =====================================================================
 
 Core developers, who ultimately merge any approved branch (from themselves,
 another developer, or any third-party contribution) will typically use
-:command:`bzr merge` to merge the branch into the trunk and push it to the main
-Launcphad site.  This is a short list of things to keep in mind when doing this
-process, so that the project history is easy to understand in the long run, and
-that generating release notes is as painless and accurate as possible.
-
-- When you merge any non-trivial functionality (from one small bug fix to a big
-  feature branch), please remember to always edit the changes_ file
-  accordingly.  This file has one main section for each release, and if you
-  edit it as you go, noting what new features, bug fixes or API changes you
-  have made, the release notes will be almost finished when they are needed
-  later.  This is much easier if done when you merge the work, rather than
-  weeks or months later by re-reading a massive Bazaar log.
-
-- When big merges are done, the practice of putting a summary commit message in
-  the merge is *extremely* useful.  It makes this kind of job much nicer,
+:command:`bzr merge` to merge the branch into the trunk and push it to the
+main Launcphad site. This is a short list of things to keep in mind when doing
+this process, so that the project history is easy to understand in the long
+run, and that generating release notes is as painless and accurate as
+possible.
+
+- When you merge any non-trivial functionality (from one small bug fix to a
+  big feature branch), please remember to always edit the :file:`changes.txt`
+  file accordingly. This file has one main section for each release, and if
+  you edit it as you go, noting what new features, bug fixes or API changes
+  you have made, the release notes will be almost finished when they are
+  needed later. This is much easier if done when you merge the work, rather
+  than weeks or months later by re-reading a massive Bazaar log.
+
+- When big merges are done, the practice of putting a summary commit message
+  in the merge is *extremely* useful. It makes this kind of job much nicer,
   because that summary log message can be almost copy/pasted without changes,
   if it was well written, rather than dissecting the next-level messages from
   the individual commits.
 
 - It's important that we remember to always credit who gave us something if
   it's not the committer.  In general, we have been fairly good on this front,
   this is just a reminder to keep things up.  As a note, if you are ever
   committing something that is completely (or almost so) a third-party
   contribution, do the commit as::
 
     $ bzr commit --author="Someone Else"
 
   This way it will show that name separately in the log, which makes it even
   easier to spot.  Obviously we often rework third party contributions
   extensively, but this is still good to keep in mind for cases when we don't
   touch the code too much.
 
 
 Documentation
 =============
 
 Standalone documentation
 ------------------------
 
 All standalone documentation should be written in plain text (``.txt``) files
 using reStructuredText [reStructuredText]_ for markup and formatting. All such
 documentation should be placed in directory :file:`docs/source` of the IPython
 source tree. The documentation in this location will serve as the main source
 for IPython documentation and all existing documentation should be converted
 to this format.
 
 To build the final documentation, we use Sphinx [Sphinx]_.  Once you have
 Sphinx installed, you can build the html docs yourself by doing::
 
     $ cd ipython-mybranch/docs
     $ make html
 
 Docstring format
 ----------------
 
 Good docstrings are very important. All new code should have docstrings that
 are formatted using reStructuredText for markup and formatting, since it is
 understood by a wide variety of tools. Details about using reStructuredText
 for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
 
 Additional PEPs of interest regarding documentation of code:
 
 * `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
 * `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
 * `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 
 Coding conventions
 ==================
 
 General
 -------
 
 In general, we'll try to follow the standard Python style conventions as
 described here:
 
 * `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
 
 Other comments:
 
 * In a large file, top level classes and functions should be 
   separated by 2-3 lines to make it easier to separate them visually.
 * Use 4 spaces for indentation.
 * Keep the ordering of methods the same in classes that have the same 
   methods.  This is particularly true for classes that implement an interface.
 
 Naming conventions
 ------------------
 
 In terms of naming conventions, we'll follow the guidelines from the `Style
 Guide for Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll
 use:
 
 * All ``lowercase`` module names.
 
 * ``CamelCase`` for class names.
 
 * ``lowercase_with_underscores`` for methods, functions, variables and
   attributes.
 
 There are, however, some important exceptions to these rules. In some cases,
 IPython code will interface with packages (Twisted, Wx, Qt) that use other
 conventions. At some level this makes it impossible to adhere to our own
 standards at all times. In particular, when subclassing classes that use other
 naming conventions, you must follow their naming conventions. To deal with
 cases like this, we propose the following policy:
 
 * If you are subclassing a class that uses different conventions, use its
   naming conventions throughout your subclass.  Thus, if you are creating a
   Twisted Protocol class, used Twisted's
   ``namingSchemeForMethodsAndAttributes.``
 
 * All IPython's official interfaces should use our conventions.  In some cases
   this will mean that you need to provide shadow names (first implement
   ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
   costs, but it will probably be necessary at times.  But, please use this
   sparingly!
 
 Implementation-specific *private* methods will use
 ``_single_underscore_prefix``. Names with a leading double underscore will
 *only* be used in special cases, as they makes subclassing difficult (such
 names are not easily seen by child classes).
 
 Occasionally some run-in lowercase names are used, but mostly for very short
 names or where we are implementing methods very similar to existing ones in a
 base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
 established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
 explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
 upon, as namespaces offer cleaner prefixing. The only case where this approach
 is justified is for classes which are expected to be imported into external
 namespaces and a very generic name (like Shell) is too likely to clash with
 something else. We'll need to revisit this issue as we clean up and refactor
 the code, but in general we should remove as many unnecessary ``IP``/``ip``
 prefixes as possible. However, if a prefix seems absolutely necessary the more
 specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
 It is extremely important that all code contributed to IPython has tests.
 Tests should be written as unittests, doctests or as entities that the Nose
 [Nose]_ testing package will find. Regardless of how the tests are written, we
 will use Nose for discovering and running the tests. Nose will be required to
 run the IPython test suite, but will not be required to simply use IPython.
 
 Tests of Twisted using code need to follow two additional guidelines:
 
 1. Twisted using tests should be written by subclassing the :class:`TestCase`
    class that comes with :mod:`twisted.trial.unittest`. 
 
 2. All :class:`Deferred` instances that are created in the test must be
    properly chained and the final one *must* be the return value of the test
    method.
 
 When these two things are done, Nose will be able to run the tests and the
 twisted reactor will be handled correctly.
 
 Each subpackage in IPython should have its own :file:`tests` directory that
 contains all of the tests for that subpackage. This allows each subpackage to
 be self-contained.  A good convention to follow is to have a file named
 :file:`test_foo.py` for each module :file:`foo.py` in the package.  This makes
 it easy to organize the tests, though like most conventions, it's OK to break
 it if logic and common sense dictate otherwise.
 
 If a subpackage has any dependencies beyond the Python standard library, the
 tests for that subpackage should be skipped if the dependencies are not
 found. This is very important so users don't get tests failing simply because
 they don't have dependencies.  We ship a set of decorators in the
 :mod:`IPython.testing` package to tag tests that may be platform-specific or
 otherwise may have restrictions; if the existing ones don't fit your needs, add
 a new decorator in that location so other tests can reuse it.
 
 To run the IPython test suite, use the :command:`iptest` command that is
 installed with IPython (if you are using IPython in-place, without installing
 it, you can find this script in the :file:`scripts` directory)::
 
     $ iptest
 
 This command colects all IPython tests into separate groups, and then calls
 either Nose with the proper options and extensions, or Twisted's
 :command:`trial`.  This ensures that tests that need the Twisted reactor
 management facilities execute separate of Nose.  If any individual test group
 fails, :command:`iptest` will print what you need to type so you can rerun that
 particular test group alone for debugging.
 
 By default, :command:`iptest` runs the entire IPython test
 suite (skipping tests that may be platform-specific or which depend on tools
 you may not have).  But you can also use it to run only one specific test file,
 or a specific test function.  For example, this will run only the
 :file:`test_magic` file from the test suite::
 
     $ iptest IPython.tests.test_magic
     ----------------------------------------------------------------------
     Ran 10 tests in 0.348s
 
     OK (SKIP=3)
     Deleting object: second_pass
 
 while the ``path:function`` syntax allows you to select a specific function in
 that file to run::
 
     $ iptest IPython.tests.test_magic:test_obj_del
     ----------------------------------------------------------------------
     Ran 1 test in 0.204s
 
     OK
 
 Since :command:`iptest` is based on nosetests, you can pass it any regular
 nosetests option.  For example, you can use ``--pdb`` or ``--pdb-failures`` to
 automatically activate the interactive Pdb debugger on errors or failures.  See
 the nosetests documentation for further details.
 
 
 A few tips for writing tests
 ----------------------------
 
 You can write tests either as normal test files, using all the conventions that
 Nose recognizes, or as doctests.  Note that *all* IPython functions should have
 at least one example that serves as a doctest, whenever technically feasible.
 However, example doctests should only be in the main docstring if they are *a
 good example*, i.e. if they convey useful information about the function.  If
 you simply would like to write a test as a doctest, put it in a separate test
 file and write a no-op function whose only purpose is its docstring.
 
 Note, however, that in a file named :file:`test_X`, functions whose only test
 is their docstring (as a doctest) and which have no test functionality of their
 own, should be called *doctest_foo* instead of *test_foo*, otherwise they get
 double-counted (the empty function call is counted as a test, which just
 inflates tests numbers artificially).  This restriction does not apply to
 functions in files with other names, due to how Nose discovers tests.
 
 You can use IPython examples in your docstrings.  Those can make full use of
 IPython functionality (magics, variable substitution, etc), but be careful to
 keep them generic enough that they run identically on all Operating Systems.
 
 The prompts in your doctests can be either of the plain Python ``>>>`` variety
 or ``In [1]:`` IPython style.  Since this is the IPython system, after all, we
 encourage you to use IPython prompts throughout, unless you are illustrating a
 specific aspect of the normal prompts (such as the ``%doctest_mode`` magic).
 
 If a test isn't safe to run inside the main nose process (e.g. because it loads
 a GUI toolkit), consider running it in a subprocess and capturing its output
 for evaluation and test decision later.  Here is an example of how to do it, by
 relying on the builtin ``_ip`` object that contains the public IPython api as
 defined in :mod:`IPython.ipapi`::
 
    def test_obj_del():
        """Test that object's __del__ methods are called on exit."""
        test_dir = os.path.dirname(__file__)
        del_file = os.path.join(test_dir,'obj_del.py')
        out = _ip.IP.getoutput('ipython %s' % del_file)
        nt.assert_equals(out,'object A deleted')
 
 
 
 If a doctest contains input whose output you don't want to verify identically
 via doctest (random output, an object id, etc), you can mark a docstring with
 ``#random``.  All of these test will have their code executed but no output
 checking will be done::
 
        >>> 1+3
        junk goes here...  # random
 
        >>> 1+2
        again,  anything goes #random
        if multiline, the random mark is only needed once.
 
        >>> 1+2
        You can also put the random marker at the end:
        # random
 
        >>> 1+2
        # random
        .. or at the beginning.
 
 In a case where you want an *entire* docstring to be executed but not verified
 (this only serves to check that the code runs without crashing, so it should be
 used very sparingly), you can put ``# all-random`` in the docstring.
        
 .. _devel_config:
 
 Release checklist
 =================
 
 Most of the release process is automated by the :file:`release` script in the
 :file:`tools` directory.  This is just a handy reminder for the release manager.
 
 #. First, run :file:`build_release`, which does all the file checking and
    building that the real release script will do.  This will let you do test
    installations, check that the build procedure runs OK, etc.  You may want to
    disable a few things like multi-version RPM building while testing, because
    otherwise the build takes really long.
 
 #. Run the release script, which makes the tar.gz, eggs and Win32 .exe
    installer.  It posts them to the site and registers the release with PyPI.
 
 #. Updating the website with announcements and links to the updated
    changes.txt in html form. Remember to put a short note both on the news
    page of the site and on Launcphad.
 
 #. Drafting a short release announcement with i) highlights and ii) a link to
    the html changes.txt.
 
 #. Make sure that the released version of the docs is live on the site.
 
 #. Celebrate!
 
 Porting to 3.0
 ==============
 
 There are no definite plans for porting of IPython to python 3. The major 
 issue is the dependency on twisted framework for the networking/threading 
 stuff. It is possible that it the traditional IPython interactive console 
 could be ported more easily since it has no such dependency. Here are a few
 things that will need to be considered when doing such a port especially
 if we want to have a codebase that works directly on both 2.x and 3.x.
 
 	1. The syntax for exceptions changed (PEP 3110). The old 
 	   `except exc, var` changed to `except exc as var`. At last
 	   count there was 78 occurences of this usage in the codebase.  This
 	   is  a particularly problematic issue, because it's not easy to
 	   implement it in a 2.5-compatible way.
 
 Because it is quite difficult to support simultaneously Python 2.5 and 3.x, we
 will likely at some point put out a release that requires strictly 2.6 and
 abandons 2.5 compatibility.  This will then allow us to port the code to using
 :func:`print` as a function, `except exc as var` syntax, etc.  But as of
 version 0.11 at least, we will retain Python 2.5 compatibility.
 
 
 .. [Bazaar] Bazaar. http://bazaar-vcs.org/
 .. [Launchpad] Launchpad. http://www.launchpad.net/ipython
 .. [reStructuredText] reStructuredText.  http://docutils.sourceforge.net/rst.html
 .. [Sphinx] Sphinx. http://sphinx.pocoo.org/
 .. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/

docs/source/development/reorg.txt

@@ -1,94 +1,84 b''
 =============================
 IPython module reorganization
 =============================
 
 Currently, IPython has many top-level modules that serve many different
 purposes. The lack of organization make it very difficult for developers to
 work on IPython and understand its design. This document contains notes about
 how we will reorganize the modules into sub-packages.
 
 .. warning::
 
     This effort will possibly break third party packages that use IPython as
     a library or hack on the IPython internals.
 
 .. warning::
 
     This effort will result in the removal from IPython of certain modules
     that are not used anymore, don't currently work, are unmaintained, etc.
 
 
 Current subpackges
 ==================
 
 IPython currently has the following sub-packages:
 
 * :mod:`IPython.config`
 
 * :mod:`IPython.Extensions`
 
 * :mod:`IPython.external`
 
 * :mod:`IPython.frontend`
 
 * :mod:`IPython.gui`
 
 * :mod:`IPython.kernel`
 
 * :mod:`IPython.testing`
 
 * :mod:`IPython.tests`
 
 * :mod:`IPython.tools`
 
 * :mod:`IPython.UserConfig`
 
 New Subpackages to be created
 =============================
 
 We propose to create the following new sub-packages:
 
 * :mod:`IPython.core`. This sub-package will contain the core of the IPython
   interpreter, but none of its extended capabilities.
 
 * :mod:`IPython.lib`. IPython has many extended capabilities that are not part
   of the IPython core. These things will go here.
 
 * :mod:`IPython.utils`. This sub-package will contain anything that might
   eventually be found in the Python standard library, like things in
   :mod:`genutils`. Each sub-module in this sub-package should contain
   functions and classes that serve a single purpose.
 
 * :mod:`IPython.deathrow`. This is for code that is untested and/or rotting
   and needs to be removed from IPython. Eventually all this code will either
   i) be revived by someone willing to maintain it with tests and docs and
   re-included into IPython or 2) be removed from IPython proper, but put into
   a separate top-level (not IPython) package that we keep around. No new code
   will be allowed here.
 
 * :mod:`IPython.quarantine`.  This is for code that doesn't meet IPython's 
   standards, but that we plan on keeping.  To be moved out of this sub-package
   a module needs to have a maintainer, tests and documentation.
 
 Procedure
 =========
 
 1. Move the file to its new location with its new name.
 2. Rename all import statements to reflect the change.
 3. Run PyFlakes on each changes module.
-3. Add tests/test_imports.py to test it.
+4. Add tests/test_imports.py to test it.
 
 Status
 ======
 
-The new subpackages have been created and the top-level modules have been
-moved and renamed. Import tests have been created for all of the moved and
-renamed modules. The build infrastructure (setup.py and friends) have been
-updated and tested on Mac and Windows. Finally, a compatibility layer has been
-added for iplib, ipapi and Shell. The follow things still need to be done::
-
-* I need to modify iptests to properly skip modules that are no longer top
-  level modules.
-
-* When running python setup.py sdist, the Sphinx API docs fail to build
-  because of something going on with IPython.core.fakemodule
+This branch was merged into trunk in early August of 2009.

docs/source/development/roadmap.txt

@@ -1,81 +1,73 b''
 .. _roadmap:
 
 ===================
 Development roadmap
 ===================
 
-IPython is an ambitious project that is still under heavy development.  However, we want IPython to become useful to as many people as possible, as quickly as possible.  To help us accomplish this, we are laying out a roadmap of where we are headed and what needs to happen to get there.  Hopefully, this will help the IPython developers figure out the best things to work on for each upcoming release.
+IPython is an ambitious project that is still under heavy development.
+However, we want IPython to become useful to as many people as possible, as
+quickly as possible. To help us accomplish this, we are laying out a roadmap
+of where we are headed and what needs to happen to get there. Hopefully, this
+will help the IPython developers figure out the best things to work on for
+each upcoming release.
 
 Work targeted to particular releases
 ====================================
 
-Release 0.10
-------------
-
-* Initial refactor of :command:`ipcluster`.
-
-* Better TextMate integration.
-
-* Merge in the daemon branch.
-
 Release 0.11
 ------------
 
-* Refactor the configuration system and command line options for
-  :command:`ipengine` and :command:`ipcontroller`. This will include the
-  creation of cluster directories that encapsulate all the configuration
-  files, log files and security related files for a particular cluster.
+* [DONE] Full module and package reorganization.
+
+* [DONE] Removal of the threaded shells and new implementation of GUI support
+  based on ``PyOSInputHook``.
 
-* Refactor :command:`ipcluster` to support the new configuration system.
+* Refactor the configuration system.
 
-* Refactor the daemon stuff to support the new configuration system.
+* Prepare to refactor IPython's core by creating a new component and 
+  application system.
 
-* Merge back in the core of the notebook.
 
 Release 0.12
 ------------
 
-* Fully integrate process startup with the daemons for full process 
-  management.
 
-* Make the capabilites of :command:`ipcluster` available from simple Python 
-  classes.
 
 Major areas of work
 ===================
 
 Refactoring the main IPython core
 ---------------------------------
 
 Process management for :mod:`IPython.kernel`
 --------------------------------------------
 
 Configuration system
 --------------------
 
 Performance problems
 --------------------
 
 Currently, we have a number of performance issues that are waiting to bite users:
 
 * The controller stores a large amount of state in Python dictionaries. Under
   heavy usage, these dicts with get very large, causing memory usage problems.
   We need to develop more scalable solutions to this problem, such as using a
   sqlite database to store this state. This will also help the controller to
   be more fault tolerant.
 
 * We currently don't have a good way of handling large objects in the
   controller. The biggest problem is that because we don't have any way of
   streaming objects, we get lots of temporary copies in the low-level buffers.
   We need to implement a better serialization approach and true streaming
   support.
 
 * The controller currently unpickles and repickles objects. We need to use the
   [push|pull]_serialized methods instead.
 
 * Currently the controller is a bottleneck. The best approach for this is to
   separate the controller itself into multiple processes, one for the core
   controller and one each for the controller interfaces.
 
 
 

docs/source/interactive/reference.txt

@@ -1,1632 +1,1539 b''
 =================
 IPython reference
 =================
 
 .. _command_line_options:
 
 Command-line usage
 ==================
 
 You start IPython with the command::
 
     $ ipython [options] files
 
 If invoked with no options, it executes all the files listed in sequence
 and drops you into the interpreter while still acknowledging any options
 you may have set in your ipythonrc file. This behavior is different from
 standard Python, which when called as python -i will only execute one
 file and ignore your configuration setup.
 
 Please note that some of the configuration options are not available at
 the command line, simply because they are not practical here. Look into
 your ipythonrc configuration file for details on those. This file
 typically installed in the $HOME/.ipython directory. For Windows users,
 $HOME resolves to C:\\Documents and Settings\\YourUserName in most
 instances. In the rest of this text, we will refer to this directory as
 IPYTHONDIR.
 
-.. _Threading options:
 
 
 Special Threading Options
 -------------------------
 
-The following special options are ONLY valid at the beginning of the
-command line, and not later. This is because they control the initial-
-ization of ipython itself, before the normal option-handling mechanism
-is active.
-
-    -gthread, -qthread, -q4thread, -wthread, -pylab: 
-	Only one of these can be given, and it can only be given as
-	the first option passed to IPython (it will have no effect in
-	any other position).  They provide threading support for the
-	GTK, Qt (versions 3 and 4) and WXPython toolkits, and for the
-	matplotlib library.
-
-	With any of the first four options, IPython starts running a
-	separate thread for the graphical toolkit's operation, so that
-	you can open and control graphical elements from within an
-	IPython command line, without blocking. All four provide
-	essentially the same functionality, respectively for GTK, Qt3,
-	Qt4 and WXWidgets (via their Python interfaces).
-
-	Note that with -wthread, you can additionally use the
-	-wxversion option to request a specific version of wx to be
-	used.  This requires that you have the wxversion Python module
-	installed, which is part of recent wxPython distributions.
-
-	If -pylab is given, IPython loads special support for the mat
-	plotlib library (http://matplotlib.sourceforge.net), allowing
-	interactive usage of any of its backends as defined in the
-	user's ~/.matplotlib/matplotlibrc file. It automatically
-	activates GTK, Qt or WX threading for IPyhton if the choice of
-	matplotlib backend requires it. It also modifies the %run
-	command to correctly execute (without blocking) any
-	matplotlib-based script which calls show() at the end.
-
-    -tk
-	The -g/q/q4/wthread options, and -pylab (if matplotlib is
-	configured to use GTK, Qt3, Qt4 or WX), will normally block Tk
-	graphical interfaces. This means that when either GTK, Qt or WX
-	threading is active, any attempt to open a Tk GUI will result in a
-	dead window, and possibly cause the Python interpreter to crash.
-	An extra option, -tk, is available to address this issue. It can
-	only be given as a second option after any of the above (-gthread,
-	-wthread or -pylab).
-
-	If -tk is given, IPython will try to coordinate Tk threading
-	with GTK, Qt or WX. This is however potentially unreliable, and
-	you will have to test on your platform and Python configuration to
-	determine whether it works for you. Debian users have reported
-	success, apparently due to the fact that Debian builds all of Tcl,
-	Tk, Tkinter and Python with pthreads support. Under other Linux
-	environments (such as Fedora Core 2/3), this option has caused
-	random crashes and lockups of the Python interpreter. Under other
-	operating systems (Mac OSX and Windows), you'll need to try it to
-	find out, since currently no user reports are available.
-    
-	There is unfortunately no way for IPython to determine at run time
-	whether -tk will work reliably or not, so you will need to do some
-	experiments before relying on it for regular work.
-
-
+Previously IPython had command line options for controlling GUI event loop
+integration (-gthread, -qthread, -q4thread, -wthread, -pylab). As of IPython
+version 0.11, these have been deprecated. Please see the new ``%gui``
+magic command or :ref:`this section <gui_support>` for details on the new
+interface.
 
 Regular Options
 ---------------
 
 After the above threading options have been given, regular options can
 follow in any order. All options can be abbreviated to their shortest
 non-ambiguous form and are case-sensitive. One or two dashes can be
 used. Some options have an alternate short form, indicated after a ``|``.
 
 Most options can also be set from your ipythonrc configuration file. See
 the provided example for more details on what the options do. Options
 given at the command line override the values set in the ipythonrc file.
 
 All options with a [no] prepended can be specified in negated form
 (-nooption instead of -option) to turn the feature off.
 
     -help  print a help message and exit.
 
     -pylab
-	this can only be given as the first option passed to IPython
-	(it will have no effect in any other position). It adds
-	special support for the matplotlib library
-	(http://matplotlib.sourceforge.ne), allowing interactive usage
-	of any of its backends as defined in the user's .matplotlibrc
-	file.  It automatically activates GTK or WX threading for
-	IPyhton if the choice of matplotlib backend requires it. It
-	also modifies the %run command to correctly execute (without
-	blocking) any matplotlib-based script which calls show() at
-	the end. See `Matplotlib support`_ for more details.
+    Deprecated.  See :ref:`Matplotlib support <matplotlib_support>`
+    for more details.
 
     -autocall <val>
 	Make IPython automatically call any callable object even if you
 	didn't type explicit parentheses. For example, 'str 43' becomes
 	'str(43)' automatically. The value can be '0' to disable the feature,
 	'1' for smart autocall, where it is not applied if there are no more
 	arguments on the line, and '2' for full autocall, where all callable
 	objects are automatically called (even if no arguments are
 	present). The default is '1'.
 
     -[no]autoindent    
 	Turn automatic indentation on/off.
 
     -[no]automagic 
 	make magic commands automatic (without needing their first character
 	to be %). Type %magic at the IPython prompt for more information.
 
     -[no]autoedit_syntax 
 	When a syntax error occurs after editing a file, automatically
 	open the file to the trouble causing line for convenient
 	fixing.
 
     -[no]banner		Print the initial information banner (default on).
 
     -c <command>
 	execute the given command string. This is similar to the -c
 	option in the normal Python interpreter.
 
     -cache_size, cs <n> 
 	size of the output cache (maximum number of entries to hold in
 	memory). The default is 1000, you can change it permanently in your
 	config file. Setting it to 0 completely disables the caching system,
 	and the minimum value accepted is 20 (if you provide a value less than
 	20, it is reset to 0 and a warning is issued) This limit is defined
 	because otherwise you'll spend more time re-flushing a too small cache
 	than working.
 
     -classic, cl 
 	Gives IPython a similar feel to the classic Python
 	prompt.
 
     -colors <scheme> 
 	Color scheme for prompts and exception reporting. Currently
 	implemented: NoColor, Linux and LightBG.
 
     -[no]color_info 
 	IPython can display information about objects via a set of functions,
 	and optionally can use colors for this, syntax highlighting source
 	code and various other elements.  However, because this information is
 	passed through a pager (like 'less') and many pagers get confused with
 	color codes, this option is off by default. You can test it and turn
 	it on permanently in your ipythonrc file if it works for you. As a
 	reference, the 'less' pager supplied with Mandrake 8.2 works ok, but
 	that in RedHat 7.2 doesn't.
 
 	Test it and turn it on permanently if it works with your
 	system. The magic function %color_info allows you to toggle this
 	interactively for testing.
 
     -[no]debug 
 	Show information about the loading process. Very useful to pin down
 	problems with your configuration files or to get details about
 	session restores.
 
     -[no]deep_reload: 
 	IPython can use the deep_reload module which reloads changes in
 	modules recursively (it replaces the reload() function, so you don't
 	need to change anything to use it).  deep_reload() forces a full
 	reload of modules whose code may have changed, which the default
 	reload() function does not.
 
 	When deep_reload is off, IPython will use the normal reload(),
 	but deep_reload will still be available as dreload(). This
 	feature is off by default [which means that you have both
 	normal reload() and dreload()].
 
     -editor <name>
 	Which editor to use with the %edit command. By default,
 	IPython will honor your EDITOR environment variable (if not
 	set, vi is the Unix default and notepad the Windows one).
 	Since this editor is invoked on the fly by IPython and is
 	meant for editing small code snippets, you may want to use a
 	small, lightweight editor here (in case your default EDITOR is
 	something like Emacs).
 
     -ipythondir <name> 
 	name of your IPython configuration directory IPYTHONDIR. This
 	can also be specified through the environment variable
 	IPYTHONDIR.
 
     -log, l 
 	generate a log file of all input. The file is named
 	ipython_log.py in your current directory (which prevents logs
 	from multiple IPython sessions from trampling each other). You
 	can use this to later restore a session by loading your
 	logfile as a file to be executed with option -logplay (see
 	below).
 
     -logfile, lf <name>   specify the name of your logfile.
 
     -logplay, lp <name> 
 
       you can replay a previous log. For restoring a session as close as
       possible to the state you left it in, use this option (don't just run
       the logfile). With -logplay, IPython will try to reconstruct the
       previous working environment in full, not just execute the commands in
       the logfile.
 
       When a session is restored, logging is automatically turned on
       again with the name of the logfile it was invoked with (it is
       read from the log header). So once you've turned logging on for
       a session, you can quit IPython and reload it as many times as
       you want and it will continue to log its history and restore
       from the beginning every time.
       
       Caveats: there are limitations in this option. The history
       variables _i*,_* and _dh don't get restored properly. In the
       future we will try to implement full session saving by writing
       and retrieving a 'snapshot' of the memory state of IPython. But
       our first attempts failed because of inherent limitations of
       Python's Pickle module, so this may have to wait.
 
     -[no]messages   
 	Print messages which IPython collects about its startup
 	process (default on).
 
     -[no]pdb
 	Automatically call the pdb debugger after every uncaught
 	exception. If you are used to debugging using pdb, this puts
 	you automatically inside of it after any call (either in
 	IPython or in code called by it) which triggers an exception
 	which goes uncaught.
 
     -pydb
 	Makes IPython use the third party "pydb" package as debugger,
 	instead of pdb. Requires that pydb is installed.
 	
     -[no]pprint
 	ipython can optionally use the pprint (pretty printer) module
 	for displaying results. pprint tends to give a nicer display
 	of nested data structures. If you like it, you can turn it on
 	permanently in your config file (default off).
 
     -profile, p <name>
 
 	assume that your config file is ipythonrc-<name> or
 	ipy_profile_<name>.py (looks in current dir first, then in
 	IPYTHONDIR).  This is a quick way to keep and load multiple
 	config files for different tasks, especially if you use the
 	include option of config files. You can keep a basic
 	IPYTHONDIR/ipythonrc file and then have other 'profiles' which
 	include this one and load extra things for particular
 	tasks. For example:
 
 	1. $HOME/.ipython/ipythonrc : load basic things you always want.
 	2. $HOME/.ipython/ipythonrc-math : load (1) and basic math-related modules.
 	3. $HOME/.ipython/ipythonrc-numeric : load (1) and Numeric and plotting modules.
 
 	Since it is possible to create an endless loop by having
 	circular file inclusions, IPython will stop if it reaches 15
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
 
         Specify the string used for input prompts. Note that if you are using
 	numbered prompts, the number is represented with a '\#' in the
 	string. Don't forget to quote strings with spaces embedded in
 	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
 	discusses in detail all the available escapes to customize your
 	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
 	prompts. The special sequence '\D' is similar to '\#', but
 	with all digits replaced dots (so you can have your
 	continuation prompt aligned with your input prompt).  Default:
 	' .\D.:' (note three spaces at the start for alignment with
 	'In [\#]').
 
     -prompt_out,po <string> 
 	String used for output prompts, also uses numbers like
 	prompt_in1. Default: 'Out[\#]:'
 
     -quick   start in bare bones mode (no config file loaded).
 
     -rcfile <name>   
 	name of your IPython resource configuration file. Normally
 	IPython loads ipythonrc (from current directory) or
 	IPYTHONDIR/ipythonrc.
 
 	If the loading of your config file fails, IPython starts with
 	a bare bones configuration (no modules loaded at all).
 
     -[no]readline
 	use the readline library, which is needed to support name
 	completion and command history, among other things.  It is
 	enabled by default, but may cause problems for users of
 	X/Emacs in Python comint or shell buffers.
 
 	Note that X/Emacs 'eterm' buffers (opened with M-x term) support
 	IPython's readline and syntax coloring fine, only 'emacs' (M-x
 	shell and C-c !) buffers do not.
 
     -screen_length, sl <n>
 	number of lines of your screen. This is used to control
 	printing of very long strings. Strings longer than this number
 	of lines will be sent through a pager instead of directly
 	printed.
 
 	The default value for this is 0, which means IPython will
 	auto-detect your screen size every time it needs to print certain
 	potentially long strings (this doesn't change the behavior of the
 	'print' keyword, it's only triggered internally). If for some
 	reason this isn't working well (it needs curses support), specify
 	it yourself. Otherwise don't change the default.
 
     -separate_in, si <string>
 
 	separator before input prompts.
 	Default: '\n'
 
     -separate_out, so <string>  
         separator before output prompts.
         Default: nothing.
 
     -separate_out2, so2
 	separator after output prompts.
 	Default: nothing.
 	For these three options, use the value 0 to specify no separator.
 
     -nosep   
 	shorthand for '-SeparateIn 0 -SeparateOut 0 -SeparateOut2
 	0'. Simply removes all input/output separators.
 
     -upgrade 
 	allows you to upgrade your IPYTHONDIR configuration when you
 	install a new version of IPython. Since new versions may
 	include new command line options or example files, this copies
 	updated ipythonrc-type files. However, it backs up (with a
 	.old extension) all files which it overwrites so that you can
 	merge back any customizations you might have in your personal
 	files. Note that you should probably use %upgrade instead,
 	it's a safer alternative.
 
 
     -Version    print version information and exit.
 
     -wxversion <string> 
-	Select a specific version of wxPython (used in conjunction
-	with -wthread). Requires the wxversion module, part of recent
-	wxPython distributions
+    Deprecated.
 
     -xmode <modename>
 
 	Mode for exception reporting.
 
 	Valid modes: Plain, Context and Verbose.
 
 	* Plain: similar to python's normal traceback printing.
 	* Context: prints 5 lines of context source code around each
 	  line in the traceback.
 	* Verbose: similar to Context, but additionally prints the
 	  variables currently visible where the exception happened
 	  (shortening their strings if too long). This can potentially be
 	  very slow, if you happen to have a huge data structure whose
 	  string representation is complex to compute. Your computer may
 	  appear to freeze for a while with cpu usage at 100%. If this
 	  occurs, you can cancel the traceback with Ctrl-C (maybe hitting it
 	  more than once).
 
 Interactive use
 ===============
 
 Warning: IPython relies on the existence of a global variable called
 _ip which controls the shell itself. If you redefine _ip to anything,
 bizarre behavior will quickly occur.
 
 Other than the above warning, IPython is meant to work as a drop-in
 replacement for the standard interactive interpreter. As such, any code
 which is valid python should execute normally under IPython (cases where
 this is not true should be reported as bugs). It does, however, offer
 many features which are not available at a standard python prompt. What
 follows is a list of these.
 
 
 Caution for Windows users
 -------------------------
 
 Windows, unfortunately, uses the '\' character as a path
 separator. This is a terrible choice, because '\' also represents the
 escape character in most modern programming languages, including
 Python. For this reason, using '/' character is recommended if you
 have problems with ``\``.  However, in Windows commands '/' flags
 options, so you can not use it for the root directory. This means that
 paths beginning at the root must be typed in a contrived manner like:
 ``%copy \opt/foo/bar.txt \tmp``
 
 .. _magic:
 
 Magic command system
 --------------------
 
 IPython will treat any line whose first character is a % as a special
 call to a 'magic' function. These allow you to control the behavior of
 IPython itself, plus a lot of system-type features. They are all
 prefixed with a % character, but parameters are given without
 parentheses or quotes.
 
 Example: typing '%cd mydir' (without the quotes) changes you working
 directory to 'mydir', if it exists.
 
 If you have 'automagic' enabled (in your ipythonrc file, via the command
 line option -automagic or with the %automagic function), you don't need
 to type in the % explicitly. IPython will scan its internal list of
 magic functions and call one if it exists. With automagic on you can
 then just type 'cd mydir' to go to directory 'mydir'. The automagic
 system has the lowest possible precedence in name searches, so defining
 an identifier with the same name as an existing magic function will
 shadow it for automagic use. You can still access the shadowed magic
 function by explicitly using the % character at the beginning of the line.
 
 An example (with automagic on) should clarify all this::
 
     In [1]: cd ipython # %cd is called by automagic
 
     /home/fperez/ipython
 
     In [2]: cd=1 # now cd is just a variable
 
     In [3]: cd .. # and doesn't work as a function anymore
 
     ------------------------------
 
         File "<console>", line 1
 
           cd ..
 
               ^
 
     SyntaxError: invalid syntax
 
     In [4]: %cd .. # but %cd always works
 
     /home/fperez
 
     In [5]: del cd # if you remove the cd variable
 
     In [6]: cd ipython # automagic can work again
 
     /home/fperez/ipython
 
 You can define your own magic functions to extend the system. The
 following example defines a new magic command, %impall::
 
     import IPython.ipapi 
 
     ip = IPython.ipapi.get()
 
     def doimp(self, arg):
 
         ip = self.api
 
         ip.ex("import %s; reload(%s); from %s import *" % (
 
         arg,arg,arg)
 
         )
 
     ip.expose_magic('impall', doimp)
 
 You can also define your own aliased names for magic functions. In your
 ipythonrc file, placing a line like::
 
     execute __IP.magic_cl = __IP.magic_clear
 
 will define %cl as a new name for %clear.
 
 Type %magic for more information, including a list of all available
 magic functions at any time and their docstrings. You can also type
 %magic_function_name? (see sec. 6.4 <#sec:dyn-object-info> for
 information on the '?' system) to get information about any particular
 magic function you are interested in.
 
 The API documentation for the :mod:`IPython.Magic` module contains the full
 docstrings of all currently available magic commands.
 
 
 Access to the standard Python help
 ----------------------------------
 
 As of Python 2.1, a help system is available with access to object docstrings
 and the Python manuals. Simply type 'help' (no quotes) to access it. You can
 also type help(object) to obtain information about a given object, and
 help('keyword') for information on a keyword. As noted :ref:`here
 <accessing_help>`, you need to properly configure your environment variable
 PYTHONDOCS for this feature to work correctly.
 
 .. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
 
 Typing ?word or word? prints detailed information about an object. If
 certain strings in the object are too long (docstrings, code, etc.) they
 get snipped in the center for brevity. This system gives access variable
 types and values, full source code for any object (if available),
 function prototypes and other useful information.
 
 Typing ??word or word?? gives access to the full information without
 snipping long strings. Long strings are sent to the screen through the
 less pager if longer than the screen and printed otherwise. On systems
 lacking the less command, IPython uses a very basic internal pager.
 
 The following magic functions are particularly useful for gathering
 information about your working environment. You can get more details by
 typing %magic or querying them individually (use %function_name? with or
 without the %), this is just a summary:
 
     * **%pdoc <object>**: Print (or run through a pager if too long) the
       docstring for an object. If the given object is a class, it will
       print both the class and the constructor docstrings.
     * **%pdef <object>**: Print the definition header for any callable
       object. If the object is a class, print the constructor information.
     * **%psource <object>**: Print (or run through a pager if too long)
       the source code for an object.
     * **%pfile <object>**: Show the entire source file where an object was
       defined via a pager, opening it at the line where the object
       definition begins.
     * **%who/%whos**: These functions give information about identifiers
       you have defined interactively (not things you loaded or defined
       in your configuration files). %who just prints a list of
       identifiers and %whos prints a table with some basic details about
       each identifier.
 
 Note that the dynamic object information functions (?/??, %pdoc, %pfile,
 %pdef, %psource) give you access to documentation even on things which
 are not really defined as separate identifiers. Try for example typing
 {}.get? or after doing import os, type os.path.abspath??.
 
 
 .. _readline:
 
 Readline-based features
 -----------------------
 
 These features require the GNU readline library, so they won't work if
 your Python installation lacks readline support. We will first describe
 the default behavior IPython uses, and then how to change it to suit
 your preferences.
 
 
 Command line completion
 +++++++++++++++++++++++
 
 At any time, hitting TAB will complete any available python commands or
 variable names, and show you a list of the possible completions if
 there's no unambiguous one. It will also complete filenames in the
 current directory if no python names match what you've typed so far.
 
 
 Search command history
 ++++++++++++++++++++++
 
 IPython provides two ways for searching through previous input and thus
 reduce the need for repetitive typing:
 
    1. Start typing, and then use Ctrl-p (previous,up) and Ctrl-n
       (next,down) to search through only the history items that match
       what you've typed so far. If you use Ctrl-p/Ctrl-n at a blank
       prompt, they just behave like normal arrow keys.
    2. Hit Ctrl-r: opens a search prompt. Begin typing and the system
       searches your history for lines that contain what you've typed so
       far, completing as much as it can.
 
 
 Persistent command history across sessions
 ++++++++++++++++++++++++++++++++++++++++++
 
 IPython will save your input history when it leaves and reload it next
 time you restart it. By default, the history file is named
 $IPYTHONDIR/history, but if you've loaded a named profile,
 '-PROFILE_NAME' is appended to the name. This allows you to keep
 separate histories related to various tasks: commands related to
 numerical work will not be clobbered by a system shell history, for
 example.
 
 
 Autoindent
 ++++++++++
 
 IPython can recognize lines ending in ':' and indent the next line,
 while also un-indenting automatically after 'raise' or 'return'.
 
 This feature uses the readline library, so it will honor your ~/.inputrc
 configuration (or whatever file your INPUTRC variable points to). Adding
 the following lines to your .inputrc file can make indenting/unindenting
 more convenient (M-i indents, M-u unindents)::
 
     $if Python 
     "\M-i": "    "  
     "\M-u": "\d\d\d\d"  
     $endif
 
 Note that there are 4 spaces between the quote marks after "M-i" above.
 
 Warning: this feature is ON by default, but it can cause problems with
 the pasting of multi-line indented code (the pasted code gets
 re-indented on each line). A magic function %autoindent allows you to
 toggle it on/off at runtime. You can also disable it permanently on in
 your ipythonrc file (set autoindent 0).
 
 
 Customizing readline behavior
 +++++++++++++++++++++++++++++
 
 All these features are based on the GNU readline library, which has an
 extremely customizable interface. Normally, readline is configured via a
 file which defines the behavior of the library; the details of the
 syntax for this can be found in the readline documentation available
 with your system or on the Internet. IPython doesn't read this file (if
 it exists) directly, but it does support passing to readline valid
 options via a simple interface. In brief, you can customize readline by
 setting the following options in your ipythonrc configuration file (note
 that these options can not be specified at the command line):
 
     * **readline_parse_and_bind**: this option can appear as many times as
       you want, each time defining a string to be executed via a
       readline.parse_and_bind() command. The syntax for valid commands
       of this kind can be found by reading the documentation for the GNU
       readline library, as these commands are of the kind which readline
       accepts in its configuration file.
     * **readline_remove_delims**: a string of characters to be removed
       from the default word-delimiters list used by readline, so that
       completions may be performed on strings which contain them. Do not
       change the default value unless you know what you're doing.
     * **readline_omit__names**: when tab-completion is enabled, hitting
       <tab> after a '.' in a name will complete all attributes of an
       object, including all the special methods whose names include
       double underscores (like __getitem__ or __class__). If you'd
       rather not see these names by default, you can set this option to
       1. Note that even when this option is set, you can still see those
       names by explicitly typing a _ after the period and hitting <tab>:
       'name._<tab>' will always complete attribute names starting with '_'.
       
       This option is off by default so that new users see all
       attributes of any objects they are dealing with.
 
 You will find the default values along with a corresponding detailed
 explanation in your ipythonrc file.
 
 
 Session logging and restoring
 -----------------------------
 
 You can log all input from a session either by starting IPython with the
 command line switches -log or -logfile (see :ref:`here <command_line_options>`)
 or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
 restoring the state of a previous session. This feature is not quite
 perfect, but can still be useful in many cases.
 
 The log files can also be used as a way to have a permanent record of
 any code you wrote while experimenting. Log files are regular text files
 which you can later open in your favorite text editor to extract code or
 to 'clean them up' before using them to replay a session.
 
 The %logstart function for activating logging in mid-session is used as
 follows:
 
 %logstart [log_name [log_mode]]
 
 If no name is given, it defaults to a file named 'log' in your
 IPYTHONDIR directory, in 'rotate' mode (see below).
 
 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
 history up to that point and then continues logging.
 
 %logstart takes a second optional parameter: logging mode. This can be
 one of (note that the modes are given unquoted):
 
     * [over:] overwrite existing log_name.
     * [backup:] rename (if exists) to log_name~ and start log_name.
     * [append:] well, that says it.
     * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
 
 The %logoff and %logon functions allow you to temporarily stop and
 resume logging to a file which had previously been started with
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
 .. _system_shell_access:
 
 System shell access
 -------------------
 
 Any input line beginning with a ! character is passed verbatim (minus
 the !, of course) to the underlying operating system. For example,
 typing !ls will run 'ls' in the current directory.
 
 Manual capture of command output
 --------------------------------
 
 If the input line begins with two exclamation marks, !!, the command is
 executed but its output is captured and returned as a python list, split
 on newlines. Any output sent by the subprocess to standard error is
 printed separately, so that the resulting list only captures standard
 output. The !! syntax is a shorthand for the %sx magic command.
 
 Finally, the %sc magic (short for 'shell capture') is similar to %sx,
 but allowing more fine-grained control of the capture details, and
 storing the result directly into a named variable. The direct use of 
 %sc is now deprecated, and you should ise the ``var = !cmd`` syntax
 instead.
 
 IPython also allows you to expand the value of python variables when
 making system calls. Any python variable or expression which you prepend
 with $ will get expanded before the system call is made::
 
     In [1]: pyvar='Hello world' 
     In [2]: !echo "A python variable: $pyvar"  
     A python variable: Hello world
 
 If you want the shell to actually see a literal $, you need to type it
 twice::
 
     In [3]: !echo "A system variable: $$HOME"  
     A system variable: /home/fperez
 
 You can pass arbitrary expressions, though you'll need to delimit them
 with {} if there is ambiguity as to the extent of the expression::
 
     In [5]: x=10 
     In [6]: y=20 
     In [13]: !echo $x+y  
     10+y  
     In [7]: !echo ${x+y}  
     30
 
 Even object attributes can be expanded::
 
     In [12]: !echo $sys.argv  
     [/home/fperez/usr/bin/ipython]
 
 
 System command aliases
 ----------------------
 
 The %alias magic function and the alias option in the ipythonrc
 configuration file allow you to define magic functions which are in fact
 system shell commands. These aliases can have parameters.
 
 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
 Then, typing '%alias_name params' will execute the system command 'cmd
 params' (from your underlying operating system).
 
 You can also define aliases with parameters using %s specifiers (one per
 parameter). The following example defines the %parts function as an
 alias to the command 'echo first %s second %s' where each %s will be
 replaced by a positional parameter to the call to %parts::
 
     In [1]: alias parts echo first %s second %s 
     In [2]: %parts A B 
     first A second B  
     In [3]: %parts A  
     Incorrect number of arguments: 2 expected.  
     parts is an alias to: 'echo first %s second %s'
 
 If called with no parameters, %alias prints the table of currently
 defined aliases.
 
 The %rehash/rehashx magics allow you to load your entire $PATH as
 ipython aliases. See their respective docstrings (or sec. 6.2
 <#sec:magic> for further details).
 
 
 .. _dreload:
 
 Recursive reload
 ----------------
 
 The dreload function does a recursive reload of a module: changes made
 to the module since you imported will actually be available without
 having to exit.
 
 
 Verbose and colored exception traceback printouts
 -------------------------------------------------
 
 IPython provides the option to see very detailed exception tracebacks,
 which can be especially useful when debugging large programs. You can
 run any Python file with the %run function to benefit from these
 detailed tracebacks. Furthermore, both normal and verbose tracebacks can
 be colored (if your terminal supports it) which makes them much easier
 to parse visually.
 
 See the magic xmode and colors functions for details (just type %magic).
 
 These features are basically a terminal version of Ka-Ping Yee's cgitb
 module, now part of the standard Python library.
 
 
 .. _input_caching:
 
 Input caching system
 --------------------
 
 IPython offers numbered prompts (In/Out) with input and output caching
 (also referred to as 'input history'). All input is saved and can be 
 retrieved as variables (besides the usual arrow key recall), in 
 addition to the %rep magic command that brings a history entry
 up for editing on the next  command line.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 _i: stores previous input. _ii: next previous. _iii: next-next previous.
 _ih : a list of all input _ih[n] is the input from line n and this list
 is aliased to the global variable In. If you overwrite In with a
 variable of your own, you can remake the assignment to the internal list
 with a simple 'In=_ih'.
 
 Additionally, global variables named _i<n> are dynamically created (<n>
 being the prompt counter), such that
 _i<n> == _ih[<n>] == In[<n>].
 
 For example, what you typed at prompt 14 is available as _i14, _ih[14]
 and In[14].
 
 This allows you to easily cut and paste multi line interactive prompts
 by printing them out: they print like a clean string, without prompt
 characters. You can also manipulate them like regular variables (they
 are strings), modify or exec them (typing 'exec _i9' will re-execute the
 contents of input prompt 9, 'exec In[9:14]+In[18]' will re-execute lines
 9 through 13 and line 18).
 
 You can also re-execute multiple lines of input easily by using the
 magic %macro function (which automates the process and allows
 re-execution without having to type 'exec' every time). The macro system
 also allows you to re-execute previous lines which include magic
 function calls (which require special processing). Type %macro? or see
 sec. 6.2 <#sec:magic> for more details on the macro system.
 
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
 You can also search ('grep') through your history by typing 
 '%hist -g somestring'. This also searches through the so called *shadow history*,
 which remembers all the commands (apart from multiline code blocks)
 you have ever entered. Handy for searching for svn/bzr URL's, IP adrresses
 etc. You can bring shadow history entries listed by '%hist -g' up for editing 
 (or re-execution by just pressing ENTER) with %rep command. Shadow history 
 entries are not available as _iNUMBER variables, and they are identified by 
 the '0' prefix in %hist -g output. That is, history entry 12 is a normal 
 history entry, but 0231 is a shadow history entry.
 
 Shadow history was added because the readline history is inherently very
 unsafe - if you have multiple IPython sessions open, the last session 
 to close will overwrite the history of previountly closed session. Likewise,
 if a crash occurs, history is never saved, whereas shadow history entries
 are added after entering every command (so a command executed 
 in another IPython session is immediately available in other IPython 
 sessions that are open).
 
 To conserve space, a command can exist in shadow history only once - it doesn't 
 make sense to store a common line like "cd .." a thousand times. The idea is 
 mainly to provide a reliable place where valuable, hard-to-remember commands can 
 always be retrieved, as opposed to providing an exact sequence of commands
 you have entered in actual order.
 
 Because shadow history has all the commands you have ever executed,
 time taken by %hist -g will increase oven time. If it ever starts to take
 too long (or it ends up containing sensitive information like passwords), 
 clear the shadow history by `%clear shadow_nuke`.
 
 Time taken to add entries to shadow history should be negligible, but 
 in any case, if you start noticing performance degradation after using 
 IPython for a long time (or running a script that floods the shadow history!), 
 you can 'compress' the shadow history by executing 
 `%clear shadow_compress`. In practice, this should never be necessary
 in normal use.
 
 .. _output_caching:
 
 Output caching system
 ---------------------
 
 For output that is returned from actions, a system similar to the input
 cache exists but using _ instead of _i. Only actions that produce a
 result (NOT assignments, for example) are cached. If you are familiar
 with Mathematica, IPython's _ variables behave exactly like
 Mathematica's % variables.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 
     * [_] (a single underscore) : stores previous output, like Python's
       default interpreter.
     * [__] (two underscores): next previous.
     * [___] (three underscores): next-next previous.
 
 Additionally, global variables named _<n> are dynamically created (<n>
 being the prompt counter), such that the result of output <n> is always
 available as _<n> (don't use the angle brackets, just the number, e.g.
 _21).
 
 These global variables are all stored in a global dictionary (not a
 list, since it only has entries for lines which returned a result)
 available under the names _oh and Out (similar to _ih and In). So the
 output from line 12 can be obtained as _12, Out[12] or _oh[12]. If you
 accidentally overwrite the Out variable you can recover it by typing
 'Out=_oh' at the prompt.
 
 This system obviously can potentially put heavy memory demands on your
 system, since it prevents Python's garbage collector from removing any
 previously computed results. You can control how many results are kept
 in memory with the option (at the command line or in your ipythonrc
 file) cache_size. If you set it to 0, the whole system is completely
 disabled and the prompts revert to the classic '>>>' of normal Python.
 
 
 Directory history
 -----------------
 
 Your history of visited directories is kept in the global list _dh, and
 the magic %cd command can be used to go to any entry in that list. The
 %dhist command allows you to view this history. Do ``cd -<TAB`` to
 conventiently view the directory history.
 
 
 Automatic parentheses and quotes
 --------------------------------
 
 These features were adapted from Nathan Gray's LazyPython. They are
 meant to allow less typing for common situations.
 
 
 Automatic parentheses
 ---------------------
 
 Callable objects (i.e. functions, methods, etc) can be invoked like this
 (notice the commas between the arguments)::
 
     >>> callable_ob arg1, arg2, arg3
 
 and the input will be translated to this::
 
     -> callable_ob(arg1, arg2, arg3)
 
 You can force automatic parentheses by using '/' as the first character
 of a line. For example::
 
     >>> /globals # becomes 'globals()'
 
 Note that the '/' MUST be the first character on the line! This won't work::
 
     >>> print /globals # syntax error
 
 In most cases the automatic algorithm should work, so you should rarely
 need to explicitly invoke /. One notable exception is if you are trying
 to call a function with a list of tuples as arguments (the parenthesis
 will confuse IPython)::
 
     In [1]: zip (1,2,3),(4,5,6) # won't work
 
 but this will work::
 
     In [2]: /zip (1,2,3),(4,5,6) 
     ---> zip ((1,2,3),(4,5,6))  
     Out[2]= [(1, 4), (2, 5), (3, 6)]
 
 IPython tells you that it has altered your command line by displaying
 the new command line preceded by ->. e.g.::
 
     In [18]: callable list 
     ----> callable (list)
 
 
 Automatic quoting
 -----------------
 
 You can force automatic quoting of a function's arguments by using ','
 or ';' as the first character of a line. For example::
 
     >>> ,my_function /home/me # becomes my_function("/home/me")
 
 If you use ';' instead, the whole argument is quoted as a single string
 (while ',' splits on whitespace)::
 
     >>> ,my_function a b c # becomes my_function("a","b","c")
 
     >>> ;my_function a b c # becomes my_function("a b c")
 
 Note that the ',' or ';' MUST be the first character on the line! This
 won't work::
 
     >>> x = ,my_function /home/me # syntax error
 
 IPython as your default Python environment
 ==========================================
 
 Python honors the environment variable PYTHONSTARTUP and will execute at
 startup the file referenced by this variable. If you put at the end of
 this file the following two lines of code::
 
     import IPython 
     IPython.Shell.IPShell().mainloop(sys_exit=1)
 
 then IPython will be your working environment anytime you start Python.
 The sys_exit=1 is needed to have IPython issue a call to sys.exit() when
 it finishes, otherwise you'll be back at the normal Python '>>>'
 prompt.
 
 This is probably useful to developers who manage multiple Python
 versions and don't want to have correspondingly multiple IPython
 versions. Note that in this mode, there is no way to pass IPython any
 command-line options, as those are trapped first by Python itself.
 
 .. _Embedding:
 
 Embedding IPython
 =================
 
 It is possible to start an IPython instance inside your own Python
 programs. This allows you to evaluate dynamically the state of your
 code, operate with your variables, analyze them, etc. Note however that
 any changes you make to values while in the shell do not propagate back
 to the running code, so it is safe to modify your values because you
 won't break your code in bizarre ways by doing so.
 
 This feature allows you to easily have a fully functional python
 environment for doing object introspection anywhere in your code with a
 simple function call. In some cases a simple print statement is enough,
 but if you need to do more detailed analysis of a code fragment this
 feature can be very valuable.
 
 It can also be useful in scientific computing situations where it is
 common to need to do some automatic, computationally intensive part and
 then stop to look at data, plots, etc.
 Opening an IPython instance will give you full access to your data and
 functions, and you can resume program execution once you are done with
 the interactive part (perhaps to stop again later, as many times as
 needed).
 
 The following code snippet is the bare minimum you need to include in
 your Python programs for this to work (detailed examples follow later)::
 
     from IPython.Shell import IPShellEmbed
 
     ipshell = IPShellEmbed()
 
     ipshell() # this call anywhere in your program will start IPython
 
 You can run embedded instances even in code which is itself being run at
 the IPython interactive prompt with '%run <filename>'. Since it's easy
 to get lost as to where you are (in your top-level IPython or in your
 embedded one), it's a good idea in such cases to set the in/out prompts
 to something different for the embedded instances. The code examples
 below illustrate this.
 
 You can also have multiple IPython instances in your program and open
 them separately, for example with different options for data
 presentation. If you close and open the same instance multiple times,
 its prompt counters simply continue from each execution to the next.
 
 Please look at the docstrings in the Shell.py module for more details on
 the use of this system.
 
 The following sample file illustrating how to use the embedding
 functionality is provided in the examples directory as example-embed.py.
 It should be fairly self-explanatory::
 
 
     #!/usr/bin/env python
 
     """An example of how to embed an IPython shell into a running program.
 
     Please see the documentation in the IPython.Shell module for more details.
 
     The accompanying file example-embed-short.py has quick code fragments for
     embedding which you can cut and paste in your code once you understand how
     things work.
 
     The code in this file is deliberately extra-verbose, meant for learning."""
 
     # The basics to get you going:
 
     # IPython sets the __IPYTHON__ variable so you can know if you have nested
     # copies running.
 
     # Try running this code both at the command line and from inside IPython (with
     # %run example-embed.py)
     try:
         __IPYTHON__
     except NameError:
         nested = 0
         args = ['']
     else:
         print "Running nested copies of IPython."
         print "The prompts for the nested copy have been modified"
         nested = 1
         # what the embedded instance will see as sys.argv:
         args = ['-pi1','In <\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
 
     # Now create an instance of the embeddable shell. The first argument is a
     # string with options exactly as you would type them if you were starting
     # IPython at the system command line. Any parameters you want to define for
     # configuration can thus be specified here.
     ipshell = IPShellEmbed(args,
                            banner = 'Dropping into IPython',
                            exit_msg = 'Leaving Interpreter, back to program.')
 
     # Make a second instance, you can have as many as you want.
     if nested:
         args[1] = 'In2<\\#>'
     else:
         args = ['-pi1','In2<\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
     ipshell2 = IPShellEmbed(args,banner = 'Second IPython instance.')
 
     print '\nHello. This is printed from the main controller program.\n'
 
     # You can then call ipshell() anywhere you need it (with an optional
     # message):
     ipshell('***Called from top level. '
             'Hit Ctrl-D to exit interpreter and continue program.\n'
             'Note that if you use %kill_embedded, you can fully deactivate\n'
             'This embedded instance so it will never turn on again')
 
     print '\nBack in caller program, moving along...\n'
 
     #---------------------------------------------------------------------------
     # More details:
 
     # IPShellEmbed instances don't print the standard system banner and
     # messages. The IPython banner (which actually may contain initialization
     # messages) is available as <instance>.IP.BANNER in case you want it.
 
     # IPShellEmbed instances print the following information everytime they
     # start:
 
     # - A global startup banner.
 
     # - A call-specific header string, which you can use to indicate where in the
     # execution flow the shell is starting.
 
     # They also print an exit message every time they exit.
 
     # Both the startup banner and the exit message default to None, and can be set
     # either at the instance constructor or at any other time with the
     # set_banner() and set_exit_msg() methods.
 
     # The shell instance can be also put in 'dummy' mode globally or on a per-call
     # basis. This gives you fine control for debugging without having to change
     # code all over the place.
 
     # The code below illustrates all this.
 
 
     # This is how the global banner and exit_msg can be reset at any point
     ipshell.set_banner('Entering interpreter - New Banner')
     ipshell.set_exit_msg('Leaving interpreter - New exit_msg')
 
     def foo(m):
         s = 'spam'
         ipshell('***In foo(). Try @whos, or print s or m:')
         print 'foo says m = ',m
 
     def bar(n):
         s = 'eggs'
         ipshell('***In bar(). Try @whos, or print s or n:')
         print 'bar says n = ',n
 
     # Some calls to the above functions which will trigger IPython:
     print 'Main program calling foo("eggs")\n'
     foo('eggs')
 
     # The shell can be put in 'dummy' mode where calls to it silently return. This
     # allows you, for example, to globally turn off debugging for a program with a
     # single call.
     ipshell.set_dummy_mode(1)
     print '\nTrying to call IPython which is now "dummy":'
     ipshell()
     print 'Nothing happened...'
     # The global 'dummy' mode can still be overridden for a single call
     print '\nOverriding dummy mode manually:'
     ipshell(dummy=0)
 
     # Reactivate the IPython shell
     ipshell.set_dummy_mode(0)
 
     print 'You can even have multiple embedded instances:'
     ipshell2()
 
     print '\nMain program calling bar("spam")\n'
     bar('spam')
 
     print 'Main program finished. Bye!'
 
     #********************** End of file <example-embed.py> ***********************
 
 Once you understand how the system functions, you can use the following
 code fragments in your programs which are ready for cut and paste::
 
 
     """Quick code snippets for embedding IPython into other programs.
 
     See example-embed.py for full details, this file has the bare minimum code for
     cut and paste use once you understand how to use the system."""
 
     #---------------------------------------------------------------------------
     # This code loads IPython but modifies a few things if it detects it's running
     # embedded in another IPython session (helps avoid confusion)
 
     try:
         __IPYTHON__
     except NameError:
         argv = ['']
         banner = exit_msg = ''
     else:
         # Command-line options for IPython (a list like sys.argv)
         argv = ['-pi1','In <\\#>:','-pi2','   .\\D.:','-po','Out<\\#>:']
         banner = '*** Nested interpreter ***'
         exit_msg = '*** Back in main IPython ***'
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
     # Now create the IPython shell instance. Put ipshell() anywhere in your code
     # where you want it to open.
     ipshell = IPShellEmbed(argv,banner=banner,exit_msg=exit_msg)
 
     #---------------------------------------------------------------------------
     # This code will load an embeddable IPython shell always with no changes for
     # nested embededings.
 
     from IPython.Shell import IPShellEmbed
     ipshell = IPShellEmbed()
     # Now ipshell() will open IPython anywhere in the code.
 
     #---------------------------------------------------------------------------
     # This code loads an embeddable shell only if NOT running inside
     # IPython. Inside IPython, the embeddable shell variable ipshell is just a
     # dummy function.
 
     try:
         __IPYTHON__
     except NameError:
         from IPython.Shell import IPShellEmbed
         ipshell = IPShellEmbed()
         # Now ipshell() will open IPython anywhere in the code
     else:
         # Define a dummy ipshell() so the same code doesn't crash inside an
         # interactive IPython
         def ipshell(): pass
 
     #******************* End of file <example-embed-short.py> ********************
 
 Using the Python debugger (pdb)
 ===============================
 
 Running entire programs via pdb
 -------------------------------
 
 pdb, the Python debugger, is a powerful interactive debugger which
 allows you to step through code, set breakpoints, watch variables,
 etc.  IPython makes it very easy to start any script under the control
 of pdb, regardless of whether you have wrapped it into a 'main()'
 function or not. For this, simply type '%run -d myscript' at an
 IPython prompt. See the %run command's documentation (via '%run?' or
 in Sec. magic_ for more details, including how to control where pdb
 will stop execution first.
 
 For more information on the use of the pdb debugger, read the included
 pdb.doc file (part of the standard Python distribution). On a stock
 Linux system it is located at /usr/lib/python2.3/pdb.doc, but the
 easiest way to read it is by using the help() function of the pdb module
 as follows (in an IPython prompt):
 
 In [1]: import pdb 
 In [2]: pdb.help()
 
 This will load the pdb.doc document in a file viewer for you automatically.
 
 
 Automatic invocation of pdb on exceptions
 -----------------------------------------
 
 IPython, if started with the -pdb option (or if the option is set in
 your rc file) can call the Python pdb debugger every time your code
 triggers an uncaught exception. This feature
 can also be toggled at any time with the %pdb magic command. This can be
 extremely useful in order to find the origin of subtle bugs, because pdb
 opens up at the point in your code which triggered the exception, and
 while your program is at this point 'dead', all the data is still
 available and you can walk up and down the stack frame and understand
 the origin of the problem.
 
 Furthermore, you can use these debugging facilities both with the
 embedded IPython mode and without IPython at all. For an embedded shell
 (see sec. Embedding_), simply call the constructor with
 '-pdb' in the argument string and automatically pdb will be called if an
 uncaught exception is triggered by your code.
 
 For stand-alone use of the feature in your programs which do not use
 IPython at all, put the following lines toward the top of your 'main'
 routine::
 
     import sys
     from IPython.core import ultratb
     sys.excepthook = ultratb.FormattedTB(mode='Verbose',
     color_scheme='Linux', call_pdb=1)
 
 The mode keyword can be either 'Verbose' or 'Plain', giving either very
 detailed or normal tracebacks respectively. The color_scheme keyword can
 be one of 'NoColor', 'Linux' (default) or 'LightBG'. These are the same
 options which can be set in IPython with -colors and -xmode.
 
 This will give any of your programs detailed, colored tracebacks with
 automatic invocation of pdb.
 
 
 Extensions for syntax processing
 ================================
 
 This isn't for the faint of heart, because the potential for breaking
 things is quite high. But it can be a very powerful and useful feature.
 In a nutshell, you can redefine the way IPython processes the user input
 line to accept new, special extensions to the syntax without needing to
 change any of IPython's own code.
 
 In the IPython/extensions directory you will find some examples
 supplied, which we will briefly describe now. These can be used 'as is'
 (and both provide very useful functionality), or you can use them as a
 starting point for writing your own extensions.
 
 
 Pasting of code starting with '>>> ' or '... '
 ----------------------------------------------
 
 In the python tutorial it is common to find code examples which have
 been taken from real python sessions. The problem with those is that all
 the lines begin with either '>>> ' or '... ', which makes it impossible
 to paste them all at once. One must instead do a line by line manual
 copying, carefully removing the leading extraneous characters.
 
 This extension identifies those starting characters and removes them
 from the input automatically, so that one can paste multi-line examples
 directly into IPython, saving a lot of time. Please look at the file
 InterpreterPasteInput.py in the IPython/extensions directory for details
 on how this is done.
 
 IPython comes with a special profile enabling this feature, called
 tutorial. Simply start IPython via 'ipython -p tutorial' and the feature
 will be available. In a normal IPython session you can activate the
 feature by importing the corresponding module with:
 In [1]: import IPython.extensions.InterpreterPasteInput
 
 The following is a 'screenshot' of how things work when this extension
 is on, copying an example from the standard tutorial::
 
     IPython profile: tutorial 
 
     *** Pasting of code with ">>>" or "..." has been enabled. 
 
     In [1]: >>> def fib2(n): # return Fibonacci series up to n 
        ...: ...     """Return a list containing the Fibonacci series up to
     n.""" 
        ...: ...     result = [] 
        ...: ...     a, b = 0, 1 
        ...: ...     while b < n: 
        ...: ...         result.append(b)    # see below 
        ...: ...         a, b = b, a+b 
        ...: ...     return result 
        ...: 
 
     In [2]: fib2(10) 
     Out[2]: [1, 1, 2, 3, 5, 8]
 
 Note that as currently written, this extension does not recognize
 IPython's prompts for pasting. Those are more complicated, since the
 user can change them very easily, they involve numbers and can vary in
 length. One could however extract all the relevant information from the
 IPython instance and build an appropriate regular expression. This is
 left as an exercise for the reader.
 
 
 Input of physical quantities with units
 ---------------------------------------
 
 The module PhysicalQInput allows a simplified form of input for physical
 quantities with units. This file is meant to be used in conjunction with
 the PhysicalQInteractive module (in the same directory) and
 Physics.PhysicalQuantities from Konrad Hinsen's ScientificPython
 (http://dirac.cnrs-orleans.fr/ScientificPython/).
 
 The Physics.PhysicalQuantities module defines PhysicalQuantity objects,
 but these must be declared as instances of a class. For example, to
 define v as a velocity of 3 m/s, normally you would write::
 
     In [1]: v = PhysicalQuantity(3,'m/s')
 
 Using the PhysicalQ_Input extension this can be input instead as: 
 In [1]: v = 3 m/s
 which is much more convenient for interactive use (even though it is
 blatantly invalid Python syntax).
 
 The physics profile supplied with IPython (enabled via 'ipython -p
 physics') uses these extensions, which you can also activate with:
 
 from math import * # math MUST be imported BEFORE PhysicalQInteractive 
 from IPython.extensions.PhysicalQInteractive import * 
 import IPython.extensions.PhysicalQInput
 
+.. _gui_support:
 
-Threading support
-=================
+GUI event loop support support
+==============================
+
+.. versionadded:: 0.11
+    The ``%gui`` magic and :mod:`IPython.lib.inputhook`.
+
+IPython has excellent support for working interactively with Graphical User
+Interface (GUI) toolkits, such as wxPython, PyQt4, PyGTK and Tk. This is
+implemented using Python's builtin ``PyOSInputHook`` hook. This implementation
+is extremely robust compared to our previous threaded based version. The
+advantages of
+
+* GUIs can be enabled and disabled dynamically at runtime.
+* The active GUI can be switched dynamically at runtime.
+* In some cases, multiple GUIs can run simultaneously with no problems.
+* There is a developer API in :mod:`IPython.lib.inputhook` for customizing 
+  all of these things.
+
+For users, enabling GUI event loop integration is simple.  You simple use the
+``%gui`` magic as follows::
+
+    %gui [-a] [GUINAME]
+
+With no arguments, ``%gui`` removes all GUI support.  Valid ``GUINAME``
+arguments are ``wx``, ``qt4``, ``gtk`` and ``tk``.  The ``-a`` option will
+create and return a running application object for the selected GUI toolkit.
+
+This to use wxPython interactively and create a running :class:`wx.App`
+object, do::
+
+    %gui -a wx
+
+For information on IPython's Matplotlib integration (and the ``pylab`` mode)
+see :ref:`this section <matplotlib_support>`.
 
-WARNING: The threading support is still somewhat experimental, and it
-has only seen reasonable testing under Linux. Threaded code is
-particularly tricky to debug, and it tends to show extremely
-platform-dependent behavior. Since I only have access to Linux machines,
-I will have to rely on user's experiences and assistance for this area
-of IPython to improve under other platforms.
-
-IPython, via the -gthread , -qthread, -q4thread and -wthread options
-(described in Sec. `Threading options`_), can run in
-multithreaded mode to support pyGTK, Qt3, Qt4 and WXPython applications
-respectively. These GUI toolkits need to control the python main loop of
-execution, so under a normal Python interpreter, starting a pyGTK, Qt3,
-Qt4 or WXPython application will immediately freeze the shell.
-
-IPython, with one of these options (you can only use one at a time),
-separates the graphical loop and IPython's code execution run into
-different threads. This allows you to test interactively (with %run, for
-example) your GUI code without blocking.
-
-A nice mini-tutorial on using IPython along with the Qt Designer
-application is available at the SciPy wiki:
-http://www.scipy.org/Cookbook/Matplotlib/Qt_with_IPython_and_Designer.
-
-
-Tk issues
----------
-
-As indicated in Sec. `Threading options`_, a special -tk option is
-provided to try and allow Tk graphical applications to coexist
-interactively with WX, Qt or GTK ones. Whether this works at all,
-however, is very platform and configuration dependent. Please
-experiment with simple test cases before committing to using this
-combination of Tk and GTK/Qt/WX threading in a production environment.
-
-
-I/O pitfalls
-------------
-
-Be mindful that the Python interpreter switches between threads every
-$N$ bytecodes, where the default value as of Python 2.3 is $N=100.$ This
-value can be read by using the sys.getcheckinterval() function, and it
-can be reset via sys.setcheckinterval(N). This switching of threads can
-cause subtly confusing effects if one of your threads is doing file I/O.
-In text mode, most systems only flush file buffers when they encounter a
-'\n'. An instruction as simple as:: 
-
-  print >> filehandle, ''hello world''
-
-actually consists of several bytecodes, so it is possible that the
-newline does not reach your file before the next thread switch.
-Similarly, if you are writing to a file in binary mode, the file won't
-be flushed until the buffer fills, and your other thread may see
-apparently truncated files.
-
-For this reason, if you are using IPython's thread support and have (for
-example) a GUI application which will read data generated by files
-written to from the IPython thread, the safest approach is to open all
-of your files in unbuffered mode (the third argument to the file/open
-function is the buffering value)::
-
-  filehandle = open(filename,mode,0)
-
-This is obviously a brute force way of avoiding race conditions with the
-file buffering. If you want to do it cleanly, and you have a resource
-which is being shared by the interactive IPython loop and your GUI
-thread, you should really handle it with thread locking and
-syncrhonization properties. The Python documentation discusses these.
+For developers that want to use IPython's GUI event loop integration in
+the form of a library, the capabilities are exposed in library form
+in the :mod:`IPython.lib.inputhook`.  Interested developers should see the
+module docstrings for more information.
+
+.. _matplotlib_support:
+
+Plotting with matplotlib
+========================
+
+
+`Matplotlib`_ provides high quality 2D and
+3D plotting for Python. Matplotlib can produce plots on screen using a variety
+of GUI toolkits, including Tk, PyGTK, PyQt4 and wxPython. It also provides a
+number of commands useful for scientific computing, all with a syntax
+compatible with that of the popular Matlab program.
+
+Many IPython users have come to rely on IPython's ``-pylab`` mode which
+automates the integration of Matplotlib with IPython. We are still in the
+process of working with the Matplotlib developers to finalize the new pylab
+API, but for now you can use Matplotlib interactively using the following
+commands::
+
+    %gui -a wx
+    import matplotlib
+    matplotlib.use('wxagg')
+    from matplotlib import pylab
+    pylab.interactive(True)
+
+All of this will soon be automated as Matplotlib beings to include
+new logic that uses our new GUI support.
 
 .. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
 
 IPython ships with a basic system for running scripts interactively in
 sections, useful when presenting code to audiences. A few tags embedded
 in comments (so that the script remains valid Python code) divide a file
 into separate blocks, and the demo can be run one block at a time, with
 IPython printing (with syntax highlighting) the block before executing
 it, and returning to the interactive prompt after each block. The
 interactive namespace is updated after each block is run with the
 contents of the demo's namespace.
 
 This allows you to show a piece of code, run it and then execute
 interactively commands based on the variables just created. Once you
 want to continue, you simply execute the next block of the demo. The
 following listing shows the markup necessary for dividing a script into
 sections for execution as a demo::
 
 
     """A simple interactive demo to illustrate the use of IPython's Demo class.
 
     Any python script can be run as a demo, but that does little more than showing
     it on-screen, syntax-highlighted in one shot.  If you add a little simple
     markup, you can stop at specified intervals and return to the ipython prompt,
     resuming execution later.
     """
 
     print 'Hello, welcome to an interactive IPython demo.'
     print 'Executing this block should require confirmation before proceeding,'
     print 'unless auto_all has been set to true in the demo object'
 
     # The mark below defines a block boundary, which is a point where IPython will
     # stop execution and return to the interactive prompt.
     # Note that in actual interactive execution, 
     # <demo> --- stop ---
 
     x = 1
     y = 2
 
     # <demo> --- stop ---
 
     # the mark below makes this block as silent
     # <demo> silent
 
     print 'This is a silent block, which gets executed but not printed.'
 
     # <demo> --- stop ---
     # <demo> auto
     print 'This is an automatic block.'
     print 'It is executed without asking for confirmation, but printed.'
     z = x+y
 
     print 'z=',x
 
     # <demo> --- stop ---
     # This is just another normal block.
     print 'z is now:', z
 
     print 'bye!'
 
 In order to run a file as a demo, you must first make a Demo object out
 of it. If the file is named myscript.py, the following code will make a
 demo::
 
     from IPython.demo import Demo
 
     mydemo = Demo('myscript.py')
 
 This creates the mydemo object, whose blocks you run one at a time by
 simply calling the object with no arguments. If you have autocall active
 in IPython (the default), all you need to do is type::
 
     mydemo 
 
 and IPython will call it, executing each block. Demo objects can be
 restarted, you can move forward or back skipping blocks, re-execute the
 last block, etc. Simply use the Tab key on a demo object to see its
 methods, and call '?' on them to see their docstrings for more usage
 details. In addition, the demo module itself contains a comprehensive
 docstring, which you can access via::
 
     from IPython import demo
 
     demo?
 
 Limitations: It is important to note that these demos are limited to
 fairly simple uses. In particular, you can not put division marks in
 indented code (loops, if statements, function definitions, etc.)
 Supporting something like this would basically require tracking the
 internal execution state of the Python interpreter, so only top-level
 divisions are allowed. If you want to be able to open an IPython
 instance at an arbitrary point in a program, you can use IPython's
 embedding facilities, described in detail in Sec. 9
 
+.. [Matplotlib] Matplotlib. http://matplotlib.sourceforge.net
 
-.. _Matplotlib support:
-
-Plotting with matplotlib
-========================
-
-The matplotlib library (http://matplotlib.sourceforge.net
-http://matplotlib.sourceforge.net) provides high quality 2D plotting for
-Python. Matplotlib can produce plots on screen using a variety of GUI
-toolkits, including Tk, GTK and WXPython. It also provides a number of
-commands useful for scientific computing, all with a syntax compatible
-with that of the popular Matlab program.
-
-IPython accepts the special option -pylab (see :ref:`here
-<command_line_options>`). This configures it to support matplotlib, honoring
-the settings in the .matplotlibrc file. IPython will detect the user's choice
-of matplotlib GUI backend, and automatically select the proper threading model
-to prevent blocking. It also sets matplotlib in interactive mode and modifies
-%run slightly, so that any matplotlib-based script can be executed using %run
-and the final show() command does not block the interactive shell.
-
-The -pylab option must be given first in order for IPython to configure its
-threading mode. However, you can still issue other options afterwards. This
-allows you to have a matplotlib-based environment customized with additional
-modules using the standard IPython profile mechanism (see :ref:`here
-<profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
-ipythonrc-myprofile after configuring matplotlib.

docs/source/parallel/index.txt

@@ -1,16 +1,15 b''
 .. _parallel_index:
 
 ====================================
 Using IPython for parallel computing
 ====================================
 
 .. toctree::
    :maxdepth: 2
 
    parallel_intro.txt
    parallel_process.txt
    parallel_multiengine.txt
    parallel_task.txt
    parallel_mpi.txt
    parallel_security.txt
-   visionhpc.txt

docs/source/parallel/parallel_intro.txt

@@ -1,205 +1,237 b''
 .. _ip1par:
 
 ============================
 Overview and getting started
 ============================
 
 Introduction
 ============
 
 This section gives an overview of IPython's sophisticated and powerful
 architecture for parallel and distributed computing. This architecture
 abstracts out parallelism in a very general way, which enables IPython to
 support many different styles of parallelism including:
 
 * Single program, multiple data (SPMD) parallelism.
 * Multiple program, multiple data (MPMD) parallelism.
 * Message passing using MPI.
 * Task farming.
 * Data parallel.
 * Combinations of these approaches.
 * Custom user defined approaches.
 
 Most importantly, IPython enables all types of parallel applications to
 be developed, executed, debugged and monitored *interactively*. Hence,
 the ``I`` in IPython.  The following are some example usage cases for IPython:
 
 * Quickly parallelize algorithms that are embarrassingly parallel
   using a number of simple approaches.  Many simple things can be
   parallelized interactively in one or two lines of code.
 
 * Steer traditional MPI applications on a supercomputer from an
   IPython session on your laptop.
 
 * Analyze and visualize large datasets (that could be remote and/or
   distributed) interactively using IPython and tools like
   matplotlib/TVTK.
 
 * Develop, test and debug new parallel algorithms 
   (that may use MPI) interactively.
 
 * Tie together multiple MPI jobs running on different systems into
   one giant distributed and parallel system.
 
 * Start a parallel job on your cluster and then have a remote 	
   collaborator connect to it and pull back data into their 
   local IPython session for plotting and analysis.
 
 * Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
 =====================
 
 The IPython architecture consists of three components:
 
 * The IPython engine.
 * The IPython controller.
 * Various controller clients.
 
 These components live in the :mod:`IPython.kernel` package and are
 installed with IPython.  They do, however, have additional dependencies
 that must be installed.  For more information, see our
 :ref:`installation documentation <install_index>`.
 
 IPython engine
 ---------------
 
 The IPython engine is a Python instance that takes Python commands over a
 network connection. Eventually, the IPython engine will be a full IPython
 interpreter, but for now, it is a regular Python interpreter. The engine
 can also handle incoming and outgoing Python objects sent over a network
 connection.  When multiple engines are started, parallel and distributed
 computing becomes possible. An important feature of an IPython engine is
 that it blocks while user code is being executed. Read on for how the
 IPython controller solves this problem to expose a clean asynchronous API
 to the user.
 
 IPython controller
 ------------------
 
 The IPython controller provides an interface for working with a set of
 engines. At an general level, the controller is a process to which
 IPython engines can connect. For each connected engine, the controller
 manages a queue. All actions that can be performed on the engine go
 through this queue. While the engines themselves block when user code is
 run, the controller hides that from the user to provide a fully
 asynchronous interface to a set of engines. 
 
 .. note::
 
     Because the controller listens on a network port for engines to 
     connect to it, it must be started *before* any engines are started.  
 
-The controller also provides a single point of contact for users who wish
-to utilize the engines connected to the controller. There are different
-ways of working with a controller. In IPython these ways correspond to different interfaces that the controller is adapted to.  Currently we have two default interfaces to the controller: 
+The controller also provides a single point of contact for users who wish to
+utilize the engines connected to the controller. There are different ways of
+working with a controller. In IPython these ways correspond to different
+interfaces that the controller is adapted to. Currently we have two default
+interfaces to the controller:
 
 * The MultiEngine interface, which provides the simplest possible way of
   working with engines interactively.
-* The Task interface, which provides presents the engines as a load balanced
+* The Task interface, which presents the engines as a load balanced
   task farming system.
 
 Advanced users can easily add new custom interfaces to enable other
 styles of parallelism. 
 
 .. note:: 
 
 	A single controller and set of engines can be accessed 
 	through multiple interfaces simultaneously.  This opens the
 	door for lots of interesting things.  
 
 Controller clients
 ------------------
 
 For each controller interface, there is a corresponding client. These
 clients allow users to interact with a set of engines through the
 interface.  Here are the two default clients:
 
 * The :class:`MultiEngineClient` class.
 * The :class:`TaskClient` class.
 
 Security
 --------
 
-By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a capability based security model [Capability]_.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in.  If not, you can't.  
-
-In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named :file:`something.furl`.  The default location of these files is the :file:`~./ipython/security` directory.
-
-To connect and authenticate to the controller an engine or client simply needs to present an appropriate FURL (that was originally created by the controller) to the controller.  Thus, the FURL files need to be copied to a location where the clients and engines can find them.  Typically, this is the :file:`~./ipython/security` directory on the host where the client/engine is running (which could be a different host than the controller).  Once the FURL files are copied over, everything should work fine.
+By default (as long as `pyOpenSSL` is installed) all network connections
+between the controller and engines and the controller and clients are secure.
+What does this mean? First of all, all of the connections will be encrypted
+using SSL. Second, the connections are authenticated. We handle authentication
+in a capability based security model [Capability]_. In this model, a
+"capability (known in some systems as a key) is a communicable, unforgeable
+token of authority". Put simply, a capability is like a key to your house. If
+you have the key to your house, you can get in. If not, you can't.
+
+In our architecture, the controller is the only process that listens on
+network ports, and is thus responsible to creating these keys. In IPython,
+these keys are known as Foolscap URLs, or FURLs, because of the underlying
+network protocol we are using. As a user, you don't need to know anything
+about the details of these FURLs, other than that when the controller starts,
+it saves a set of FURLs to files named :file:`something.furl`. The default
+location of these files is the :file:`~./ipython/security` directory.
+
+To connect and authenticate to the controller an engine or client simply needs
+to present an appropriate FURL (that was originally created by the controller)
+to the controller. Thus, the FURL files need to be copied to a location where
+the clients and engines can find them. Typically, this is the
+:file:`~./ipython/security` directory on the host where the client/engine is
+running (which could be a different host than the controller). Once the FURL
+files are copied over, everything should work fine.
 
 Currently, there are three FURL files that the controller creates:
 
 ipcontroller-engine.furl
     This FURL file is the key that gives an engine the ability to connect
     to a controller.
 
 ipcontroller-tc.furl
     This FURL file is the key that a :class:`TaskClient` must use to
     connect to the task interface of a controller.
     
 ipcontroller-mec.furl
     This FURL file is the key that a :class:`MultiEngineClient` must use
     to connect to the multiengine interface of a controller.
 
 More details of how these FURL files are used are given below.
 
 A detailed description of the security model and its implementation in IPython
 can be found :ref:`here <parallelsecurity>`.
 
 Getting Started
 ===============
 
-To use IPython for parallel computing, you need to start one instance of
-the controller and one or more instances of the engine. Initially, it is best to simply start a controller and engines on a single host using the :command:`ipcluster` command.  To start a controller and 4 engines on you localhost, just do::
+To use IPython for parallel computing, you need to start one instance of the
+controller and one or more instances of the engine. Initially, it is best to
+simply start a controller and engines on a single host using the
+:command:`ipcluster` command. To start a controller and 4 engines on your
+localhost, just do::
 
     $ ipcluster local -n 4
 
-More details about starting the IPython controller and engines can be found :ref:`here <parallel_process>`
+More details about starting the IPython controller and engines can be found
+:ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
 are ready to use the engines to do something useful. To make sure
 everything is working correctly, try the following commands:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 	
 	In [4]: mec.get_ids()
 	Out[4]: [0, 1, 2, 3]
 	
 	In [5]: mec.execute('print "Hello World"')
 	Out[5]: 
 	<Results List>
 	[0] In [1]: print "Hello World"
 	[0] Out[1]: Hello World
 
 	[1] In [1]: print "Hello World"
 	[1] Out[1]: Hello World
 
 	[2] In [1]: print "Hello World"
 	[2] Out[1]: Hello World
 
 	[3] In [1]: print "Hello World"
 	[3] Out[1]: Hello World
 
-Remember, a client also needs to present a FURL file to the controller.  How does this happen?  When a multiengine client is created with no arguments, the client tries to find the corresponding FURL file in the local :file:`~./ipython/security` directory.  If it finds it, you are set.  If you have put the FURL file in a different location or it has a different name, create the client like this::
+Remember, a client also needs to present a FURL file to the controller. How
+does this happen? When a multiengine client is created with no arguments, the
+client tries to find the corresponding FURL file in the local
+:file:`~./ipython/security` directory. If it finds it, you are set. If you
+have put the FURL file in a different location or it has a different name,
+create the client like this::
 
     mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
 
 Same thing hold true of creating a task client::
 
     tc = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
-You are now ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+You are now ready to learn more about the :ref:`MultiEngine
+<parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the
+controller.
 
 .. note:: 
 
     Don't forget that the engine, multiengine client and task client all have
     *different* furl files. You must move *each* of these around to an
     appropriate location so that the engines and clients can use them to
     connect to the controller.
 
 .. [Capability] Capability-based security, http://en.wikipedia.org/wiki/Capability-based_security
 

docs/source/parallel/parallel_mpi.txt

@@ -1,157 +1,176 b''
 .. _parallelmpi:
 
 =======================
 Using MPI with IPython
 =======================
 
-Often, a parallel algorithm will require moving data between the engines.  One way of accomplishing this is by doing a pull and then a push using the multiengine client.  However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination.
+Often, a parallel algorithm will require moving data between the engines. One
+way of accomplishing this is by doing a pull and then a push using the
+multiengine client. However, this will be slow as all the data has to go
+through the controller to the client and then back through the controller, to
+its final destination.
 
-A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_.  IPython's parallel computing architecture has been designed from the ground up to integrate with MPI.  This document describes how to use MPI with IPython.
+A much better way of moving data between engines is to use a message passing
+library, such as the Message Passing Interface (MPI) [MPI]_. IPython's
+parallel computing architecture has been designed from the ground up to
+integrate with MPI. This document describes how to use MPI with IPython.
 
 Additional installation requirements
 ====================================
 
 If you want to use MPI with IPython, you will need to install:
 
 * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH.
 * The mpi4py [mpi4py]_ package.
 
 .. note::
 
     The mpi4py package is not a strict requirement. However, you need to
     have *some* way of calling MPI from Python. You also need some way of
     making sure that :func:`MPI_Init` is called when the IPython engines start
     up. There are a number of ways of doing this and a good number of
     associated subtleties. We highly recommend just using mpi4py as it
     takes care of most of these problems. If you want to do something
     different, let us know and we can help you get started.
 
 Starting the engines with MPI enabled
 =====================================
 
 To use code that calls MPI, there are typically two things that MPI requires.
 
 1. The process that wants to call MPI must be started using
    :command:`mpiexec` or a batch system (like PBS) that has MPI support.
 2. Once the process starts, it must call :func:`MPI_Init`.
 
-There are a couple of ways that you can start the IPython engines and get these things to happen.
+There are a couple of ways that you can start the IPython engines and get
+these things to happen.
 
 Automatic starting using :command:`mpiexec` and :command:`ipcluster`
 --------------------------------------------------------------------
 
-The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`::
+The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`,
+which will first start a controller and then a set of engines using
+:command:`mpiexec`::
 
     $ ipcluster mpiexec -n 4
 
 This approach is best as interrupting :command:`ipcluster` will automatically
 stop and clean up the controller and engines.
 
 Manual starting using :command:`mpiexec`
 ----------------------------------------
 
-If you want to start the IPython engines using the :command:`mpiexec`, just do::
+If you want to start the IPython engines using the :command:`mpiexec`, just
+do::
 
     $ mpiexec -n 4 ipengine --mpi=mpi4py
 
 This requires that you already have a controller running and that the FURL
 files for the engines are in place. We also have built in support for
 PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
 starting the engines with::
 
     	mpiexec -n 4 ipengine --mpi=pytrilinos
 
 Automatic starting using PBS and :command:`ipcluster`
 -----------------------------------------------------
 
-The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`.
+The :command:`ipcluster` command also has built-in integration with PBS. For
+more information on this approach, see our documentation on :ref:`ipcluster
+<parallel_process>`.
 
 Actually using MPI
 ==================
 
-Once the engines are running with MPI enabled, you are ready to go.  You can now call any code that uses MPI in the IPython engines.  And, all of this can be done interactively.  Here we show a simple example that uses mpi4py [mpi4py]_.
+Once the engines are running with MPI enabled, you are ready to go. You can
+now call any code that uses MPI in the IPython engines. And, all of this can
+be done interactively. Here we show a simple example that uses mpi4py
+[mpi4py]_.
 
-First, lets define a simply function that uses MPI to calculate the sum of a distributed array.  Save the following text in a file called :file:`psum.py`:
+First, lets define a simply function that uses MPI to calculate the sum of a
+distributed array. Save the following text in a file called :file:`psum.py`:
 
 .. sourcecode:: python
 
     from mpi4py import MPI
     import numpy as np
     
     def psum(a):
         s = np.sum(a)
         return MPI.COMM_WORLD.Allreduce(s,MPI.SUM)
 
 Now, start an IPython cluster in the same directory as :file:`psum.py`::
 
     $ ipcluster mpiexec -n 4
 
-Finally, connect to the cluster and use this function interactively.  In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function:
+Finally, connect to the cluster and use this function interactively. In this
+case, we create a random array on each engine and sum up all the random arrays
+using our :func:`psum` function:
 
 .. sourcecode:: ipython
 
     In [1]: from IPython.kernel import client
     
     In [2]: mec = client.MultiEngineClient()
     
     In [3]: mec.activate()
     
     In [4]: px import numpy as np
     Parallel execution on engines: all
     Out[4]: 
     <Results List>
     [0] In [13]: import numpy as np
     [1] In [13]: import numpy as np
     [2] In [13]: import numpy as np
     [3] In [13]: import numpy as np
     
     In [6]: px a = np.random.rand(100)
     Parallel execution on engines: all
     Out[6]: 
     <Results List>
     [0] In [15]: a = np.random.rand(100)
     [1] In [15]: a = np.random.rand(100)
     [2] In [15]: a = np.random.rand(100)
     [3] In [15]: a = np.random.rand(100)
     
     In [7]: px from psum import psum
     Parallel execution on engines: all
     Out[7]: 
     <Results List>
     [0] In [16]: from psum import psum
     [1] In [16]: from psum import psum
     [2] In [16]: from psum import psum
     [3] In [16]: from psum import psum
     
     In [8]: px s = psum(a)
     Parallel execution on engines: all
     Out[8]: 
     <Results List>
     [0] In [17]: s = psum(a)
     [1] In [17]: s = psum(a)
     [2] In [17]: s = psum(a)
     [3] In [17]: s = psum(a)
     
     In [9]: px print s
     Parallel execution on engines: all
     Out[9]: 
     <Results List>
     [0] In [18]: print s
     [0] Out[18]: 187.451545803
     
     [1] In [18]: print s
     [1] Out[18]: 187.451545803
     
     [2] In [18]: print s
     [2] Out[18]: 187.451545803
 
     [3] In [18]: print s
     [3] Out[18]: 187.451545803
 
 Any Python code that makes calls to MPI can be used in this manner, including
 compiled C, C++ and Fortran libraries that have been exposed to Python.
 
 .. [MPI] Message Passing Interface.  http://www-unix.mcs.anl.gov/mpi/
 .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/
 .. [OpenMPI] Open MPI. http://www.open-mpi.org/
 .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/

docs/source/parallel/parallel_multiengine.txt

@@ -1,828 +1,835 b''
 .. _parallelmultiengine:
 
 ===============================
 IPython's multiengine interface
 ===============================
 
 The multiengine interface represents one possible way of working with a set of
 IPython engines. The basic idea behind the multiengine interface is that the
 capabilities of each engine are directly and explicitly exposed to the user.
 Thus, in the multiengine interface, each engine is given an id that is used to
 identify the engine and give it work to do. This interface is very intuitive
 and is designed with interactive usage in mind, and is thus the best place for
 new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
 	$ ipcluster local -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``MultiEngineClient`` instance
 =========================================
 
 The first step is to import the IPython :mod:`IPython.kernel.client` module
 and then create a :class:`MultiEngineClient` instance:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 
 This form assumes that the :file:`ipcontroller-mec.furl` is in the
 :file:`~./ipython/security` directory on the client's host. If not, the
 location of the FURL file must be given as an argument to the
 constructor:
 
 .. sourcecode:: ipython
 
     In [2]: mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
 
 To make sure there are engines connected to the controller, use can get a list
 of engine ids:
 
 .. sourcecode:: ipython
 
 	In [3]: mec.get_ids()
 	Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
 Quick and easy parallelism
 ==========================
 
-In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  The multiengine interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.
+In many cases, you simply want to apply a Python function to a sequence of
+objects, but *in parallel*. The multiengine interface provides two simple ways
+of accomplishing this: a parallel version of :func:`map` and ``@parallel``
+function decorator.
 
 Parallel map
 ------------
 
 Python's builtin :func:`map` functions allows a function to be applied to a
 sequence element-by-element. This type of code is typically trivial to
 parallelize. In fact, the multiengine interface in IPython already has a
 parallel version of :meth:`map` that works just like its serial counterpart:
 
 .. sourcecode:: ipython
 
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 .. note::
 
     The multiengine interface version of :meth:`map` does not do any load
     balancing.  For a load balanced version, see the task interface.
 
 .. seealso::
 
     The :meth:`map` method has a number of options that can be controlled by
     the :meth:`mapper` method.  See its docstring for more information.
 
 Parallel function decorator
 ---------------------------
 
-Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function:
+Parallel functions are just like normal function, but they can be called on
+sequences and *in parallel*. The multiengine interface provides a decorator
+that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @mec.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....: 
 
     In [11]: f(range(32))    # this is done in parallel
     Out[11]: 
     [0.0,10.0,160.0,...]
 
 See the docstring for the :meth:`parallel` decorator for options.
 
 Running Python commands
 =======================
 
 The most basic type of operation that can be performed on the engines is to
 execute Python code. Executing Python code can be done in blocking or
 non-blocking mode (blocking is default) using the :meth:`execute` method.
 
 Blocking execution
 ------------------
 
 In blocking mode, the :class:`MultiEngineClient` object (called ``mec`` in
 these examples) submits the command to the controller, which places the
 command in the engines' queues for execution. The :meth:`execute` call then
 blocks until the engines are done executing the command:
 
 .. sourcecode:: ipython
 
 	# The default is to run on all engines
 	In [4]: mec.execute('a=5')
 	Out[4]: 
 	<Results List>
 	[0] In [1]: a=5
 	[1] In [1]: a=5
 	[2] In [1]: a=5
 	[3] In [1]: a=5
 
 	In [5]: mec.execute('b=10')
 	Out[5]: 
 	<Results List>
 	[0] In [2]: b=10
 	[1] In [2]: b=10
 	[2] In [2]: b=10
 	[3] In [2]: b=10
 
 Python commands can be executed on specific engines by calling execute using
 the ``targets`` keyword argument:
 
 .. sourcecode:: ipython
 
 	In [6]: mec.execute('c=a+b',targets=[0,2])
 	Out[6]: 
 	<Results List>
 	[0] In [3]: c=a+b
 	[2] In [3]: c=a+b
 
 
 	In [7]: mec.execute('c=a-b',targets=[1,3])
 	Out[7]: 
 	<Results List>
 	[1] In [3]: c=a-b
 	[3] In [3]: c=a-b
 
 
 	In [8]: mec.execute('print c')
 	Out[8]: 
 	<Results List>
 	[0] In [4]: print c
 	[0] Out[4]: 15
 
 	[1] In [4]: print c
 	[1] Out[4]: -5
 
 	[2] In [4]: print c
 	[2] Out[4]: 15
 
 	[3] In [4]: print c
 	[3] Out[4]: -5
 
 This example also shows one of the most important things about the IPython
 engines: they have a persistent user namespaces. The :meth:`execute` method
 returns a Python ``dict`` that contains useful information:
 
 .. sourcecode:: ipython
 
 	In [9]: result_dict = mec.execute('d=10; print d')
 
 	In [10]: for r in result_dict:
 	   ....:     print r
 	   ....:     
 	   ....:     
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 0, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 1, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 2, 'stdout': '10\n'}
 	{'input': {'translated': 'd=10; print d', 'raw': 'd=10; print d'}, 'number': 5, 'id': 3, 'stdout': '10\n'}
 
 Non-blocking execution
 ----------------------
 
 In non-blocking mode, :meth:`execute` submits the command to be executed and
 then returns a :class:`PendingResult` object immediately. The
 :class:`PendingResult` object gives you a way of getting a result at a later
 time through its :meth:`get_result` method or :attr:`r` attribute. This allows
 you to quickly submit long running commands without blocking your local
 Python/IPython session:
 
 .. sourcecode:: ipython
 
 	# In blocking mode
 	In [6]: mec.execute('import time')
 	Out[6]:
 	<Results List>
 	[0] In [1]: import time
 	[1] In [1]: import time
 	[2] In [1]: import time
 	[3] In [1]: import time
 
 	# In non-blocking mode
 	In [7]: pr = mec.execute('time.sleep(10)',block=False)
 
 	# Now block for the result
 	In [8]: pr.get_result()
 	Out[8]:
 	<Results List>
 	[0] In [2]: time.sleep(10)
 	[1] In [2]: time.sleep(10)
 	[2] In [2]: time.sleep(10)
 	[3] In [2]: time.sleep(10)
 
 	# Again in non-blocking mode
 	In [9]: pr = mec.execute('time.sleep(10)',block=False)
 
 	# Poll to see if the result is ready
 	In [10]: pr.get_result(block=False)
 
 	# A shorthand for get_result(block=True)
 	In [11]: pr.r
 	Out[11]:
 	<Results List>
 	[0] In [3]: time.sleep(10)
 	[1] In [3]: time.sleep(10)
 	[2] In [3]: time.sleep(10)
 	[3] In [3]: time.sleep(10)
 
 Often, it is desirable to wait until a set of :class:`PendingResult` objects
 are done. For this, there is a the method :meth:`barrier`. This method takes a
 tuple of :class:`PendingResult` objects and blocks until all of the associated
 results are ready:
 
 .. sourcecode:: ipython
 
 	In [72]: mec.block=False
 
 	# A trivial list of PendingResults objects
 	In [73]: pr_list = [mec.execute('time.sleep(3)') for i in range(10)]
 
 	# Wait until all of them are done
 	In [74]: mec.barrier(pr_list)
 
 	# Then, their results are ready using get_result or the r attribute
 	In [75]: pr_list[0].r
 	Out[75]: 
 	<Results List>
 	[0] In [20]: time.sleep(3)
 	[1] In [19]: time.sleep(3)
 	[2] In [20]: time.sleep(3)
 	[3] In [19]: time.sleep(3)
 
 
 The ``block`` and ``targets`` keyword arguments and attributes
 --------------------------------------------------------------
 
 Most methods in the multiengine interface (like :meth:`execute`) accept
 ``block`` and ``targets`` as keyword arguments. As we have seen above, these
 keyword arguments control the blocking mode and which engines the command is
 applied to. The :class:`MultiEngineClient` class also has :attr:`block` and
 :attr:`targets` attributes that control the default behavior when the keyword
 arguments are not provided. Thus the following logic is used for :attr:`block`
 and :attr:`targets`:
 
 * If no keyword argument is provided, the instance attributes are used.
 * Keyword argument, if provided override the instance attributes.
 
 The following examples demonstrate how to use the instance attributes:
 
 .. sourcecode:: ipython
 
 	In [16]: mec.targets = [0,2]
 
 	In [17]: mec.block = False
 
 	In [18]: pr = mec.execute('a=5')
 
 	In [19]: pr.r
 	Out[19]: 
 	<Results List>
 	[0] In [6]: a=5
 	[2] In [6]: a=5
 
 	# Note targets='all' means all engines
 	In [20]: mec.targets = 'all'
 
 	In [21]: mec.block = True
 
 	In [22]: mec.execute('b=10; print b')
 	Out[22]: 
 	<Results List>
 	[0] In [7]: b=10; print b
 	[0] Out[7]: 10
 
 	[1] In [6]: b=10; print b
 	[1] Out[6]: 10
 
 	[2] In [7]: b=10; print b
 	[2] Out[7]: 10
 
 	[3] In [6]: b=10; print b
 	[3] Out[6]: 10
 
 The :attr:`block` and :attr:`targets` instance attributes also determine the
 behavior of the parallel magic commands.
 
 
 Parallel magic commands
 -----------------------
 
 We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
 that make it more pleasant to execute Python commands on the engines
 interactively. These are simply shortcuts to :meth:`execute` and
 :meth:`get_result`. The ``%px`` magic executes a single Python command on the
 engines specified by the :attr:`targets` attribute of the
 :class:`MultiEngineClient` instance (by default this is ``'all'``):
 
 .. sourcecode:: ipython
 
 	# Make this MultiEngineClient active for parallel magic commands
 	In [23]: mec.activate()
 
 	In [24]: mec.block=True
 
 	In [25]: import numpy
 
 	In [26]: %px import numpy
 	Executing command on Controller
 	Out[26]:
 	<Results List>
 	[0] In [8]: import numpy
 	[1] In [7]: import numpy
 	[2] In [8]: import numpy
 	[3] In [7]: import numpy
 
 
 	In [27]: %px a = numpy.random.rand(2,2)
 	Executing command on Controller
 	Out[27]:
 	<Results List>
 	[0] In [9]: a = numpy.random.rand(2,2)
 	[1] In [8]: a = numpy.random.rand(2,2)
 	[2] In [9]: a = numpy.random.rand(2,2)
 	[3] In [8]: a = numpy.random.rand(2,2)
 
 
 	In [28]: %px print numpy.linalg.eigvals(a)
 	Executing command on Controller
 	Out[28]:
 	<Results List>
 	[0] In [10]: print numpy.linalg.eigvals(a)
 	[0] Out[10]: [ 1.28167017  0.14197338]
 
 	[1] In [9]: print numpy.linalg.eigvals(a)
 	[1] Out[9]: [-0.14093616  1.27877273]
 
 	[2] In [10]: print numpy.linalg.eigvals(a)
 	[2] Out[10]: [-0.37023573  1.06779409]
 
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
 The ``%result`` magic gets and prints the stdin/stdout/stderr of the last
 command executed on each engine. It is simply a shortcut to the
 :meth:`get_result` method:
 
 .. sourcecode:: ipython
 
 	In [29]: %result
 	Out[29]:
 	<Results List>
 	[0] In [10]: print numpy.linalg.eigvals(a)
 	[0] Out[10]: [ 1.28167017  0.14197338]
 
 	[1] In [9]: print numpy.linalg.eigvals(a)
 	[1] Out[9]: [-0.14093616  1.27877273]
 
 	[2] In [10]: print numpy.linalg.eigvals(a)
 	[2] Out[10]: [-0.37023573  1.06779409]
 
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
 The ``%autopx`` magic switches to a mode where everything you type is executed
 on the engines given by the :attr:`targets` attribute:
 
 .. sourcecode:: ipython
 
 	In [30]: mec.block=False
 
 	In [31]: %autopx
 	Auto Parallel Enabled
 	Type %autopx to disable
 
 	In [32]: max_evals = []
 	<IPython.kernel.multiengineclient.PendingResult object at 0x17b8a70>
 
 	In [33]: for i in range(100):
 	   ....:     a = numpy.random.rand(10,10)
 	   ....:     a = a+a.transpose()
 	   ....:     evals = numpy.linalg.eigvals(a)
 	   ....:     max_evals.append(evals[0].real)
 	   ....:
 	   ....:
 	<IPython.kernel.multiengineclient.PendingResult object at 0x17af8f0>
 
 	In [34]: %autopx
 	Auto Parallel Disabled
 
 	In [35]: mec.block=True
 
 	In [36]: px print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	Executing command on Controller
 	Out[36]:
 	<Results List>
 	[0] In [13]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[0] Out[13]: Average max eigenvalue is:  10.1387247332
 
 	[1] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[1] Out[12]: Average max eigenvalue is:  10.2076902286
 
 	[2] In [13]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[2] Out[13]: Average max eigenvalue is:  10.1891484655
 
 	[3] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[3] Out[12]: Average max eigenvalue is:  10.1158837784
 
 
 Moving Python objects around
 ============================
 
 In addition to executing code on engines, you can transfer Python objects to
 and from your IPython session and the engines. In IPython, these operations
 are called :meth:`push` (sending an object to the engines) and :meth:`pull`
 (getting an object from the engines).
 
 Basic push and pull
 -------------------
 
 Here are some examples of how you use :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
 	In [38]: mec.push(dict(a=1.03234,b=3453))
 	Out[38]: [None, None, None, None]
 
 	In [39]: mec.pull('a')
 	Out[39]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 	In [40]: mec.pull('b',targets=0)
 	Out[40]: [3453]
 
 	In [41]: mec.pull(('a','b'))
 	Out[41]: [[1.03234, 3453], [1.03234, 3453], [1.03234, 3453], [1.03234, 3453]]
 
 	In [42]: mec.zip_pull(('a','b'))
 	Out[42]: [(1.03234, 1.03234, 1.03234, 1.03234), (3453, 3453, 3453, 3453)]
 
 	In [43]: mec.push(dict(c='speed'))
 	Out[43]: [None, None, None, None]
 
 	In [44]: %px print c
 	Executing command on Controller
 	Out[44]: 
 	<Results List>
 	[0] In [14]: print c
 	[0] Out[14]: speed
 
 	[1] In [13]: print c
 	[1] Out[13]: speed
 
 	[2] In [14]: print c
 	[2] Out[14]: speed
 
 	[3] In [13]: print c
 	[3] Out[13]: speed
 
 In non-blocking mode :meth:`push` and :meth:`pull` also return
 :class:`PendingResult` objects:
 
 .. sourcecode:: ipython
 
 	In [47]: mec.block=False
 
 	In [48]: pr = mec.pull('a')
 
 	In [49]: pr.r
 	Out[49]: [1.03234, 1.03234, 1.03234, 1.03234]
 
 
 Push and pull for functions
 ---------------------------
 
 Functions can also be pushed and pulled using :meth:`push_function` and
 :meth:`pull_function`:
 
 .. sourcecode:: ipython
 
     In [52]: mec.block=True
 
 	In [53]: def f(x):
 	   ....:     return 2.0*x**4
 	   ....: 
 
 	In [54]: mec.push_function(dict(f=f))
 	Out[54]: [None, None, None, None]
 
 	In [55]: mec.execute('y = f(4.0)')
 	Out[55]: 
 	<Results List>
 	[0] In [15]: y = f(4.0)
 	[1] In [14]: y = f(4.0)
 	[2] In [15]: y = f(4.0)
 	[3] In [14]: y = f(4.0)
 
 
 	In [56]: px print y
 	Executing command on Controller
 	Out[56]: 
 	<Results List>
 	[0] In [16]: print y
 	[0] Out[16]: 512.0
 
 	[1] In [15]: print y
 	[1] Out[15]: 512.0
 
 	[2] In [16]: print y
 	[2] Out[16]: 512.0
 
 	[3] In [15]: print y
 	[3] Out[15]: 512.0
 
 
 Dictionary interface
 --------------------
 
 As a shorthand to :meth:`push` and :meth:`pull`, the
 :class:`MultiEngineClient` class implements some of the Python dictionary
 interface. This make the remote namespaces of the engines appear as a local
 dictionary. Underneath, this uses :meth:`push` and :meth:`pull`:
 
 .. sourcecode:: ipython
 
 	In [50]: mec.block=True
 
 	In [51]: mec['a']=['foo','bar']
 
 	In [52]: mec['a']
 	Out[52]: [['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar'], ['foo', 'bar']]
 
 Scatter and gather
 ------------------
 
 Sometimes it is useful to partition a sequence and push the partitions to
 different engines. In MPI language, this is know as scatter/gather and we
 follow that terminology. However, it is important to remember that in
 IPython's :class:`MultiEngineClient` class, :meth:`scatter` is from the
 interactive IPython session to the engines and :meth:`gather` is from the
 engines back to the interactive IPython session. For scatter/gather operations
 between engines, MPI should be used:
 
 .. sourcecode:: ipython
 
 	In [58]: mec.scatter('a',range(16))
 	Out[58]: [None, None, None, None]
 
 	In [59]: px print a
 	Executing command on Controller
 	Out[59]: 
 	<Results List>
 	[0] In [17]: print a
 	[0] Out[17]: [0, 1, 2, 3]
 
 	[1] In [16]: print a
 	[1] Out[16]: [4, 5, 6, 7]
 
 	[2] In [17]: print a
 	[2] Out[17]: [8, 9, 10, 11]
 
 	[3] In [16]: print a
 	[3] Out[16]: [12, 13, 14, 15]
 
 
 	In [60]: mec.gather('a')
 	Out[60]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
 
 Other things to look at
 =======================
 
 How to do parallel list comprehensions
 --------------------------------------
 
 In many cases list comprehensions are nicer than using the map function. While
 we don't have fully parallel list comprehensions, it is simple to get the
 basic effect using :meth:`scatter` and :meth:`gather`:
 
 .. sourcecode:: ipython
 
 	In [66]: mec.scatter('x',range(64))
 	Out[66]: [None, None, None, None]
 
 	In [67]: px y = [i**10 for i in x]
 	Executing command on Controller
 	Out[67]: 
 	<Results List>
 	[0] In [19]: y = [i**10 for i in x]
 	[1] In [18]: y = [i**10 for i in x]
 	[2] In [19]: y = [i**10 for i in x]
 	[3] In [18]: y = [i**10 for i in x]
 
 
 	In [68]: y = mec.gather('y')
 
 	In [69]: print y
 	[0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
 Parallel exceptions
 -------------------
 
 In the multiengine interface, parallel commands can raise Python exceptions,
 just like serial commands. But, it is a little subtle, because a single
 parallel command can actually raise multiple exceptions (one for each engine
 the command was run on). To express this idea, the MultiEngine interface has a
 :exc:`CompositeError` exception class that will be raised in most cases. The
 :exc:`CompositeError` class is a special type of exception that wraps one or
 more other types of exceptions. Here is how it works:
 
 .. sourcecode:: ipython
 
 	In [76]: mec.block=True
 
 	In [77]: mec.execute('1/0')
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block)
 	    432         targets, block = self._findTargetsAndBlock(targets, block)
 	    433         result = blockingCallFromThread(self.smultiengine.execute, lines,
 	--> 434             targets=targets, block=block)
 	    435         if block:
 	    436             result = ResultList(result)
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
-Notice how the error message printed when :exc:`CompositeError` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions:
+Notice how the error message printed when :exc:`CompositeError` is raised has
+information about the individual exceptions that were raised on each engine.
+If you want, you can even raise one of these original exceptions:
 
 .. sourcecode:: ipython
 
 	In [80]: try:
 	   ....:     mec.execute('1/0')
 	   ....: except client.CompositeError, e:
 	   ....:     e.raise_exception()
 	   ....:     
 	   ....:     
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/error.pyc in raise_exception(self, excid)
 	    156             raise IndexError("an exception with index %i does not exist"%excid)
 	    157         else:
 	--> 158             raise et, ev, etb
 	    159 
 	    160 def collect_exceptions(rlist, method):
 
 	ZeroDivisionError: integer division or modulo by zero
 
 If you are working in IPython, you can simple type ``%debug`` after one of
 these :exc:`CompositeError` exceptions is raised, and inspect the exception
 instance:
 
 .. sourcecode:: ipython
 
 	In [81]: mec.execute('1/0')
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in execute(self, lines, targets, block)
 	    432         targets, block = self._findTargetsAndBlock(targets, block)
 	    433         result = blockingCallFromThread(self.smultiengine.execute, lines,
 	--> 434             targets=targets, block=block)
 	    435         if block:
 	    436             result = ResultList(result)
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
 	In [82]: %debug
 	> 
 	
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.py(74)blockingCallFromThread()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 
 	# With the debugger running, e is the exceptions instance.  We can tab complete
 	# on it and see the extra methods that are available.
 	ipdb> e.
 	e.__class__         e.__getitem__       e.__new__           e.__setstate__      e.args
 	e.__delattr__       e.__getslice__      e.__reduce__        e.__str__           e.elist
 	e.__dict__          e.__hash__          e.__reduce_ex__     e.__weakref__       e.message
 	e.__doc__           e.__init__          e.__repr__          e._get_engine_str   e.print_tracebacks
 	e.__getattribute__  e.__module__        e.__setattr__       e._get_traceback    e.raise_exception
 	ipdb> e.print_tracebacks()
 	[0:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[1:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[2:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 	[3:execute]: 
 	---------------------------------------------------------------------------
 	ZeroDivisionError                         Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<string> in <module>()
 
 	ZeroDivisionError: integer division or modulo by zero
 
 .. note::
 
     The above example appears to be broken right now because of a change in 
     how we are using Twisted.
 
 All of this same error handling magic even works in non-blocking mode:
 
 .. sourcecode:: ipython
 
 	In [83]: mec.block=False
 
 	In [84]: pr = mec.execute('1/0')
 
 	In [85]: pr.r
 	---------------------------------------------------------------------------
 	CompositeError                            Traceback (most recent call last)
 
 	/ipython1-client-r3021/docs/examples/<ipython console> in <module>()
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in _get_r(self)
 	    170 
 	    171     def _get_r(self):
 	--> 172         return self.get_result(block=True)
 	    173 
 	    174     r = property(_get_r)
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_result(self, default, block)
 	    131                 return self.result
 	    132         try:
 	--> 133             result = self.client.get_pending_deferred(self.result_id, block)
 	    134         except error.ResultNotCompleted:
 	    135             return default
 
 	/ipython1-client-r3021/ipython1/kernel/multiengineclient.pyc in get_pending_deferred(self, deferredID, block)
 	    385 
 	    386     def get_pending_deferred(self, deferredID, block):
 	--> 387         return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block)
 	    388 
 	    389     def barrier(self, pendingResults):
 
 	/ipython1-client-r3021/ipython1/kernel/twistedutil.pyc in blockingCallFromThread(f, *a, **kw)
 	     72             result.raiseException()
 	     73         except Exception, e:
 	---> 74             raise e
 	     75     return result
 	     76 
 
 	CompositeError: one or more exceptions from call to method: execute
 	[0:execute]: ZeroDivisionError: integer division or modulo by zero
 	[1:execute]: ZeroDivisionError: integer division or modulo by zero
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
 

docs/source/parallel/parallel_process.txt

@@ -1,351 +1,387 b''
 .. _parallel_process:
 
 ===========================================
 Starting the IPython controller and engines
 ===========================================
 
 To use IPython for parallel computing, you need to start one instance of
 the controller and one or more instances of the engine. The controller
 and each engine can run on different machines or on the same machine.
 Because of this, there are many different possibilities.
 
 Broadly speaking, there are two ways of going about starting a controller and engines:
 
 * In an automated manner using the :command:`ipcluster` command.
 * In a more manual way using the :command:`ipcontroller` and 
   :command:`ipengine` commands.
 
-This document describes both of these methods.  We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases.
+This document describes both of these methods. We recommend that new users
+start with the :command:`ipcluster` command as it simplifies many common usage
+cases.
 
 General considerations
 ======================
 
-Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines.  These things come up no matter which method you use to start your IPython cluster.
+Before delving into the details about how you can start a controller and
+engines using the various methods, we outline some of the general issues that
+come up when starting the controller and engines. These things come up no
+matter which method you use to start your IPython cluster.
 
-Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``.  The following steps are then required:
+Let's say that you want to start the controller on ``host0`` and engines on
+hosts ``host1``-``hostn``. The following steps are then required:
 
 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
    ``host0``.
 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
    controller from ``host0`` to hosts ``host1``-``hostn``.
 3. Start the engines on hosts ``host1``-``hostn`` by running 
    :command:`ipengine`.  This command has to be told where the FURL file 
    (:file:`ipcontroller-engine.furl`) is located.
 
 At this point, the controller and engines will be connected. By default, the
 FURL files created by the controller are put into the
 :file:`~/.ipython/security` directory. If the engines share a filesystem with
 the controller, step 2 can be skipped as the engines will automatically look
 at that location.
 
 The final step required required to actually use the running controller from a
 client is to move the FURL files :file:`ipcontroller-mec.furl` and
 :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
-be run.  If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically.  Otherwise, the full path to them has to be passed to the client's constructor.
+be run. If these file are put into the :file:`~/.ipython/security` directory
+of the client's host, they will be found automatically. Otherwise, the full
+path to them has to be passed to the client's constructor.
 
 Using :command:`ipcluster`
 ==========================
 
-The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations:
+The :command:`ipcluster` command provides a simple way of starting a
+controller and engines in the following situations:
 
 1. When the controller and engines are all run on localhost. This is useful
    for testing or running on a multicore computer.
 2. When engines are started using the :command:`mpirun` command that comes 
    with most MPI [MPI]_ implementations
 3. When engines are started using the PBS [PBS]_ batch system.
 4. When the controller is started on localhost and the engines are started on 
    remote nodes using :command:`ssh`.
 
 .. note::
 
     It is also possible for advanced users to add support to
     :command:`ipcluster` for starting controllers and engines using other
     methods (like Sun's Grid Engine for example).
 
 .. note::
 
     Currently :command:`ipcluster` requires that the
     :file:`~/.ipython/security` directory live on a shared filesystem that is
     seen by both the controller and engines. If you don't have a shared file
     system you will need to use :command:`ipcontroller` and
     :command:`ipengine` directly.  This constraint can be relaxed if you are 
     using the :command:`ssh` method to start the cluster.
 
 Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
 and :command:`ipengine` to perform the steps described above.
 
 Using :command:`ipcluster` in local mode
 ----------------------------------------
 
 To start one controller and 4 engines on localhost, just do::
 
     $ ipcluster local -n 4
 
 To see other command line options for the local mode, do::
 
     $ ipcluster local -h
 
 Using :command:`ipcluster` in mpiexec/mpirun mode
 -------------------------------------------------
 
 The mpiexec/mpirun mode is useful if you:
 
 1. Have MPI installed.
 2. Your systems are configured to use the :command:`mpiexec` or
    :command:`mpirun` commands to start MPI processes.
 
 .. note::
 
     The preferred command to use is :command:`mpiexec`.  However, we also 
     support :command:`mpirun` for backwards compatibility.  The underlying
     logic used is exactly the same, the only difference being the name of the 
     command line program that is called.
 
 If these are satisfied, you can start an IPython cluster using::
 
     $ ipcluster mpiexec -n 4
 
 This does the following:
 
 1. Starts the IPython controller on current host.
 2. Uses :command:`mpiexec` to start 4 engines.
 
-On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`.  However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting.  The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
+On newer MPI implementations (such as OpenMPI), this will work even if you
+don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
+implementations actually require each process to call :func:`MPI_Init` upon
+starting. The easiest way of having this done is to install the mpi4py
+[mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
 
     $ ipcluster mpiexec -n 4 --mpi=mpi4py
 
-Unfortunately, even this won't work for some MPI implementations.  If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time.  Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use.  However, this custom Python executable approach will not work with :command:`ipcluster` currently.
+Unfortunately, even this won't work for some MPI implementations. If you are
+having problems with this, you will likely have to use a custom Python
+executable that itself calls :func:`MPI_Init` at the appropriate time.
+Fortunately, mpi4py comes with such a custom Python executable that is easy to
+install and use. However, this custom Python executable approach will not work
+with :command:`ipcluster` currently.
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster mpiexec -h
 
 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
 
 
 Using :command:`ipcluster` in PBS mode
 --------------------------------------
 
-The PBS mode uses the Portable Batch System [PBS]_ to start the engines.  To use this mode, you first need to create a PBS script template that will be used to start the engines.  Here is a sample PBS script template:
+The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To
+use this mode, you first need to create a PBS script template that will be
+used to start the engines. Here is a sample PBS script template:
 
 .. sourcecode:: bash
 
     #PBS -N ipython
     #PBS -j oe
     #PBS -l walltime=00:10:00
     #PBS -l nodes=${n/4}:ppn=4
     #PBS -q parallel
 
     cd $$PBS_O_WORKDIR
     export PATH=$$HOME/usr/local/bin
     export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
     /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
 
 There are a few important points about this template:
 
 1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
    template engine.
 
 2. Instead of putting in the actual number of engines, use the notation
    ``${n}`` to indicate the number of engines to be started. You can also uses
    expressions like ``${n/4}`` in the template to indicate the number of 
    nodes.
 
 3. Because ``$`` is a special character used by the template engine, you must  
    escape any ``$`` by using ``$$``.  This is important when referring to 
    environment variables in the template.
 
 4. Any options to :command:`ipengine` should be given in the batch script 
    template.
 
 5. Depending on the configuration of you system, you may have to set 
    environment variables in the script template.
 
-Once you have created such a script, save it with a name like :file:`pbs.template`.  Now you are ready to start your job::
+Once you have created such a script, save it with a name like
+:file:`pbs.template`. Now you are ready to start your job::
 
     $ ipcluster pbs -n 128 --pbs-script=pbs.template
 
 Additional command line options for this mode can be found by doing::
 
     $ ipcluster pbs -h
 
 Using :command:`ipcluster` in SSH mode
 --------------------------------------
 
 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
 nodes and the :command:`ipcontroller` on localhost.
 
-When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins.
+When using using this mode it highly recommended that you have set up SSH keys
+and are using ssh-agent [SSH]_ for password-less logins.
 
-To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile":
+To use this mode you need a python file describing the cluster, here is an
+example of such a "clusterfile":
 
 .. sourcecode:: python
     
     send_furl = True
     engines = { 'host1.example.com' : 2,
                 'host2.example.com' : 5,
                 'host3.example.com' : 1,
                 'host4.example.com' : 8 }
 
-Since this is a regular python file usual python syntax applies. Things to note:
+Since this is a regular python file usual python syntax applies. Things to
+note:
 
 * The `engines` dict, where the keys is the host we want to run engines on and
   the value is the number of engines to run on that host.
 * send_furl can either be `True` or `False`, if `True` it will copy over the
   furl needed for :command:`ipengine` to each host.
 
 The ``--clusterfile`` command line option lets you specify the file to use for
 the cluster definition. Once you have your cluster file and you can
 :command:`ssh` into the remote hosts with out an password you are ready to
 start your cluster like so:
 
 .. sourcecode:: bash 
 
    $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py
 
 
-Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts:
+Two helper shell scripts are used to start and stop :command:`ipengine` on
+remote hosts:
 
 * sshx.sh 
 * engine_killer.sh
 
-Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there.  On most Unix, Linux and OS X systems this is /tmp.
+Defaults for both of these are contained in the source code for
+:command:`ipcluster`. The default scripts are written to a local file in a
+tmep directory and then copied to a temp directory on the remote host and
+executed from there. On most Unix, Linux and OS X systems this is /tmp.
 
 The default sshx.sh is the following:
 
 .. sourcecode:: bash
 
    #!/bin/sh
    "$@" &> /dev/null &
    echo $!
 
 If you want to use a custom sshx.sh script you need to use the ``--sshx``
 option and specify the file to use. Using a custom sshx.sh file could be
 helpful when you need to setup the environment on the remote host before
 executing :command:`ipengine`.
 
 For a detailed options list:
 
 .. sourcecode:: bash
 
    $ ipcluster ssh -h
 
 Current limitations of the SSH mode of :command:`ipcluster` are:
 
 * Untested on Windows. Would require a working :command:`ssh` on Windows.
   Also, we are using shell scripts to setup and execute commands on remote
   hosts.
 * :command:`ipcontroller` is started on localhost, with no option to start it
   on a remote node.
 
 Using the :command:`ipcontroller` and :command:`ipengine` commands
 ==================================================================
 
-It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines.  This approach gives you full control over all aspects of the startup process.
+It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
+commands to start your controller and engines. This approach gives you full
+control over all aspects of the startup process.
 
 Starting the controller and engine on your local machine
 --------------------------------------------------------
 
 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
 local machine, do the following.
 
 First start the controller::
 
 	$ ipcontroller
 	
-Next, start however many instances of the engine you want using (repeatedly) the command::
+Next, start however many instances of the engine you want using (repeatedly)
+the command::
 
 	$ ipengine
 
-The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
+The engines should start and automatically connect to the controller using the
+FURL files in :file:`~./ipython/security`. You are now ready to use the
+controller and engines from IPython.
 
 .. warning:: 
 	
 	The order of the above operations is very important.  You *must*
  	start the controller before the engines, since the engines connect 
 	to the controller as they get started.
 
 .. note::
 
     On some platforms (OS X), to put the controller and engine into the
     background you may need to give these commands in the form ``(ipcontroller
     &)`` and ``(ipengine &)`` (with the parentheses) for them to work
     properly.
 
 Starting the controller and engines on different hosts
 ------------------------------------------------------
 
 When the controller and engines are running on different hosts, things are
 slightly more complicated, but the underlying ideas are the same:
 
 1. Start the controller on a host using :command:`ipcontroller`.
-2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
+2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on
+   the controller's host to the host where the engines will run.
 3. Use :command:`ipengine` on the engine's hosts to start the engines.
 
-The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
+The only thing you have to be careful of is to tell :command:`ipengine` where
+the :file:`ipcontroller-engine.furl` file is located. There are two ways you
+can do this:
 
 * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
   directory on the engine's host, where it will be found automatically.
 * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
   flag.
 
 The ``--furl-file`` flag works like this::
 
     $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
 
 .. note::
     
     If the controller's and engine's hosts all have a shared file system  
     (:file:`~./ipython/security` is the same on all of them), then things
     will just work!
 
 Make FURL files persistent
 ---------------------------
 
 At fist glance it may seem that that managing the FURL files is a bit
 annoying. Going back to the house and key analogy, copying the FURL around
 each time you start the controller is like having to make a new key every time
 you want to unlock the door and enter your house. As with your house, you want
 to be able to create the key (or FURL file) once, and then simply use it at
 any point in the future.
 
 This is possible, but before you do this, you **must** remove any old FURL
 files in the :file:`~/.ipython/security` directory.
 
 .. warning::
 
     You **must** remove old FURL files before using persistent FURL files.
 
 Then, The only thing you have to do is decide what ports the controller will
 listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
 These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
 Then, just copy the furl files over the first time and you are set. You can
 start and stop the controller and engines any many times as you want in the
 future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 
     You may ask the question: what ports does the controller listen on if you
     don't tell is to use specific ones? The default is to use high random port
     numbers. We do this for two reasons: i) to increase security through
     obscurity and ii) to multiple controllers on a given host to start and
     automatically use different ports.
 
 Log files
 ---------
 
 All of the components of IPython have log files associated with them.
 These log files can be extremely useful in debugging problems with
 IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
 the log files to us will often help us to debug any problems.
 
 
 .. [PBS] Portable Batch System.  http://www.openpbs.org/
 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent

docs/source/parallel/parallel_security.txt

@@ -1,363 +1,364 b''
 .. _parallelsecurity:
 
 ===========================
 Security details of IPython
 ===========================
 
 IPython's :mod:`IPython.kernel` package exposes the full power of the Python
 interpreter over a TCP/IP network for the purposes of parallel computing. This
 feature brings up the important question of IPython's security model. This
 document gives details about this model and how it is implemented in IPython's
 architecture.
 
 Processs and network topology
 =============================
 
 To enable parallel computing, IPython has a number of different processes that
 run. These processes are discussed at length in the IPython documentation and
 are summarized here:
 
 * The IPython *engine*.  This process is a full blown Python
   interpreter in which user code is executed.  Multiple
   engines are started to make parallel computing possible.
 * The IPython *controller*.  This process manages a set of 
   engines, maintaining a queue for each and presenting
   an asynchronous interface to the set of engines.
 * The IPython *client*.  This process is typically an
   interactive Python process that is used to coordinate the
   engines to get a parallel computation done.
 
 Collectively, these three processes are called the IPython *kernel*.
 
 These three processes communicate over TCP/IP connections with a well defined
 topology. The IPython controller is the only process that listens on TCP/IP
 sockets. Upon starting, an engine connects to a controller and registers
 itself with the controller. These engine/controller TCP/IP connections persist
 for the lifetime of each engine.
 
 The IPython client also connects to the controller using one or more TCP/IP
 connections. These connections persist for the lifetime of the client only.
 
 A given IPython controller and set of engines typically has a relatively short
 lifetime. Typically this lifetime corresponds to the duration of a single
 parallel simulation performed by a single user. Finally, the controller,
 engines and client processes typically execute with the permissions of that
 same user. More specifically, the controller and engines are *not* executed as
 root or with any other superuser permissions.
 
 Application logic
 =================
 
 When running the IPython kernel to perform a parallel computation, a user
 utilizes the IPython client to send Python commands and data through the
 IPython controller to the IPython engines, where those commands are executed
 and the data processed. The design of IPython ensures that the client is the
-only access point for the capabilities of the engines.  That is, the only way of addressing the engines is through a client.
+only access point for the capabilities of the engines. That is, the only way
+of addressing the engines is through a client.
 
 A user can utilize the client to instruct the IPython engines to execute
 arbitrary Python commands. These Python commands can include calls to the
 system shell, access the filesystem, etc., as required by the user's
 application code. From this perspective, when a user runs an IPython engine on
 a host, that engine has the same capabilities and permissions as the user
 themselves (as if they were logged onto the engine's host with a terminal).
 
 Secure network connections
 ==========================
 
 Overview
 --------
 
 All TCP/IP connections between the client and controller as well as the
 engines and controller are fully encrypted and authenticated. This section
 describes the details of the encryption and authentication approached used
 within IPython.
 
 IPython uses the Foolscap network protocol [Foolscap]_ for all communications
 between processes. Thus, the details of IPython's security model are directly
 related to those of Foolscap. Thus, much of the following discussion is
 actually just a discussion of the security that is built in to Foolscap.
 
 Encryption
 ----------
 
 For encryption purposes, IPython and Foolscap use the well known Secure Socket
 Layer (SSL) protocol [RFC5246]_. We use the implementation of this protocol
 provided by the OpenSSL project through the pyOpenSSL [pyOpenSSL]_ Python
 bindings to OpenSSL.
 
 Authentication
 --------------
 
 IPython clients and engines must also authenticate themselves with the
 controller. This is handled in a capabilities based security model
 [Capability]_. In this model, the controller creates a strong cryptographic
 key or token that represents each set of capability that the controller
 offers. Any party who has this key and presents it to the controller has full
 access to the corresponding capabilities of the controller. This model is
 analogous to using a physical key to gain access to physical items
 (capabilities) behind a locked door.
 
 For a capabilities based authentication system to prevent unauthorized access,
 two things must be ensured:
 
 * The keys must be cryptographically strong.  Otherwise attackers could gain 
   access by a simple brute force key guessing attack.
 * The actual keys must be distributed only to authorized parties.
 
 The keys in Foolscap are called Foolscap URL's or FURLs. The following section
 gives details about how these FURLs are created in Foolscap. The IPython
 controller creates a number of FURLs for different purposes:
 
 * One FURL that grants IPython engines access to the controller.  Also
   implicit in this access is permission to execute code sent by an
   authenticated IPython client.
 * Two or more FURLs that grant IPython clients access to the controller.
   Implicit in this access is permission to give the controller's engine code 
   to execute.
 
 Upon starting, the controller creates these different FURLS and writes them
-files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus, only the
-user who starts the controller has access to the FURLs.
+files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus,
+only the user who starts the controller has access to the FURLs.
 
 For an IPython client or engine to authenticate with a controller, it must
 present the appropriate FURL to the controller upon connecting. If the
 FURL matches what the controller expects for a given capability, access is
 granted. If not, access is denied. The exchange of FURLs is done after
 encrypted communications channels have been established to prevent attackers
 from capturing them.
 
 .. note::
 
     The FURL is similar to an unsigned private key in SSH.
 
 Details of the Foolscap handshake
 ---------------------------------
 
 In this section we detail the precise security handshake that takes place at
 the beginning of any network connection in IPython. For the purposes of this
 discussion, the SERVER is the IPython controller process and the CLIENT is the
 IPython engine or client process.
 
 Upon starting, all IPython processes do the following:
 
 1. Create a public key x509 certificate (ISO/IEC 9594).
 2. Create a hash of the contents of the certificate using the SHA-1 algorithm.
    The base-32 encoded version of this hash is saved by the process as its 
    process id (actually in Foolscap, this is the Tub id, but here refer to
    it as the process id).
 
 Upon starting, the IPython controller also does the following:
 
 1. Save the x509 certificate to disk in a secure location. The CLIENT
    certificate is never saved to disk.
 2. Create a FURL for each capability that the controller has.  There are
    separate capabilities the controller offers for clients and engines.  The
    FURL is created using:  a) the process id of the SERVER, b) the IP 
    address and port the SERVER is listening on and c) a 160 bit,    
    cryptographically secure string that represents the capability (the
    "capability id").
 3. The FURLs are saved to disk in a secure location on the SERVER's host.
 
 For a CLIENT to be able to connect to the SERVER and access a capability of
 that SERVER, the CLIENT must have knowledge of the FURL for that SERVER's
 capability. This typically requires that the file containing the FURL be
 moved from the SERVER's host to the CLIENT's host. This is done by the end
 user who started the SERVER and wishes to have a CLIENT connect to the SERVER.
 
 When a CLIENT connects to the SERVER, the following handshake protocol takes
 place:
 
 1. The CLIENT tells the SERVER what process (or Tub) id it expects the SERVER
    to have.
 2. If the SERVER has that process id, it notifies the CLIENT that it will now 
    enter encrypted mode.  If the SERVER has a different id, the SERVER aborts.
 3. Both CLIENT and SERVER initiate the SSL handshake protocol.
 4. Both CLIENT and SERVER request the certificate of their peer and verify
    that certificate.  If this succeeds, all further communications are 
    encrypted.
 5. Both CLIENT and SERVER send a hello block containing connection parameters
    and their process id.
 6. The CLIENT and SERVER check that their peer's stated process id matches the
    hash of the x509 certificate the peer presented.  If not, the connection is
    aborted.
 7. The CLIENT verifies that the SERVER's stated id matches the id of the
    SERVER the CLIENT is intending to connect to.  If not, the connection is 
    aborted.
 8. The CLIENT and SERVER elect a master who decides on the final connection
    parameters.
 
 The public/private key pair associated with each process's x509 certificate
 are completely hidden from this handshake protocol. There are however, used
 internally by OpenSSL as part of the SSL handshake protocol. Each process
 keeps their own private key hidden and sends its peer only the public key
 (embedded in the certificate).
 
 Finally, when the CLIENT requests access to a particular SERVER capability,
 the following happens:
 
 1. The CLIENT asks the SERVER for access to a capability by presenting that 
    capabilities id.
 2. If the SERVER has a capability with that id, access is granted. If not,
    access is not granted.
 3. Once access has been gained, the CLIENT can use the capability.
 
 Specific security vulnerabilities
 =================================
 
 There are a number of potential security vulnerabilities present in IPython's
 architecture. In this section we discuss those vulnerabilities and detail how
 the security architecture described above prevents them from being exploited.
 
 Unauthorized clients
 --------------------
 
 The IPython client can instruct the IPython engines to execute arbitrary
 Python code with the permissions of the user who started the engines. If an
 attacker were able to connect their own hostile IPython client to the IPython
 controller, they could instruct the engines to execute code.
 
 This attack is prevented by the capabilities based client authentication
 performed after the encrypted channel has been established. The relevant
 authentication information is encoded into the FURL that clients must
 present to gain access to the IPython controller. By limiting the distribution
 of those FURLs, a user can grant access to only authorized persons.
 
 It is highly unlikely that a client FURL could be guessed by an attacker
 in a brute force guessing attack. A given instance of the IPython controller
 only runs for a relatively short amount of time (on the order of hours). Thus
 an attacker would have only a limited amount of time to test a search space of
 size 2**320. Furthermore, even if a controller were to run for a longer amount
 of time, this search space is quite large (larger for instance than that of
 typical username/password pair).
 
 Unauthorized engines
 --------------------
 
 If an attacker were able to connect a hostile engine to a user's controller,
 the user might unknowingly send sensitive code or data to the hostile engine.
 This attacker's engine would then have full access to that code and data.
 
 This type of attack is prevented in the same way as the unauthorized client
 attack, through the usage of the capabilities based authentication scheme.
 
 Unauthorized controllers
 ------------------------
 
 It is also possible that an attacker could try to convince a user's IPython
 client or engine to connect to a hostile IPython controller. That controller
 would then have full access to the code and data sent between the IPython
 client and the IPython engines.
 
 Again, this attack is prevented through the FURLs, which ensure that a
 client or engine connects to the correct controller. It is also important to
 note that the FURLs also encode the IP address and port that the
 controller is listening on, so there is little chance of mistakenly connecting
 to a controller running on a different IP address and port.
 
 When starting an engine or client, a user must specify which FURL to use
 for that connection. Thus, in order to introduce a hostile controller, the
 attacker must convince the user to use the FURLs associated with the
 hostile controller. As long as a user is diligent in only using FURLs from
 trusted sources, this attack is not possible.
 
 Other security measures
 =======================
 
 A number of other measures are taken to further limit the security risks
 involved in running the IPython kernel.
 
 First, by default, the IPython controller listens on random port numbers.
 While this can be overridden by the user, in the default configuration, an
 attacker would have to do a port scan to even find a controller to attack.
 When coupled with the relatively short running time of a typical controller
 (on the order of hours), an attacker would have to work extremely hard and
 extremely *fast* to even find a running controller to attack.
 
 Second, much of the time, especially when run on supercomputers or clusters,
 the controller is running behind a firewall. Thus, for engines or client to
 connect to the controller:
 
 * The different processes have to all be behind the firewall.
 
 or:
 
 * The user has to use SSH port forwarding to tunnel the
   connections through the firewall.
    
 In either case, an attacker is presented with addition barriers that prevent
 attacking or even probing the system.
 
 Summary
 =======
 
 IPython's architecture has been carefully designed with security in mind. The
 capabilities based authentication model, in conjunction with the encrypted
 TCP/IP channels, address the core potential vulnerabilities in the system,
 while still enabling user's to use the system in open networks.
 
 Other questions
 ===============
 
 About keys
 ----------
 
 Can you clarify the roles of the certificate and its keys versus the FURL,
 which is also called a key?
 
 The certificate created by IPython processes is a standard public key x509
 certificate, that is used by the SSL handshake protocol to setup encrypted
 channel between the controller and the IPython engine or client. This public
 and private key associated with this certificate are used only by the SSL
 handshake protocol in setting up this encrypted channel.
 
 The FURL serves a completely different and independent purpose from the
 key pair associated with the certificate. When we refer to a FURL as a
 key, we are using the word "key" in the capabilities based security model
 sense. This has nothing to do with "key" in the public/private key sense used
 in the SSL protocol.
 
 With that said the FURL is used as an cryptographic key, to grant
 IPython engines and clients access to particular capabilities that the
 controller offers.
 
 Self signed certificates
 ------------------------
 
 Is the controller creating a self-signed certificate? Is this created for per
 instance/session, one-time-setup or each-time the controller is started?
 
 The Foolscap network protocol, which handles the SSL protocol details, creates
 a self-signed x509 certificate using OpenSSL for each IPython process. The
 lifetime of the certificate is handled differently for the IPython controller
 and the engines/client.
 
 For the IPython engines and client, the certificate is only held in memory for
 the lifetime of its process. It is never written to disk.
 
 For the controller, the certificate can be created anew each time the
 controller starts or it can be created once and reused each time the
 controller starts. If at any point, the certificate is deleted, a new one is
 created the next time the controller starts.
 
 SSL private key
 ---------------
 
 How the private key (associated with the certificate) is distributed?
 
 In the usual implementation of the SSL protocol, the private key is never
 distributed. We follow this standard always.
 
 SSL versus Foolscap authentication
 ----------------------------------
 
 Many SSL connections only perform one sided authentication (the server to the
 client). How is the client authentication in IPython's system related to SSL
 authentication?
 
 We perform a two way SSL handshake in which both parties request and verify
 the certificate of their peer. This mutual authentication is handled by the
 SSL handshake and is separate and independent from the additional
 authentication steps that the CLIENT and SERVER perform after an encrypted
 channel is established.
 
 .. [RFC5246] <http://tools.ietf.org/html/rfc5246>

docs/source/parallel/parallel_task.txt

@@ -1,99 +1,120 b''
 .. _paralleltask:
 
 ==========================
 The IPython task interface
 ==========================
 
-The task interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the multiengine interface, in the task interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful.
-
-Best of all the user can use both of these interfaces running at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the task interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign tasks to engines explicitly.
+The task interface to the controller presents the engines as a fault tolerant,
+dynamic load-balanced system or workers. Unlike the multiengine interface, in
+the task interface, the user have no direct access to individual engines. In
+some ways, this interface is simpler, but in other ways it is more powerful.
+
+Best of all the user can use both of these interfaces running at the same time
+to take advantage or both of their strengths. When the user can break up the
+user's work into segments that do not depend on previous execution, the task
+interface is ideal. But it also has more power and flexibility, allowing the
+user to guide the distribution of jobs, without having to assign tasks to
+engines explicitly.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
 controller and four IPython engines. The simplest way of doing this is to use
 the :command:`ipcluster` command::
 
 	$ ipcluster local -n 4
 	
 For more detailed information about starting the controller and engines, see
 our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``TaskClient`` instance
 =========================================
 
 The first step is to import the IPython :mod:`IPython.kernel.client` module
 and then create a :class:`TaskClient` instance:
 
 .. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: tc = client.TaskClient()
 
 This form assumes that the :file:`ipcontroller-tc.furl` is in the
 :file:`~./ipython/security` directory on the client's host. If not, the
 location of the FURL file must be given as an argument to the
 constructor:
 
 .. sourcecode:: ipython
 
     In [2]: mec = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
 
 Quick and easy parallelism
 ==========================
 
-In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  Like the multiengine interface, the task interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.  However, the verions in the task interface have one important difference:  they are dynamically load balanced.  Thus, if the execution time per item varies significantly, you should use the versions in the task interface.
+In many cases, you simply want to apply a Python function to a sequence of
+objects, but *in parallel*. Like the multiengine interface, the task interface
+provides two simple ways of accomplishing this: a parallel version of
+:func:`map` and ``@parallel`` function decorator. However, the verions in the
+task interface have one important difference: they are dynamically load
+balanced. Thus, if the execution time per item varies significantly, you
+should use the versions in the task interface.
 
 Parallel map
 ------------
 
-The parallel :meth:`map` in the task interface is similar to that in the multiengine interface:
+The parallel :meth:`map` in the task interface is similar to that in the
+multiengine interface:
 
 .. sourcecode:: ipython
 
 	In [63]: serial_result = map(lambda x:x**10, range(32))
 
 	In [64]: parallel_result = tc.map(lambda x:x**10, range(32))
 
 	In [65]: serial_result==parallel_result
 	Out[65]: True
 
 Parallel function decorator
 ---------------------------
 
-Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function:
+Parallel functions are just like normal function, but they can be called on
+sequences and *in parallel*. The multiengine interface provides a decorator
+that turns any Python function into a parallel function:
 
 .. sourcecode:: ipython
 
     In [10]: @tc.parallel()
        ....: def f(x):
        ....:     return 10.0*x**4
        ....: 
 
     In [11]: f(range(32))    # this is done in parallel
     Out[11]: 
     [0.0,10.0,160.0,...]
 
 More details
 ============
 
-The :class:`TaskClient` has many more powerful features that allow quite a bit of flexibility in how tasks are defined and run.  The next places to look are in the following classes:
+The :class:`TaskClient` has many more powerful features that allow quite a bit
+of flexibility in how tasks are defined and run. The next places to look are
+in the following classes:
 
 * :class:`IPython.kernel.client.TaskClient`
 * :class:`IPython.kernel.client.StringTask`
 * :class:`IPython.kernel.client.MapTask`
 
 The following is an overview of how to use these classes together:
 
 1. Create a :class:`TaskClient`.
 2. Create one or more instances of :class:`StringTask` or :class:`MapTask`
    to define your tasks.
 3. Submit your tasks to using the :meth:`run` method of your
    :class:`TaskClient` instance.
 4. Use :meth:`TaskClient.get_task_result` to get the results of the
    tasks.
 
-We are in the process of developing more detailed information about the task interface.  For now, the docstrings of the :class:`TaskClient`, :class:`StringTask` and :class:`MapTask` classes should be consulted.
+We are in the process of developing more detailed information about the task
+interface. For now, the docstrings of the :class:`TaskClient`,
+:class:`StringTask` and :class:`MapTask` classes should be consulted.