Update docs for automatic API building.

docs/autogen_api.py

@@ -0,0 +1,33
+#!/usr/bin/env python
+"""Script to auto-generate our API docs.
+"""
+# stdlib imports
+import os
+import sys
+
+# local imports
+sys.path.append(os.path.abspath('sphinxext'))
+from apigen import ApiDocWriter
+
+#*****************************************************************************
+if __name__ == '__main__':
+    pjoin = os.path.join
+    package = 'IPython'
+    outdir = pjoin('source','api','generated')
+    docwriter = ApiDocWriter(package,rst_extension='.txt')
+    docwriter.package_skip_patterns += [r'\.fixes$',
+                                        r'\.externals$',
+                                        r'\.Extensions',
+                                        r'\.kernel.config',
+                                        r'\.attic',
+                                        ]
+    docwriter.module_skip_patterns += [ r'\.FakeModule',
+                                        r'\.cocoa',
+                                        r'\.ipdoctest',
+                                        r'\.Gnuplot',
+                                        ]
+    docwriter.write_api_docs(outdir)
+    docwriter.write_index(outdir, 'gen',
+                          relative_to = pjoin('source','api')
+                          )
+    print '%d files written' % len(docwriter.written_modules)

docs/source/api/index.txt

@@ -0,0 +1,12
+.. _api-index:
+
+###################
+  The IPython API
+###################
+
+.. htmlonly::
+
+   :Release: |version|
+   :Date: |today|
+
+.. include:: generated/gen.txt

docs/source/parallel/visionhpc.txt

@@ -0,0 +1,246
+==================================
+ IPython/Vision Beam Pattern Demo
+==================================
+
+
+Installing and testing IPython at OSC systems
+=============================================
+
+All components were installed from source and I have my environment set up to
+include ~/usr/local in my various necessary paths ($PATH, $PYTHONPATH, etc).
+Other than a slow filesystem for unpacking tarballs, the install went without a
+hitch.  For each needed component, I just downloaded the source tarball,
+unpacked it via::
+
+  tar xzf (or xjf if it's bz2) filename.tar.{gz,bz2}
+
+and then installed them (including IPython itself) with::
+
+  cd dirname/  # path to unpacked tarball
+  python setup.py install --prefix=~/usr/local/
+
+The components I installed are listed below.  For each one I give the main
+project link as well as a direct one to the file I actually dowloaded and used.
+
+- nose, used for testing:
+http://somethingaboutorange.com/mrl/projects/nose/
+http://somethingaboutorange.com/mrl/projects/nose/nose-0.10.3.tar.gz
+
+- Zope interface, used to declare interfaces in twisted and ipython.  Note:
+you must get this from the page linked below and not fro the defaul
+one(http://www.zope.org/Products/ZopeInterface) because the latter has an
+older version, it hasn't been updated in a long time.  This pypi link has
+the current release (3.4.1 as of this writing):
+http://pypi.python.org/pypi/zope.interface
+http://pypi.python.org/packages/source/z/zope.interface/zope.interface-3.4.1.tar.gz
+
+- pyopenssl, security layer used by foolscap.  Note: version 0.7 *must* be
+used: 
+http://sourceforge.net/projects/pyopenssl/
+http://downloads.sourceforge.net/pyopenssl/pyOpenSSL-0.6.tar.gz?modtime=1212595285&big_mirror=0
+
+
+- Twisted, used for all networking:
+http://twistedmatrix.com/trac/wiki/Downloads
+http://tmrc.mit.edu/mirror/twisted/Twisted/8.1/Twisted-8.1.0.tar.bz2
+
+- Foolscap, used for managing connections securely:
+http://foolscap.lothar.com/trac
+http://foolscap.lothar.com/releases/foolscap-0.3.1.tar.gz
+
+
+- IPython itself:
+http://ipython.scipy.org/
+http://ipython.scipy.org/dist/ipython-0.9.1.tar.gz
+
+
+I then ran the ipython test suite via::
+
+  iptest -vv
+
+and it passed with only::
+
+    ======================================================================
+    ERROR: testGetResult_2
+    ----------------------------------------------------------------------
+    DirtyReactorAggregateError: Reactor was unclean.
+    Selectables:
+    <Negotiation #0 on 10105>
+
+    ----------------------------------------------------------------------
+    Ran 419 tests in 33.971s
+
+    FAILED (SKIP=4, errors=1)
+
+In three more runs of the test suite I was able to reproduce this error
+sometimes but not always; for now I think we can move on but we need to
+investigate further.  Especially if we start seeing problems in real use (the
+test suite stresses the networking layer in particular ways that aren't
+necessarily typical of normal use).
+
+Next, I started an 8-engine cluster via::
+
+    perez@opt-login01[~]> ipcluster -n 8
+    Starting controller: Controller PID: 30845
+    ^X      Starting engines:    Engines PIDs:   [30846, 30847, 30848, 30849,
+    30850, 30851, 30852, 30853]
+    Log files: /home/perez/.ipython/log/ipcluster-30845-*
+
+    Your cluster is up and running.
+
+    [... etc]
+
+and in a separate ipython session checked that the cluster is running and I can
+access all the engines::
+
+    In [1]: from IPython.kernel import client
+
+    In [2]: mec = client.MultiEngineClient()
+
+    In [3]: mec.get_ids()
+    Out[3]: [0, 1, 2, 3, 4, 5, 6, 7]
+
+and run trivial code in them (after importing the ``random`` module in all
+engines)::
+
+    In [11]: mec.execute("x=random.randint(0,10)")
+    Out[11]:
+    <Results List>
+    [0] In [3]: x=random.randint(0,10)
+    [1] In [3]: x=random.randint(0,10)
+    [2] In [3]: x=random.randint(0,10)
+    [3] In [3]: x=random.randint(0,10)
+    [4] In [3]: x=random.randint(0,10)
+    [5] In [3]: x=random.randint(0,10)
+    [6] In [3]: x=random.randint(0,10)
+    [7] In [3]: x=random.randint(0,10)
+
+    In [12]: mec.pull('x')
+    Out[12]: [10, 0, 8, 10, 2, 9, 10, 7]
+
+
+We'll continue conducting more complex tests later, including instaling Vision
+locally and running the beam demo.
+
+
+Michel's original instructions
+==============================
+
+I got a Vision network that reproduces the beam pattern demo working:
+
+.. image:: vision_beam_pattern.png
+   :width: 400
+   :target: vision_beam_pattern.png
+   :align: center
+
+
+I created a package called beamPattern that provides the function run() in its
+__init__.py file.
+
+A subpackage beamPattern/VisionInterface provides Vision nodes for:
+
+- computing Elevation and Azimuth from a 3D vector
+
+- Reading .mat files
+
+- taking the results gathered from the engines and creating the output that a
+  single engine would have had produced
+
+The Mec node connect to a controller. In my network it was local but an furl
+can be specified to connect to a remote controller.
+
+The PRun Func node is from the IPython library of nodes. the import statement
+is used to get the run function from the beamPattern package and bu puting
+"run" in the function entry of this node we push this function to the engines.
+In addition to the node will create input ports for all arguments of the
+function being pushed (i.e. the run function)
+
+The second input port on PRun Fun take an integer specifying the rank of the
+argument we want to scatter. All other arguments will be pushed to the engines.
+
+The ElevAzim node has a 3D vector widget and computes the El And Az values
+which are passed into the PRun Fun node through the ports created
+automatically. The Mat node allows to select the .mat file, reads it and passed
+the data to the locdata port created automatically on PRun Func
+
+The calculation is executed in parallel, and the results are gathered and
+output.  Instead of having a list of 3 vectors we nd up with a list of n*3
+vectors where n is the number of engines. unpackDectorResults will turn it into
+a list of 3. We then plot x, y, and 10*log10(z)
+
+
+Installation
+------------
+
+- inflate beamPattern into the site-packages directory for the MGL tools.
+
+- place the appended IPythonNodes.py and StandardNodes.py into the Vision
+  package of the MGL tools.
+
+- place the appended items.py in the NetworkEditor package of the MGL tools
+
+- run vision for the network beamPat5_net.py::
+
+    vision beamPat5_net.py
+
+Once the network is running, you can:
+
+- double click on the MEC node and either use an emptty string for the furl to
+  connect to a local engine or cut and paste the furl to the engine you want to
+  use
+
+- click on the yellow lighting bold to run the network.
+
+- Try modifying the MAT file or change the Vector used top compute elevation
+  and Azimut.
+  
+
+Fernando's notes
+================
+
+- I had to install IPython and all its dependencies for the python used by the
+  MGL tools.
+
+- Then I had to install scipy 0.6.0 for it, since the nodes needed Scipy.  To
+  do this I sourced the mglenv.sh script and then ran::
+
+    python setup.py install --prefix=~/usr/opt/mgl
+
+
+Using PBS
+=========
+
+The following PBS script can be used to start the engines::
+
+    #PBS -N bgranger-ipython
+    #PBS -j oe
+    #PBS -l walltime=00:10:00
+    #PBS -l nodes=4:ppn=4
+
+    cd $PBS_O_WORKDIR
+    export PATH=$HOME/usr/local/bin
+    export PYTHONPATH=$HOME/usr/local/lib/python2.4/site-packages
+    /usr/local/bin/mpiexec -n 16 ipengine
+
+
+If this file is called ``ipython_pbs.sh``, then the in one login windows
+(i.e. on the head-node -- ``opt-login01.osc.edu``), run ``ipcontroller``. In
+another login window on the same node, run the above script::
+
+    qsub ipython_pbs.sh
+
+If you look at the first window, you will see some diagnostic output
+from ipcontroller. You can then get the furl from your own
+``~/.ipython/security`` directory and then connect to it remotely.
+
+You might need to set up an SSH tunnel, however; if this doesn't work as
+advertised::
+
+    ssh -L 10115:localhost:10105 bic
+
+
+Links to other resources
+========================
+
+- http://www.osc.edu/~unpingco/glenn_NewLynx2_Demo.avi
+

docs/sphinxext/docscrape.py

@@ -0,0 +1,497
+"""Extract reference documentation from the NumPy source tree.
+
+"""
+
+import inspect
+import textwrap
+import re
+import pydoc
+from StringIO import StringIO
+from warnings import warn
+4
+class Reader(object):
+    """A line-based string reader.
+
+    """
+    def __init__(self, data):
+        """
+        Parameters
+        ----------
+        data : str
+           String with lines separated by '\n'.
+
+        """
+        if isinstance(data,list):
+            self._str = data
+        else:
+            self._str = data.split('\n') # store string as list of lines
+
+        self.reset()
+
+    def __getitem__(self, n):
+        return self._str[n]
+
+    def reset(self):
+        self._l = 0 # current line nr
+
+    def read(self):
+        if not self.eof():
+            out = self[self._l]
+            self._l += 1
+            return out
+        else:
+            return ''
+
+    def seek_next_non_empty_line(self):
+        for l in self[self._l:]:
+            if l.strip():
+                break
+            else:
+                self._l += 1
+
+    def eof(self):
+        return self._l >= len(self._str)
+
+    def read_to_condition(self, condition_func):
+        start = self._l
+        for line in self[start:]:
+            if condition_func(line):
+                return self[start:self._l]
+            self._l += 1
+            if self.eof():
+                return self[start:self._l+1]
+        return []
+
+    def read_to_next_empty_line(self):
+        self.seek_next_non_empty_line()
+        def is_empty(line):
+            return not line.strip()
+        return self.read_to_condition(is_empty)
+
+    def read_to_next_unindented_line(self):
+        def is_unindented(line):
+            return (line.strip() and (len(line.lstrip()) == len(line)))
+        return self.read_to_condition(is_unindented)
+
+    def peek(self,n=0):
+        if self._l + n < len(self._str):
+            return self[self._l + n]
+        else:
+            return ''
+
+    def is_empty(self):
+        return not ''.join(self._str).strip()
+
+
+class NumpyDocString(object):
+    def __init__(self,docstring):
+        docstring = textwrap.dedent(docstring).split('\n')
+
+        self._doc = Reader(docstring)
+        self._parsed_data = {
+            'Signature': '',
+            'Summary': [''],
+            'Extended Summary': [],
+            'Parameters': [],
+            'Returns': [],
+            'Raises': [],
+            'Warns': [],
+            'Other Parameters': [],
+            'Attributes': [],
+            'Methods': [],
+            'See Also': [],
+            'Notes': [],
+            'Warnings': [],
+            'References': '',
+            'Examples': '',
+            'index': {}
+            }
+
+        self._parse()
+
+    def __getitem__(self,key):
+        return self._parsed_data[key]
+
+    def __setitem__(self,key,val):
+        if not self._parsed_data.has_key(key):
+            warn("Unknown section %s" % key)
+        else:
+            self._parsed_data[key] = val
+
+    def _is_at_section(self):
+        self._doc.seek_next_non_empty_line()
+
+        if self._doc.eof():
+            return False
+
+        l1 = self._doc.peek().strip()  # e.g. Parameters
+
+        if l1.startswith('.. index::'):
+            return True
+
+        l2 = self._doc.peek(1).strip() #    ---------- or ==========
+        return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1))
+
+    def _strip(self,doc):
+        i = 0
+        j = 0
+        for i,line in enumerate(doc):
+            if line.strip(): break
+
+        for j,line in enumerate(doc[::-1]):
+            if line.strip(): break
+
+        return doc[i:len(doc)-j]
+
+    def _read_to_next_section(self):
+        section = self._doc.read_to_next_empty_line()
+
+        while not self._is_at_section() and not self._doc.eof():
+            if not self._doc.peek(-1).strip(): # previous line was empty
+                section += ['']
+
+            section += self._doc.read_to_next_empty_line()
+
+        return section
+
+    def _read_sections(self):
+        while not self._doc.eof():
+            data = self._read_to_next_section()
+            name = data[0].strip()
+
+            if name.startswith('..'): # index section
+                yield name, data[1:]
+            elif len(data) < 2:
+                yield StopIteration
+            else:
+                yield name, self._strip(data[2:])
+
+    def _parse_param_list(self,content):
+        r = Reader(content)
+        params = []
+        while not r.eof():
+            header = r.read().strip()
+            if ' : ' in header:
+                arg_name, arg_type = header.split(' : ')[:2]
+            else:
+                arg_name, arg_type = header, ''
+
+            desc = r.read_to_next_unindented_line()
+            desc = dedent_lines(desc)
+
+            params.append((arg_name,arg_type,desc))
+
+        return params
+
+    
+    _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"
+                           r" (?P<name2>[a-zA-Z0-9_.-]+))\s*", re.X)
+    def _parse_see_also(self, content):
+        """
+        func_name : Descriptive text
+            continued text
+        another_func_name : Descriptive text
+        func_name1, func_name2, :meth:`func_name`, func_name3
+
+        """
+        items = []
+
+        def parse_item_name(text):
+            """Match ':role:`name`' or 'name'"""
+            m = self._name_rgx.match(text)
+            if m:
+                g = m.groups()
+                if g[1] is None:
+                    return g[3], None
+                else:
+                    return g[2], g[1]
+            raise ValueError("%s is not a item name" % text)
+
+        def push_item(name, rest):
+            if not name:
+                return
+            name, role = parse_item_name(name)
+            items.append((name, list(rest), role))
+            del rest[:]
+
+        current_func = None
+        rest = []
+        
+        for line in content:
+            if not line.strip(): continue
+
+            m = self._name_rgx.match(line)
+            if m and line[m.end():].strip().startswith(':'):
+                push_item(current_func, rest)
+                current_func, line = line[:m.end()], line[m.end():]
+                rest = [line.split(':', 1)[1].strip()]
+                if not rest[0]:
+                    rest = []
+            elif not line.startswith(' '):
+                push_item(current_func, rest)
+                current_func = None
+                if ',' in line:
+                    for func in line.split(','):
+                        push_item(func, [])
+                elif line.strip():
+                    current_func = line
+            elif current_func is not None:
+                rest.append(line.strip())
+        push_item(current_func, rest)
+        return items
+
+    def _parse_index(self, section, content):
+        """
+        .. index: default
+           :refguide: something, else, and more
+
+        """
+        def strip_each_in(lst):
+            return [s.strip() for s in lst]
+
+        out = {}
+        section = section.split('::')
+        if len(section) > 1:
+            out['default'] = strip_each_in(section[1].split(','))[0]
+        for line in content:
+            line = line.split(':')
+            if len(line) > 2:
+                out[line[1]] = strip_each_in(line[2].split(','))
+        return out
+    
+    def _parse_summary(self):
+        """Grab signature (if given) and summary"""
+        if self._is_at_section():
+            return
+
+        summary = self._doc.read_to_next_empty_line()
+        summary_str = " ".join([s.strip() for s in summary]).strip()
+        if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str):
+            self['Signature'] = summary_str
+            if not self._is_at_section():
+                self['Summary'] = self._doc.read_to_next_empty_line()
+        else:
+            self['Summary'] = summary
+
+        if not self._is_at_section():
+            self['Extended Summary'] = self._read_to_next_section()
+    
+    def _parse(self):
+        self._doc.reset()
+        self._parse_summary()
+
+        for (section,content) in self._read_sections():
+            if not section.startswith('..'):
+                section = ' '.join([s.capitalize() for s in section.split(' ')])
+            if section in ('Parameters', 'Attributes', 'Methods',
+                           'Returns', 'Raises', 'Warns'):
+                self[section] = self._parse_param_list(content)
+            elif section.startswith('.. index::'):
+                self['index'] = self._parse_index(section, content)
+            elif section == 'See Also':
+                self['See Also'] = self._parse_see_also(content)
+            else:
+                self[section] = content
+
+    # string conversion routines
+
+    def _str_header(self, name, symbol='-'):
+        return [name, len(name)*symbol]
+
+    def _str_indent(self, doc, indent=4):
+        out = []
+        for line in doc:
+            out += [' '*indent + line]
+        return out
+
+    def _str_signature(self):
+        if self['Signature']:
+            return [self['Signature'].replace('*','\*')] + ['']
+        else:
+            return ['']
+
+    def _str_summary(self):
+        if self['Summary']:
+            return self['Summary'] + ['']
+        else:
+            return []
+
+    def _str_extended_summary(self):
+        if self['Extended Summary']:
+            return self['Extended Summary'] + ['']
+        else:
+            return []
+
+    def _str_param_list(self, name):
+        out = []
+        if self[name]:
+            out += self._str_header(name)
+            for param,param_type,desc in self[name]:
+                out += ['%s : %s' % (param, param_type)]
+                out += self._str_indent(desc)
+            out += ['']
+        return out
+
+    def _str_section(self, name):
+        out = []
+        if self[name]:
+            out += self._str_header(name)
+            out += self[name]
+            out += ['']
+        return out
+
+    def _str_see_also(self, func_role):
+        if not self['See Also']: return []
+        out = []
+        out += self._str_header("See Also")
+        last_had_desc = True
+        for func, desc, role in self['See Also']:
+            if role:
+                link = ':%s:`%s`' % (role, func)
+            elif func_role:
+                link = ':%s:`%s`' % (func_role, func)
+            else:
+                link = "`%s`_" % func
+            if desc or last_had_desc:
+                out += ['']
+                out += [link]
+            else:
+                out[-1] += ", %s" % link
+            if desc:
+                out += self._str_indent([' '.join(desc)])
+                last_had_desc = True
+            else:
+                last_had_desc = False
+        out += ['']
+        return out
+
+    def _str_index(self):
+        idx = self['index']
+        out = []
+        out += ['.. index:: %s' % idx.get('default','')]
+        for section, references in idx.iteritems():
+            if section == 'default':
+                continue
+            out += ['   :%s: %s' % (section, ', '.join(references))]
+        return out
+
+    def __str__(self, func_role=''):
+        out = []
+        out += self._str_signature()
+        out += self._str_summary()
+        out += self._str_extended_summary()
+        for param_list in ('Parameters','Returns','Raises'):
+            out += self._str_param_list(param_list)
+        out += self._str_section('Warnings')
+        out += self._str_see_also(func_role)
+        for s in ('Notes','References','Examples'):
+            out += self._str_section(s)
+        out += self._str_index()
+        return '\n'.join(out)
+
+
+def indent(str,indent=4):
+    indent_str = ' '*indent
+    if str is None:
+        return indent_str
+    lines = str.split('\n')
+    return '\n'.join(indent_str + l for l in lines)
+
+def dedent_lines(lines):
+    """Deindent a list of lines maximally"""
+    return textwrap.dedent("\n".join(lines)).split("\n")
+
+def header(text, style='-'):
+    return text + '\n' + style*len(text) + '\n'
+
+
+class FunctionDoc(NumpyDocString):
+    def __init__(self, func, role='func', doc=None):
+        self._f = func
+        self._role = role # e.g. "func" or "meth"
+        if doc is None:
+            doc = inspect.getdoc(func) or ''
+        try:
+            NumpyDocString.__init__(self, doc)
+        except ValueError, e:
+            print '*'*78
+            print "ERROR: '%s' while parsing `%s`" % (e, self._f)
+            print '*'*78
+            #print "Docstring follows:"
+            #print doclines
+            #print '='*78
+
+        if not self['Signature']:
+            func, func_name = self.get_func()
+            try:
+                # try to read signature
+                argspec = inspect.getargspec(func)
+                argspec = inspect.formatargspec(*argspec)
+                argspec = argspec.replace('*','\*')
+                signature = '%s%s' % (func_name, argspec)
+            except TypeError, e:
+                signature = '%s()' % func_name
+            self['Signature'] = signature
+
+    def get_func(self):
+        func_name = getattr(self._f, '__name__', self.__class__.__name__)
+        if inspect.isclass(self._f):
+            func = getattr(self._f, '__call__', self._f.__init__)
+        else:
+            func = self._f
+        return func, func_name
+            
+    def __str__(self):
+        out = ''
+
+        func, func_name = self.get_func()
+        signature = self['Signature'].replace('*', '\*')
+
+        roles = {'func': 'function',
+                 'meth': 'method'}
+
+        if self._role:
+            if not roles.has_key(self._role):
+                print "Warning: invalid role %s" % self._role
+            out += '.. %s:: %s\n    \n\n' % (roles.get(self._role,''),
+                                             func_name)
+
+        out += super(FunctionDoc, self).__str__(func_role=self._role)
+        return out
+
+
+class ClassDoc(NumpyDocString):
+    def __init__(self,cls,modulename='',func_doc=FunctionDoc,doc=None):
+        if not inspect.isclass(cls):
+            raise ValueError("Initialise using a class. Got %r" % cls)
+        self._cls = cls
+
+        if modulename and not modulename.endswith('.'):
+            modulename += '.'
+        self._mod = modulename
+        self._name = cls.__name__
+        self._func_doc = func_doc
+
+        if doc is None:
+            doc = pydoc.getdoc(cls)
+
+        NumpyDocString.__init__(self, doc)
+
+    @property
+    def methods(self):
+        return [name for name,func in inspect.getmembers(self._cls)
+                if not name.startswith('_') and callable(func)]
+
+    def __str__(self):
+        out = ''
+        out += super(ClassDoc, self).__str__()
+        out += "\n\n"
+
+        #for m in self.methods:
+        #    print "Parsing `%s`" % m
+        #    out += str(self._func_doc(getattr(self._cls,m), 'meth')) + '\n\n'
+        #    out += '.. index::\n   single: %s; %s\n\n' % (self._name, m)
+
+        return out
+
+

docs/sphinxext/docscrape_sphinx.py

@@ -0,0 +1,136
+import re, inspect, textwrap, pydoc
+from docscrape import NumpyDocString, FunctionDoc, ClassDoc
+
+class SphinxDocString(NumpyDocString):
+    # string conversion routines
+    def _str_header(self, name, symbol='`'):
+        return ['.. rubric:: ' + name, '']
+
+    def _str_field_list(self, name):
+        return [':' + name + ':']
+
+    def _str_indent(self, doc, indent=4):
+        out = []
+        for line in doc:
+            out += [' '*indent + line]
+        return out
+
+    def _str_signature(self):
+        return ['']
+        if self['Signature']:
+            return ['``%s``' % self['Signature']] + ['']
+        else:
+            return ['']
+
+    def _str_summary(self):
+        return self['Summary'] + ['']
+
+    def _str_extended_summary(self):
+        return self['Extended Summary'] + ['']
+
+    def _str_param_list(self, name):
+        out = []
+        if self[name]:
+            out += self._str_field_list(name)
+            out += ['']
+            for param,param_type,desc in self[name]:
+                out += self._str_indent(['**%s** : %s' % (param.strip(),
+                                                          param_type)])
+                out += ['']
+                out += self._str_indent(desc,8)
+                out += ['']
+        return out
+
+    def _str_section(self, name):
+        out = []
+        if self[name]:
+            out += self._str_header(name)
+            out += ['']
+            content = textwrap.dedent("\n".join(self[name])).split("\n")
+            out += content
+            out += ['']
+        return out
+
+    def _str_see_also(self, func_role):
+        out = []
+        if self['See Also']:
+            see_also = super(SphinxDocString, self)._str_see_also(func_role)
+            out = ['.. seealso::', '']
+            out += self._str_indent(see_also[2:])
+        return out
+
+    def _str_warnings(self):
+        out = []
+        if self['Warnings']:
+            out = ['.. warning::', '']
+            out += self._str_indent(self['Warnings'])
+        return out
+
+    def _str_index(self):
+        idx = self['index']
+        out = []
+        if len(idx) == 0:
+            return out
+
+        out += ['.. index:: %s' % idx.get('default','')]
+        for section, references in idx.iteritems():
+            if section == 'default':
+                continue
+            elif section == 'refguide':
+                out += ['   single: %s' % (', '.join(references))]
+            else:
+                out += ['   %s: %s' % (section, ','.join(references))]
+        return out
+
+    def _str_references(self):
+        out = []
+        if self['References']:
+            out += self._str_header('References')
+            if isinstance(self['References'], str):
+                self['References'] = [self['References']]
+            out.extend(self['References'])
+            out += ['']
+        return out
+
+    def __str__(self, indent=0, func_role="obj"):
+        out = []
+        out += self._str_signature()
+        out += self._str_index() + ['']
+        out += self._str_summary()
+        out += self._str_extended_summary()
+        for param_list in ('Parameters', 'Attributes', 'Methods',
+                           'Returns','Raises'):
+            out += self._str_param_list(param_list)
+        out += self._str_warnings()
+        out += self._str_see_also(func_role)
+        out += self._str_section('Notes')
+        out += self._str_references()
+        out += self._str_section('Examples')
+        out = self._str_indent(out,indent)
+        return '\n'.join(out)
+
+class SphinxFunctionDoc(SphinxDocString, FunctionDoc):
+    pass
+
+class SphinxClassDoc(SphinxDocString, ClassDoc):
+    pass
+
+def get_doc_object(obj, what=None, doc=None):
+    if what is None:
+        if inspect.isclass(obj):
+            what = 'class'
+        elif inspect.ismodule(obj):
+            what = 'module'
+        elif callable(obj):
+            what = 'function'
+        else:
+            what = 'object'
+    if what == 'class':
+        return SphinxClassDoc(obj, '', func_doc=SphinxFunctionDoc, doc=doc)
+    elif what in ('function', 'method'):
+        return SphinxFunctionDoc(obj, '', doc=doc)
+    else:
+        if doc is None:
+            doc = pydoc.getdoc(obj)
+        return SphinxDocString(doc)
+

docs/sphinxext/numpydoc.py

@@ -0,0 +1,116
+"""
+========
+numpydoc
+========
+
+Sphinx extension that handles docstrings in the Numpy standard format. [1]
+
+It will:
+
+- Convert Parameters etc. sections to field lists.
+- Convert See Also section to a See also entry.
+- Renumber references.
+- Extract the signature from the docstring, if it can't be determined otherwise.
+
+.. [1] http://projects.scipy.org/scipy/numpy/wiki/CodingStyleGuidelines#docstring-standard
+
+"""
+
+import os, re, pydoc
+from docscrape_sphinx import get_doc_object, SphinxDocString
+import inspect
+
+def mangle_docstrings(app, what, name, obj, options, lines,
+                      reference_offset=[0]):
+    if what == 'module':
+        # Strip top title
+        title_re = re.compile(r'^\s*[#*=]{4,}\n[a-z0-9 -]+\n[#*=]{4,}\s*',
+                              re.I|re.S)
+        lines[:] = title_re.sub('', "\n".join(lines)).split("\n")
+    else:
+        doc = get_doc_object(obj, what, "\n".join(lines))
+        lines[:] = str(doc).split("\n")
+
+    if app.config.numpydoc_edit_link and hasattr(obj, '__name__') and \
+           obj.__name__:
+        if hasattr(obj, '__module__'):
+            v = dict(full_name="%s.%s" % (obj.__module__, obj.__name__))
+        else:
+            v = dict(full_name=obj.__name__)
+        lines += ['', '.. htmlonly::', '']
+        lines += ['    %s' % x for x in
+                  (app.config.numpydoc_edit_link % v).split("\n")]
+
+    # replace reference numbers so that there are no duplicates
+    references = []
+    for l in lines:
+        l = l.strip()
+        if l.startswith('.. ['):
+            try:
+                references.append(int(l[len('.. ['):l.index(']')]))
+            except ValueError:
+                print "WARNING: invalid reference in %s docstring" % name
+
+    # Start renaming from the biggest number, otherwise we may
+    # overwrite references.
+    references.sort()
+    if references:
+        for i, line in enumerate(lines):
+            for r in references:
+                new_r = reference_offset[0] + r
+                lines[i] = lines[i].replace('[%d]_' % r,
+                                            '[%d]_' % new_r)
+                lines[i] = lines[i].replace('.. [%d]' % r,
+                                            '.. [%d]' % new_r)
+
+    reference_offset[0] += len(references)
+
+def mangle_signature(app, what, name, obj, options, sig, retann):
+    # Do not try to inspect classes that don't define `__init__`
+    if (inspect.isclass(obj) and
+        'initializes x; see ' in pydoc.getdoc(obj.__init__)):
+        return '', ''
+
+    if not (callable(obj) or hasattr(obj, '__argspec_is_invalid_')): return
+    if not hasattr(obj, '__doc__'): return
+
+    doc = SphinxDocString(pydoc.getdoc(obj))
+    if doc['Signature']:
+        sig = re.sub("^[^(]*", "", doc['Signature'])
+        return sig, ''
+
+def initialize(app):
+    try:
+        app.connect('autodoc-process-signature', mangle_signature)
+    except:
+        monkeypatch_sphinx_ext_autodoc()
+
+def setup(app, get_doc_object_=get_doc_object):
+    global get_doc_object
+    get_doc_object = get_doc_object_
+    
+    app.connect('autodoc-process-docstring', mangle_docstrings)
+    app.connect('builder-inited', initialize)
+    app.add_config_value('numpydoc_edit_link', None, True)
+
+#------------------------------------------------------------------------------
+# Monkeypatch sphinx.ext.autodoc to accept argspecless autodocs (Sphinx < 0.5)
+#------------------------------------------------------------------------------
+
+def monkeypatch_sphinx_ext_autodoc():
+    global _original_format_signature
+    import sphinx.ext.autodoc
+
+    if sphinx.ext.autodoc.format_signature is our_format_signature:
+        return
+
+    print "[numpydoc] Monkeypatching sphinx.ext.autodoc ..."
+    _original_format_signature = sphinx.ext.autodoc.format_signature
+    sphinx.ext.autodoc.format_signature = our_format_signature
+
+def our_format_signature(what, obj):
+    r = mangle_signature(None, what, None, obj, None, None, None)
+    if r is not None:
+        return r[0]
+    else:
+        return _original_format_signature(what, obj)

IPython/completer.py

@@ -1,640 +1,639
 """Word completion for IPython.
 
 This module is a fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3, but we need a lot more
 functionality specific to IPython, so this module will continue to live as an
 IPython-specific utility.
 
----------------------------------------------------------------------------
 Original rlcompleter documentation:
 
 This requires the latest extension to the readline module (the
 completes keywords, built-ins and globals in __main__; when completing
 NAME.NAME..., it evaluates (!) the expression up to the last dot and
 completes its attributes.
 
 It's very cool to do "import string" type "string.", hit the
 completion key (twice), and see the list of names defined by the
 string module!
 
 Tip: to use the tab key as the completion key, call
 
     readline.parse_and_bind("tab: complete")
 
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
 generally cause the completion to fail).  This is a feature -- since
 readline sets the tty device in raw (or cbreak) mode, printing a
 traceback wouldn't work well without some complicated hoopla to save,
 reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
 application defined code to be executed if an object with a
 __getattr__ hook is found.  Since it is the responsibility of the
 application (or the user) to enable this feature, I consider this an
 acceptable risk.  More complicated expressions (e.g. function calls or
 indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
 raw_input(), and thus these also benefit/suffer from the completer
 features.  Clearly an interactive application can benefit by
 specifying its own completer function and using raw_input() for all
 its input.
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
 
 """
 
 #*****************************************************************************
 #
 # Since this file is essentially a minimally modified copy of the rlcompleter
 # module which is part of the standard Python distribution, I assume that the
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
 import __builtin__
 import __main__
 import glob
 import keyword
 import os
 import re
 import shlex
 import sys
 import IPython.rlineimpl as readline    
 import itertools
 from IPython.ipstruct import Struct
 from IPython import ipapi
 from IPython import generics
 import types
 
 # Python 2.4 offers sets as a builtin
 try:
     set()
 except NameError:
     from sets import Set as set
 
 from IPython.genutils import debugx, dir2
 
 __all__ = ['Completer','IPCompleter']
 
 class Completer:
     def __init__(self,namespace=None,global_namespace=None):
         """Create a new completer for the command line.
 
         Completer([namespace,global_namespace]) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
 
         Completer instances should be used as the completion mechanism of
         readline via the set_completer() call:
 
         readline.set_completer(Completer(my_namespace).complete)
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = 1
         else:
             self.use_main_ns = 0
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
             
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [keyword.kwlist,
                     __builtin__.__dict__.keys(),
                     self.namespace.keys(),
                     self.global_namespace.keys()]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
         import re
 
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
         if not m:
             return []
         
         expr, attr = m.group(1, 3)
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         words = dir2(obj)
         
         try:
             words = generics.complete_object(obj, words)
         except ipapi.TryNext:
             pass
         # Build match list to return
         n = len(attr)
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
     def __init__(self,shell,namespace=None,global_namespace=None,
                  omit__names=0,alias_table=None):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
         via readline.set_completer().
 
         Inputs:
 
         - shell: a pointer to the ipython shell itself.  This is needed
         because this completer knows about magic functions, and those can
         only be accessed via the ipython instance.
 
         - namespace: an optional dict where completions are performed.
 
         - global_namespace: secondary optional dict for completions, to
         handle cases (such as IPython embedded inside functions) where
         both Python scopes are visible.
 
         - The optional omit__names parameter sets the completer to omit the
         'magic' names (__magicname__) for python objects unless the text
         to be completed explicitly starts with one or more underscores.
 
         - If alias_table is supplied, it should be a dictionary of aliases
         to complete. """
 
         Completer.__init__(self,namespace,global_namespace)
         self.magic_prefix = shell.name+'.magic_'
         self.magic_escape = shell.ESC_MAGIC
         self.readline = readline
         delims = self.readline.get_completer_delims()
         delims = delims.replace(self.magic_escape,'')
         self.readline.set_completer_delims(delims)
         self.get_line_buffer = self.readline.get_line_buffer
         self.get_endidx = self.readline.get_endidx
         self.omit__names = omit__names
         self.merge_completions = shell.rc.readline_merge_completions        
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
         
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
         self.matchers = [self.python_matches,
                          self.file_matches,
                          self.alias_matches,
                          self.python_func_kw_matches]
 
     
     # Code contributed by Alex Schmolck, for ipython/emacs integration
     def all_completions(self, text):
         """Return all possible completions for the benefit of emacs."""
 
         completions = []
         comp_append = completions.append
         try:
             for i in xrange(sys.maxint):
                 res = self.complete(text, i)
 
                 if not res: break
 
                 comp_append(res)
         #XXX workaround for ``notDefined.<tab>``
         except NameError:
             pass
         return completions
     # /end Alex Schmolck code.
 
     def _clean_glob(self,text):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self,text):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]            
 
     def file_matches(self, text):
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         #print 'Completer->file_matches: <%s>' % text # dbg
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
         if sys.platform == 'win32':
             protectables = ' '
         else:
             protectables = ' ()'
 
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
             
         def protect_filename(s):
             return "".join([(ch in protectables and '\\' + ch or ch)
                             for ch in s])
 
         def single_dir_expand(matches):
             "Recursively expand match lists containing a single dir."
             
             if len(matches) == 1 and os.path.isdir(matches[0]):
                 # Takes care of links to directories also.  Use '/'
                 # explicitly, even under Windows, so that name completions
                 # don't end up escaped.
                 d = matches[0]
                 if d[-1] in ['/','\\']:
                     d = d[:-1]
 
                 subdirs = os.listdir(d)
                 if subdirs:
                     matches = [ (d + '/' + p) for p in subdirs]
                     return single_dir_expand(matches)
                 else:
                     return matches
             else:
                 return matches
         
         lbuf = self.lbuf
         open_quotes = 0  # track strings with open quotes
         try:
             lsplit = shlex.split(lbuf)[-1]
         except ValueError:
             # typically an unmatched ", or backslash without escaped char.
             if lbuf.count('"')==1:
                 open_quotes = 1
                 lsplit = lbuf.split('"')[-1]
             elif lbuf.count("'")==1:
                 open_quotes = 1
                 lsplit = lbuf.split("'")[-1]
             else:
                 return []
         except IndexError:
             # tab pressed on empty line
             lsplit = ""
 
         if lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped
             # name
             has_protectables = 1
             text0,text = text,lsplit
         else:
             has_protectables = 0
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         m0 = self.clean_glob(text.replace('\\',''))
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 + 
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names at all (and we _shouldn't_, as it
                 # would cause bugs when the filesystem call is made).
                 matches = m0
             else:
                 matches = [text_prefix + 
                            protect_filename(f) for f in m0]
 
         #print 'mm',matches  # dbg
         return single_dir_expand(matches)
 
     def alias_matches(self, text):
         """Match internal system aliases"""        
         #print 'Completer->alias_matches:',text,'lb',self.lbuf # dbg
         
         # if we are not in the first 'item', alias matching 
         # doesn't make sense - unless we are starting with 'sudo' command.
         if ' ' in self.lbuf.lstrip() and not self.lbuf.lstrip().startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
         if text == "":
             return aliases
         else:
             return [alias for alias in aliases if alias.startswith(text)]
 
     def python_matches(self,text):
         """Match attributes or global python names"""
 
         #print 'Completer->python_matches, txt=<%s>' % text # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\._.*?',txt) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
             # this is so completion finds magics when automagic is on:
             if (matches == [] and 
                  not text.startswith(os.sep) and
                  not ' ' in self.lbuf):
                 matches = self.attr_matches(self.magic_prefix+text)
         return matches
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
 
         if not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             # for classes, check for __init__,__new__
             if inspect.isclass(obj):
                 obj = (getattr(obj,'__init__',None) or
                        getattr(obj,'__new__',None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 obj = obj.__call__
             # XXX: is there a way to handle the builtins ?
         try:
             args,_,_1,defaults = inspect.getargspec(obj)
             if defaults:
                 return args[-len(defaults):]
         except TypeError: pass
         return []
 
     def python_func_kw_matches(self,text):
         """Match named parameters (kwargs) of the last open function"""
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?' |    # single quoted strings or
                 ".*?" |    # double quoted strings or
                 \w+   |    # identifier
                 \S         # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis e.g. for "foo (1+bar(x), pa", the candidate is "foo"
         tokens = regexp.findall(self.get_line_buffer())
         tokens.reverse()
         iterTokens = iter(tokens); openPar = 0
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
         while True:
             try:
                 ids.append(iterTokens.next())
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not iterTokens.next() == '.':
                     break
             except StopIteration:
                 break
         # lookup the candidate callable matches either using global_matches
         # or attr_matches for dotted names
         if len(ids) == 1:
             callableMatches = self.global_matches(ids[0])
         else:
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
             try: namedArgs = self._default_arguments(eval(callableMatch,
                                                          self.namespace))
             except: continue
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
         return argMatches
 
     def dispatch_custom_completer(self,text):
         #print "Custom! '%s' %s" % (text, self.custom_completers) # dbg
         line = self.full_lbuf        
         if not line.strip():
             return None
 
         event = Struct()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         #print "\ncustom:{%s]\n" % event # dbg
         
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
               self.magic_escape + cmd)            
         else:
             try_magic = []
         
         
         for c in itertools.chain(
                                  self.custom_completers.s_matches(cmd),
                                  try_magic,
                                  self.custom_completers.flat_matches(self.lbuf)):
             #print "try",c # dbg
             try:
                 res = c(event)
                 # first, try case sensitive match
                 withcase = [r for r in res if r.startswith(text)]
                 if withcase:
                     return withcase
                 # if none, then case insensitive ones are ok too
                 return [r for r in res if r.lower().startswith(text.lower())]
             except ipapi.TryNext:
                 pass
             
         return None
                
     def complete(self, text, state,line_buffer=None):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         :Keywords:
         - line_buffer: string
         If not given, the completer attempts to obtain the current line buffer
         via readline.  This keyword allows clients which are requesting for
         text completions in non-readline contexts to inform the completer of
         the entire text.
         """
 
         #print '\n*** COMPLETE: <%s> (%s)' % (text,state)  # dbg
 
         # if there is only a tab on a line with only whitespace, instead
         # of the mostly useless 'do you want to see all million
         # completions' message, just do the right thing and give the user
         # his tab!  Incidentally, this enables pasting of tabbed text from
         # an editor (as long as autoindent is off).
 
         # It should be noted that at least pyreadline still shows
         # file completions - is there a way around it?
         
         # don't apply this on 'dumb' terminals, such as emacs buffers, so we
         # don't interfere with their own tab-completion mechanism.
         if line_buffer is None:
             self.full_lbuf = self.get_line_buffer()
         else:
             self.full_lbuf = line_buffer
             
         if not (self.dumb_terminal or self.full_lbuf.strip()):
             self.readline.insert_text('\t')
             return None
         
         magic_escape = self.magic_escape
         magic_prefix = self.magic_prefix
 
         self.lbuf = self.full_lbuf[:self.get_endidx()]
 
         try:
             if text.startswith(magic_escape):
                 text = text.replace(magic_escape,magic_prefix)
             elif text.startswith('~'):
                 text = os.path.expanduser(text)
             if state == 0:
                 custom_res = self.dispatch_custom_completer(text)
                 if custom_res is not None:
                     # did custom completers produce something?
                     self.matches = custom_res
                 else:
                     # Extend the list of completions with the results of each
                     # matcher, so we return results to the user from all
                     # namespaces.
                     if self.merge_completions:
                         self.matches = []
                         for matcher in self.matchers:
                             self.matches.extend(matcher(text))
                     else:
                         for matcher in self.matchers:
                             self.matches = matcher(text)
                             if self.matches:
                                 break
                     def uniq(alist):
                         set = {}
                         return [set.setdefault(e,e) for e in alist if e not in set]
                     self.matches = uniq(self.matches)                
             try:
                 ret = self.matches[state].replace(magic_prefix,magic_escape)                
                 return ret
             except IndexError:
                 return None
         except:
             #from IPython.ultraTB import AutoFormattedTB; # dbg
             #tb=AutoFormattedTB('Verbose');tb() #dbg
             
             # If completion fails, don't annoy the user.
             return None

IPython/genutils.py

@@ -1,2164 +1,2164
 # -*- coding: utf-8 -*-
 """
 General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
 $Id: genutils.py 2998 2008-01-31 10:06:04Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
 __license__ = Release.license
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
 
 
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
 
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
 
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
 
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
 
     Options:
 
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
 
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 #
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
 
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
 
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[0]
 
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[1]
 
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
         return u+s
 
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
 
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))
 
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
 
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
 
     This is pretty much deprecated and rarely used,
     genutils.getoutputerror may be what you need.
 
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
 
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
     >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
     """
 
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
 
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
 
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
     r"""Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
 
     If the input file test.ini contains (we put it in a string to keep the test
     self-contained):
 
     >>> test_ini = '''\
     ... i 3
     ... x 4.5
     ... y 5.5
     ... s hi ho'''
 
     Then we can use it as follows:
     >>> type_conv={int:'i',float:'x',None:'s'}
 
     >>> d = read_dict(test_ini)
 
     >>> sorted(d.items())
     [('i', '3'), ('s', 'hi ho'), ('x', '4.5'), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv,purge=True)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5)]
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
 
     # first read in table of values as strings
     if '\n' in filename:
         lines = filename.splitlines()
         file = None
     else:
         file = open(filename,'r')
         lines = file.readlines()
     for line in lines:
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
 
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
 
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
 
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
 
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
 
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
 
     if '\\library.zip\\' in IPython.__file__.lower():
         root, rest = IPython.__file__.lower().split('library.zip')
         if isdir(root + '_ipython'):
             os.environ["IPYKITROOT"] = root.rstrip('\\')
             return root
 
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 
 def get_ipython_dir():
     """Get the IPython directory for this platform and user.
     
     This uses the logic in `get_home_dir` to find the home directory
     and the adds either .ipython or _ipython to the end of the path.
     """
     if os.name == 'posix':
          ipdir_def = '.ipython'
     else:
          ipdir_def = '_ipython'
     home_dir = get_home_dir()
     ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
                                            os.path.join(home_dir,ipdir_def)))
     return ipdir
 
 def get_security_dir():
     """Get the IPython security directory.
     
     This directory is the default location for all security related files,
     including SSL/TLS certificates and FURL files.
     
     If the directory does not exist, it is created with 0700 permissions.
     If it exists, permissions are set to 0700.
     """
     security_dir = os.path.join(get_ipython_dir(), 'security')
     if not os.path.isdir(security_dir):
         os.mkdir(security_dir, 0700)
     else:
         os.chmod(security_dir, 0700)
     return security_dir
         
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
 
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
     if hasattr(arg,  'hideonce') and arg.hideonce:
         arg.hideonce = False
         return
 
     nlprint(arg)
 
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
 
-    Effectively this turns string: cd \ao\ao\
-    to: r"cd \ao\ao\_"[:-1]
-
-    Note the use of raw string and padding at the end to allow trailing backslash.
+    XXX - example removed because it caused encoding errors in documentation
+    generation.  We need a new example that doesn't contain invalid chars.
 
+    Note the use of raw string and padding at the end to allow trailing
+    backslash.
     """
 
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
 
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>> number = 19
 
     >>> text = "python"
 
     >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     Python 2.1 rules!
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened.
 
     Examples:
 
     >>> qw('1 2')
     ['1', '2']
 
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
 
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
     """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
 
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
 
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
 
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
 
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
 
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
 
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
 
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
 
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" %
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
 
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus
 
 #*************************** end of file <genutils.py> **********************

IPython/ipapi.py

@@ -1,687 +1,686
 """IPython customization API
 
 Your one-stop module for configuring & extending ipython
 
 The API will probably break when ipython 1.0 is released, but so 
 will the other configuration method (rc files).
 
 All names prefixed by underscores are for internal use, not part 
 of the public api.
 
 Below is an example that you can just put to a module and import from ipython. 
 
 A good practice is to install the config script below as e.g. 
 
 ~/.ipython/my_private_conf.py
 
 And do 
 
 import_mod my_private_conf 
 
 in ~/.ipython/ipythonrc
 
 That way the module is imported at startup and you can have all your
 personal configuration (as opposed to boilerplate ipythonrc-PROFILENAME 
 stuff) in there. 
 
------------------------------------------------
 import IPython.ipapi
 ip = IPython.ipapi.get()
 
 def ankka_f(self, arg):
     print 'Ankka',self,'says uppercase:',arg.upper()
 
 ip.expose_magic('ankka',ankka_f)
 
 ip.magic('alias sayhi echo "Testing, hi ok"')
 ip.magic('alias helloworld echo "Hello world"')
 ip.system('pwd')
 
 ip.ex('import re')
 ip.ex('''
 def funcci(a,b):
     print a+b
 print funcci(3,4)
 ''')
 ip.ex('funcci(348,9)')
 
 def jed_editor(self,filename, linenum=None):
     print 'Calling my own editor, jed ... via hook!'
     import os
     if linenum is None: linenum = 0
     os.system('jed +%d %s' % (linenum, filename))
     print 'exiting jed'
 
 ip.set_hook('editor',jed_editor)
 
 o = ip.options
 o.autocall = 2  # FULL autocall mode
 
 print 'done!'
 """
 
 #-----------------------------------------------------------------------------
 # Modules and globals
 
 # stdlib imports
 import __builtin__
 import sys
 
 # contains the most recently instantiated IPApi
 _RECENT_IP = None
 
 #-----------------------------------------------------------------------------
 # Code begins
 
 class TryNext(Exception):
     """Try next hook exception.
      
     Raise this in your hook function to indicate that the next hook handler
     should be used to handle the operation.  If you pass arguments to the
     constructor those arguments will be used by the next hook instead of the
     original ones.
     """
 
     def __init__(self, *args, **kwargs):
         self.args = args
         self.kwargs = kwargs
 
 
 class UsageError(Exception):
     """ Error in magic function arguments, etc.
     
     Something that probably won't warrant a full traceback, but should
     nevertheless interrupt a macro / batch file.   
     """
 
 
 class IPyAutocall:
     """ Instances of this class are always autocalled
     
     This happens regardless of 'autocall' variable state. Use this to
     develop macro-like mechanisms.
     """
     
     def set_ip(self,ip):
         """ Will be used to set _ip point to current ipython instance b/f call
         
         Override this method if you don't want this to happen.
         
         """
         self._ip = ip
     
 
 class IPythonNotRunning:
     """Dummy do-nothing class.
 
     Instances of this class return a dummy attribute on all accesses, which
     can be called and warns.  This makes it easier to write scripts which use
     the ipapi.get() object for informational purposes to operate both with and
     without ipython.  Obviously code which uses the ipython object for
     computations will not work, but this allows a wider range of code to
     transparently work whether ipython is being used or not."""
 
     def __init__(self,warn=True):
         if warn:
             self.dummy = self._dummy_warn
         else:
             self.dummy = self._dummy_silent
     
     def __str__(self):
         return "<IPythonNotRunning>"
 
     __repr__ = __str__
 
     def __getattr__(self,name):
         return self.dummy
 
     def _dummy_warn(self,*args,**kw):
         """Dummy function, which doesn't do anything but warn."""
 
         print ("IPython is not running, this is a dummy no-op function")
 
     def _dummy_silent(self,*args,**kw):
         """Dummy function, which doesn't do anything and emits no warnings."""
         pass
 
 
 def get(allow_dummy=False,dummy_warn=True):
     """Get an IPApi object.
 
     If allow_dummy is true, returns an instance of IPythonNotRunning 
     instead of None if not running under IPython.
 
     If dummy_warn is false, the dummy instance will be completely silent.
 
     Running this should be the first thing you do when writing extensions that
     can be imported as normal modules. You can then direct all the
     configuration operations against the returned object.
     """
     global _RECENT_IP
     if allow_dummy and not _RECENT_IP:
         _RECENT_IP = IPythonNotRunning(dummy_warn)
     return _RECENT_IP
 
 
 class IPApi(object):
     """ The actual API class for configuring IPython 
     
     You should do all of the IPython configuration by getting an IPApi object
     with IPython.ipapi.get() and using the attributes and methods of the
     returned object."""
     
     def __init__(self,ip):
         
         global _RECENT_IP
 
         # All attributes exposed here are considered to be the public API of
         # IPython.  As needs dictate, some of these may be wrapped as
         # properties.
 
         self.magic = ip.ipmagic
         
         self.system = ip.system
         
         self.set_hook = ip.set_hook
         
         self.set_custom_exc = ip.set_custom_exc
 
         self.user_ns = ip.user_ns
         self.user_ns['_ip'] = self
 
         self.set_crash_handler = ip.set_crash_handler
 
         # Session-specific data store, which can be used to store
         # data that should persist through the ipython session.
         self.meta =  ip.meta
     
         # The ipython instance provided
         self.IP = ip
 
         self.extensions = {}
 
         self.dbg = DebugTools(self)
         
         _RECENT_IP = self
 
     # Use a property for some things which are added to the instance very
     # late.  I don't have time right now to disentangle the initialization
     # order issues, so a property lets us delay item extraction while
     # providing a normal attribute API.
     def get_db(self):
         """A handle to persistent dict-like database (a PickleShareDB object)"""
         return self.IP.db
 
     db = property(get_db,None,None,get_db.__doc__)
 
     def get_options(self):
         """All configurable variables."""
         
         # catch typos by disabling new attribute creation. If new attr creation
         # is in fact wanted (e.g. when exposing new options), do
         # allow_new_attr(True) for the received rc struct.
         
         self.IP.rc.allow_new_attr(False)
         return self.IP.rc
 
     options = property(get_options,None,None,get_options.__doc__)
     
     def expose_magic(self,magicname, func):
         """Expose own function as magic function for ipython 
     
         def foo_impl(self,parameter_s=''):
             'My very own magic!. (Use docstrings, IPython reads them).'
             print 'Magic function. Passed parameter is between < >:'
             print '<%s>' % parameter_s
             print 'The self object is:',self
     
         ipapi.expose_magic('foo',foo_impl)
         """
         
         import new
         im = new.instancemethod(func,self.IP, self.IP.__class__)
         old = getattr(self.IP, "magic_" + magicname, None)
         if old:
             self.dbg.debug_stack("Magic redefinition '%s', old %s" %
                                  (magicname,old) )
             
         setattr(self.IP, "magic_" + magicname, im)
     
     def ex(self,cmd):
         """ Execute a normal python statement in user namespace """
         exec cmd in self.user_ns
     
     def ev(self,expr):
         """ Evaluate python expression expr in user namespace 
         
         Returns the result of evaluation"""
         return eval(expr,self.user_ns)
     
     def runlines(self,lines):
         """ Run the specified lines in interpreter, honoring ipython directives.
         
         This allows %magic and !shell escape notations.
         
         Takes either all lines in one string or list of lines.
         """
 
         def cleanup_ipy_script(script):
             """ Make a script safe for _ip.runlines() 
             
             - Removes empty lines Suffixes all indented blocks that end with
             - unindented lines with empty lines
             """
             
             res = []
             lines = script.splitlines()
 
             level = 0
             for l in lines:
                 lstripped = l.lstrip()
                 stripped = l.strip()                
                 if not stripped:
                     continue
                 newlevel = len(l) - len(lstripped)
                 def is_secondary_block_start(s):
                     if not s.endswith(':'):
                         return False
                     if (s.startswith('elif') or 
                         s.startswith('else') or 
                         s.startswith('except') or
                         s.startswith('finally')):
                         return True
                         
                 if level > 0 and newlevel == 0 and \
                        not is_secondary_block_start(stripped): 
                     # add empty line
                     res.append('')
                     
                 res.append(l)
                 level = newlevel
             return '\n'.join(res) + '\n'
         
         if isinstance(lines,basestring):
             script = lines            
         else:
             script = '\n'.join(lines)
         clean=cleanup_ipy_script(script)
         # print "_ip.runlines() script:\n",clean # dbg
         self.IP.runlines(clean)
         
     def to_user_ns(self,vars, interactive = True):
         """Inject a group of variables into the IPython user namespace.
 
         Inputs:
 
          - vars: string with variable names separated by whitespace, or a
          dict with name/value pairs.
 
          - interactive: if True (default), the var will be listed with
         %whos et. al.
          
         This utility routine is meant to ease interactive debugging work,
         where you want to easily propagate some internal variable in your code
         up to the interactive namespace for further exploration.
 
         When you run code via %run, globals in your script become visible at
         the interactive prompt, but this doesn't happen for locals inside your
         own functions and methods.  Yet when debugging, it is common to want
         to explore some internal variables further at the interactive propmt.
 
         Examples:
 
         To use this, you first must obtain a handle on the ipython object as
         indicated above, via:
 
         import IPython.ipapi
         ip = IPython.ipapi.get()
 
         Once this is done, inside a routine foo() where you want to expose
         variables x and y, you do the following:
 
         def foo():
             ...
             x = your_computation()
             y = something_else()
             
             # This pushes x and y to the interactive prompt immediately, even
             # if this routine crashes on the next line after:
             ip.to_user_ns('x y')
             ...
             
             # To expose *ALL* the local variables from the function, use:
             ip.to_user_ns(locals())
 
             ...
             # return           
         
 
         If you need to rename variables, the dict input makes it easy.  For
         example, this call exposes variables 'foo' as 'x' and 'bar' as 'y'
         in IPython user namespace:
 
         ip.to_user_ns(dict(x=foo,y=bar))    
         """
 
         # print 'vars given:',vars # dbg
         
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(vars,dict):
             # If a dict was given, no need to change anything.
             vdict = vars
         elif isinstance(vars,basestring):
             # If a string with names was given, get the caller's frame to
             # evaluate the given names in
             cf = sys._getframe(1)
             vdict = {}
             for name in vars.split():
                 try:
                     vdict[name] = eval(name,cf.f_globals,cf.f_locals)
                 except:
                     print ('could not get var. %s from %s' %
                     (name,cf.f_code.co_name))
         else:
             raise ValueError('vars must be a string or a dict')
             
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         config_ns = self.IP.user_config_ns
         if interactive:
             for name,val in vdict.iteritems():
                 config_ns.pop(name,None)
         else:
             for name,val in vdict.iteritems():
                 config_ns[name] = val                    
 
     def expand_alias(self,line):
         """ Expand an alias in the command line 
         
         Returns the provided command line, possibly with the first word 
         (command) translated according to alias expansion rules.
         
         [ipython]|16> _ip.expand_aliases("np myfile.txt")
                  <16> 'q:/opt/np/notepad++.exe myfile.txt'
         """
         
         pre,fn,rest = self.IP.split_user_input(line)
         res = pre + self.IP.expand_aliases(fn,rest)
         return res
 
     def itpl(self, s, depth = 1):
         """ Expand Itpl format string s.
         
         Only callable from command line (i.e. prefilter results);
         If you use in your scripts, you need to use a bigger depth!
         """
         return self.IP.var_expand(s, depth)
         
     def defalias(self, name, cmd):
         """ Define a new alias
         
         _ip.defalias('bb','bldmake bldfiles')
         
         Creates a new alias named 'bb' in ipython user namespace
         """
 
         self.dbg.check_hotname(name)
         
         if name in self.IP.alias_table:
             self.dbg.debug_stack("Alias redefinition: '%s' => '%s' (old '%s')"
                                  % (name, cmd, self.IP.alias_table[name]))
 
         if callable(cmd):
             self.IP.alias_table[name] = cmd
             import IPython.shadowns
             setattr(IPython.shadowns, name,cmd)
             return
             
         if isinstance(cmd,basestring):
             nargs = cmd.count('%s')
             if nargs>0 and cmd.find('%l')>=0:
                 raise Exception('The %s and %l specifiers are mutually '
                                 'exclusive in alias definitions.')
                   
             self.IP.alias_table[name] = (nargs,cmd)
             return
         
         # just put it in - it's probably (0,'foo')
         self.IP.alias_table[name] = cmd
     
     def defmacro(self, *args):
         """ Define a new macro
     
         2 forms of calling:
         
         mac = _ip.defmacro('print "hello"\nprint "world"')
         
         (doesn't put the created macro on user namespace)
         
         _ip.defmacro('build', 'bldmake bldfiles\nabld build winscw udeb')
         
         (creates a macro named 'build' in user namespace)
         """
         
         import IPython.macro
         
         if len(args) == 1:
             return IPython.macro.Macro(args[0])
         elif len(args) == 2:
             self.user_ns[args[0]] = IPython.macro.Macro(args[1])
         else:
             return Exception("_ip.defmacro must be called with 1 or 2 arguments")
         
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
         
         Requires readline.
         
         Example:
         
         [D:\ipython]|1> _ip.set_next_input("Hello Word")
         [D:\ipython]|2> Hello Word_  # cursor is here        
         """
 
         self.IP.rl_next_input = s
 
     def load(self, mod):
         """ Load an extension.
         
         Some modules should (or must) be 'load()':ed, rather than just imported.
         
         Loading will do:
         
         - run init_ipython(ip)
         - run ipython_firstrun(ip)
         """
         
         if mod in self.extensions:
             # just to make sure we don't init it twice
             # note that if you 'load' a module that has already been
             # imported, init_ipython gets run anyway
             
             return self.extensions[mod]
         __import__(mod)
         m = sys.modules[mod]
         if hasattr(m,'init_ipython'):
             m.init_ipython(self)
             
         if hasattr(m,'ipython_firstrun'):
             already_loaded = self.db.get('firstrun_done', set())
             if mod not in already_loaded:
                 m.ipython_firstrun(self)
                 already_loaded.add(mod)
                 self.db['firstrun_done'] = already_loaded
             
         self.extensions[mod] = m
         return m
 
 
 class DebugTools:
     """ Used for debugging mishaps in api usage
     
     So far, tracing redefinitions is supported.
     """
     
     def __init__(self, ip):
         self.ip = ip
         self.debugmode = False
         self.hotnames = set()
         
     def hotname(self, name_to_catch):
         self.hotnames.add(name_to_catch)
         
     def debug_stack(self, msg = None):
         if not self.debugmode:
             return
         
         import traceback
         if msg is not None:
             print '====== %s  ========' % msg
         traceback.print_stack()
 
     def check_hotname(self,name):
         if name in self.hotnames:
             self.debug_stack( "HotName '%s' caught" % name)
 
 
 def launch_new_instance(user_ns = None,shellclass = None):
     """ Make and start a new ipython instance.
     
     This can be called even without having an already initialized 
     ipython session running.
     
     This is also used as the egg entry point for the 'ipython' script.
     
     """
     ses = make_session(user_ns,shellclass)
     ses.mainloop()
 
 
 def make_user_ns(user_ns = None):
     """Return a valid user interactive namespace.
 
     This builds a dict with the minimal information needed to operate as a
     valid IPython user namespace, which you can pass to the various embedding
     classes in ipython.
 
     This API is currently deprecated. Use ipapi.make_user_namespaces() instead
     to make both the local and global namespace objects simultaneously.
 
     :Parameters:
         user_ns : dict-like, optional
             The current user namespace. The items in this namespace should be
             included in the output. If None, an appropriate blank namespace
             should be created.
 
     :Returns:
         A dictionary-like object to be used as the local namespace of the
         interpreter.
     """
 
     raise NotImplementedError
 
 
 def make_user_global_ns(ns = None):
     """Return a valid user global namespace.
 
     Similar to make_user_ns(), but global namespaces are really only needed in
     embedded applications, where there is a distinction between the user's
     interactive namespace and the global one where ipython is running.
 
     This API is currently deprecated. Use ipapi.make_user_namespaces() instead
     to make both the local and global namespace objects simultaneously.
 
     :Parameters:
         ns : dict, optional
             The current user global namespace. The items in this namespace
             should be included in the output. If None, an appropriate blank
             namespace should be created.
 
     :Returns:
         A true dict to be used as the global namespace of the interpreter.
     """
 
     raise NotImplementedError
 
 # Record the true objects in order to be able to test if the user has overridden
 # these API functions.
 _make_user_ns = make_user_ns
 _make_user_global_ns = make_user_global_ns
 
 
 def make_user_namespaces(user_ns = None,user_global_ns = None):
     """Return a valid local and global user interactive namespaces.
 
     This builds a dict with the minimal information needed to operate as a
     valid IPython user namespace, which you can pass to the various embedding
     classes in ipython.  The default implementation returns the same dict for
     both the locals and the globals to allow functions to refer to variables in
     the namespace.  Customized implementations can return different dicts.  The
     locals dictionary can actually be anything following the basic mapping
     protocol of a dict, but the globals dict must be a true dict, not even
     a subclass.  It is recommended that any custom object for the locals
     namespace synchronize with the globals dict somehow.
 
     Raises TypeError if the provided globals namespace is not a true dict.
 
     :Parameters:
         user_ns : dict-like, optional
             The current user namespace. The items in this namespace should be
             included in the output. If None, an appropriate blank namespace
             should be created.
         user_global_ns : dict, optional
             The current user global namespace. The items in this namespace
             should be included in the output. If None, an appropriate blank
             namespace should be created.
 
     :Returns:
         A tuple pair of dictionary-like object to be used as the local namespace
         of the interpreter and a dict to be used as the global namespace.
     """
 
     if user_ns is None:
         if make_user_ns is not _make_user_ns:
             # Old API overridden.
             # FIXME: Issue DeprecationWarning, or just let the old API live on?
             user_ns = make_user_ns(user_ns)
         else:
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             user_ns = {'__name__'     :'__main__',
                        '__builtins__' : __builtin__,
                        }
     else:
         user_ns.setdefault('__name__','__main__')
         user_ns.setdefault('__builtins__',__builtin__)
 
     if user_global_ns is None:
         if make_user_global_ns is not _make_user_global_ns:
             # Old API overridden.
             user_global_ns = make_user_global_ns(user_global_ns)
         else:
             user_global_ns = user_ns
     if type(user_global_ns) is not dict:
         raise TypeError("user_global_ns must be a true dict; got %r"
             % type(user_global_ns))
 
     return user_ns, user_global_ns
 
 
 def make_session(user_ns = None, shellclass = None):
     """Makes, but does not launch an IPython session.
     
     Later on you can call obj.mainloop() on the returned object.
 
     Inputs:
 
       - user_ns(None): a dict to be used as the user's namespace with initial
       data.
     
     WARNING: This should *not* be run when a session exists already."""
 
     import IPython.Shell
     if shellclass is None:
         return IPython.Shell.start(user_ns)
     return shellclass(user_ns = user_ns)

IPython/kernel/core/util.py

@@ -1,197 +1,197
 # encoding: utf-8
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os
 import sys
 
 
 # This class is mostly taken from IPython.
 class InputList(list):
     """ Class to store user input.
 
     It's basically a list, but slices return a string instead of a list, thus
     allowing things like (assuming 'In' is an instance):
 
     exec In[4:7]
 
     or
 
     exec In[5:9] + In[14] + In[21:25]
     """
 
     def __getslice__(self, i, j):
         return ''.join(list.__getslice__(self, i, j))
 
     def add(self, index, command):
         """ Add a command to the list with the appropriate index.
 
         If the index is greater than the current length of the list, empty
         strings are added in between.
         """
 
         length = len(self)
         if length == index:
             self.append(command)
         elif length > index:
             self[index] = command
         else:
             extras = index - length
             self.extend([''] * extras)
             self.append(command)
 
 
 class Bunch(dict):
     """ A dictionary that exposes its keys as attributes.
     """
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         self.__dict__ = self
 
 
 def esc_quotes(strng):
     """ Return the input string with single and double quotes escaped out.
     """
 
     return strng.replace('"', '\\"').replace("'", "\\'")
 
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
     
-    Effectively this turns string: cd \ao\ao\
-    to: r"cd \ao\ao\_"[:-1]
+    XXX - example removed because it caused encoding errors in documentation
+    generation.  We need a new example that doesn't contain invalid chars.
     
     Note the use of raw string and padding at the end to allow trailing
     backslash.
     """
     
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # Give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = ''.join([raw, quote, s, tailpadding, quote, tail])
     return res
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def system_shell(cmd, verbose=False, debug=False, header=''):
     """ Execute a command in the system shell; always return None.
 
     Parameters
     ----------
     cmd : str
         The command to execute.
     verbose : bool
         If True, print the command to be executed.
     debug : bool
         Only print, do not actually execute.
     header : str
         Header to print to screen prior to the executed command. No extra
         newlines are added.
 
     Description
     -----------
     This returns None so it can be conveniently used in interactive loops
     without getting the return value (typically 0) printed many times.
     """
 
     if verbose or debug: 
         print header + cmd
 
     # Flush stdout so we don't mangle python's buffering.
     sys.stdout.flush()
     if not debug:
         os.system(cmd)
 
 # Override shell() for win32 to deal with network shares.
 if os.name in ('nt', 'dos'):
 
     system_shell_ori = system_shell
 
     def system_shell(cmd, verbose=False, debug=False, header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # Change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses).
             os.chdir("c:")
             # Issue pushd to the UNC-share and then run the command.
             try:
                 system_shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             system_shell_ori(cmd,verbose,debug,header)
 
     system_shell.__doc__ = system_shell_ori.__doc__
 
 def getoutputerror(cmd, verbose=False, debug=False, header='', split=False):
     """ Executes a command and returns the output.
 
     Parameters
     ----------
     cmd : str
         The command to execute.
     verbose : bool
         If True, print the command to be executed.
     debug : bool
         Only print, do not actually execute.
     header : str
         Header to print to screen prior to the executed command. No extra
         newlines are added.
     split : bool
         If True, return the output as a list split on newlines.
 
     """
 
     if verbose or debug: 
         print header+cmd
 
     if not cmd:
         # Return empty lists or strings.
         if split:
             return [], []
         else:
             return '', ''
 
     if not debug:
         # fixme: use subprocess.
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'), terr.split('\n')
         else:
             return tout, terr
 

IPython/testing/decorators.py

@@ -1,244 +1,244
 """Decorators for labeling test objects.
 
 Decorators that merely return a modified version of the original function
 object are straightforward.  Decorators that return a new function object need
 to use nose.tools.make_decorator(original_function)(decorator) in returning the
 decorator, in order to preserve metadata such as function name, setup and
 teardown functions and so on - see nose.tools for more information.
 
 This module provides a set of useful decorators meant to be ready to use in
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
 """
 
 # Stdlib imports
 import inspect
 import sys
 
 # Third-party imports
 
 # This is Michele Simionato's decorator module, also kept verbatim.
 from decorator_msim import decorator, update_wrapper
 
 # Grab the numpy-specific decorators which we keep in a file that we
 # occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
 # of numpy.testing.decorators.
 from decorators_numpy import *
 
 ##############################################################################
 # Local code begins
 
 # Utility functions
 
 def apply_wrapper(wrapper,func):
     """Apply a wrapper to a function for decoration.
 
     This mixes Michele Simionato's decorator tool with nose's make_decorator,
     to apply a wrapper in a decorator so that all nose attributes, as well as
     function signature and other properties, survive the decoration cleanly.
     This will ensure that wrapped functions can still be well introspected via
     IPython, for example.
     """
     import nose.tools
 
     return decorator(wrapper,nose.tools.make_decorator(func)(wrapper))
 
 
 def make_label_dec(label,ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
     :Parameters:
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
     :Keywords:
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
     :Returns:
       A decorator.
 
     :Examples:
 
     A simple labeling decorator:
     >>> slow = make_label_dec('slow')
     >>> print slow.__doc__
     Labels a test as 'slow'.
 
     And one that uses multiple labels and a custom docstring:
     >>> rare = make_label_dec(['slow','hard'],
     ... "Mix labels 'slow' and 'hard' for rare tests.")
     >>> print rare.__doc__
     Mix labels 'slow' and 'hard' for rare tests.
 
     Now, let's test using this one:
     >>> @rare
     ... def f(): pass
     ...
     >>>
     >>> f.slow
     True
     >>> f.hard
     True
     """
 
     if isinstance(label,basestring):
         labels = [label]
     else:
         labels = label
         
     # Validate that the given label(s) are OK for use in setattr() by doing a
     # dry run on a dummy function.
     tmp = lambda : None
     for label in labels:
         setattr(tmp,label,True)
 
     # This is the actual decorator we'll return
     def decor(f):
         for label in labels:
             setattr(f,label,True)
         return f
     
     # Apply the user's docstring, or autogenerate a basic one
     if ds is None:
         ds = "Labels a test as %r." % label
     decor.__doc__ = ds
     
     return decor
 
 
 # Inspired by numpy's skipif, but uses the full apply_wrapper utility to
 # preserve function metadata better and allows the skip condition to be a
 # callable.
 def skipif(skip_condition, msg=None):
     ''' Make function raise SkipTest exception if skip_condition is true
 
     Parameters
-    ---------
+    ----------
     skip_condition : bool or callable. 
  	Flag to determine whether to skip test.  If the condition is a 
  	callable, it is used at runtime to dynamically make the decision.  This 
  	is useful for tests that may require costly imports, to delay the cost 
  	until the test suite is actually executed.        
     msg : string
         Message to give on raising a SkipTest exception
 
    Returns
    -------
    decorator : function
        Decorator, which, when applied to a function, causes SkipTest
        to be raised when the skip_condition was True, and the function
        to be called normally otherwise.
 
     Notes
     -----
     You will see from the code that we had to further decorate the
     decorator with the nose.tools.make_decorator function in order to
     transmit function name, and various other metadata.
     '''
 
     def skip_decorator(f):
         # Local import to avoid a hard nose dependency and only incur the 
         # import time overhead at actual test-time. 
         import nose
 
         # Allow for both boolean or callable skip conditions.
         if callable(skip_condition):
             skip_val = lambda : skip_condition()
         else:
             skip_val = lambda : skip_condition
 
         def get_msg(func,msg=None):
             """Skip message with information about function being skipped."""
             if msg is None: out = 'Test skipped due to test condition.'
             else: out = msg
             return "Skipping test: %s. %s" % (func.__name__,out)
 
         # We need to define *two* skippers because Python doesn't allow both
         # return with value and yield inside the same function.
         def skipper_func(*args, **kwargs):
             """Skipper for normal test functions."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 return f(*args, **kwargs) 
 
         def skipper_gen(*args, **kwargs):
             """Skipper for test generators."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 for x in f(*args, **kwargs):
                     yield x
 
         # Choose the right skipper to use when building the actual generator.
         if nose.util.isgenerator(f):
             skipper = skipper_gen
         else:
             skipper = skipper_func
             
         return nose.tools.make_decorator(f)(skipper)
 
     return skip_decorator
 
 # A version with the condition set to true, common case just to attacha message
 # to a skip decorator
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
     :Parameters:
       msg : string
         Optional message to be added.
 
     :Returns:
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
       """
 
     return skipif(True,msg)
 
 
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
 def numpy_not_available():
     """Can numpy be imported?  Returns true if numpy does NOT import.
 
     This is used to make a decorator to skip tests that require numpy to be
     available, but delay the 'import numpy' to test execution time.
     """
     try:
         import numpy
         np_not_avail = False
     except ImportError:
         np_not_avail = True
 
     return np_not_avail
 
 #-----------------------------------------------------------------------------
 # Decorators for public use
 
 skip_doctest = make_label_dec('skip_doctest',
     """Decorator - mark a function or method for skipping its doctest.
 
     This decorator allows you to mark a function whose docstring you wish to
     omit from testing, while preserving the docstring for introspection, help,
     etc.""")                              
 
 # Decorators to skip certain tests on specific platforms.
 skip_win32 = skipif(sys.platform=='win32',
                     "This test does not run under Windows")
 skip_linux = skipif(sys.platform=='linux2',"This test does not run under Linux")
 skip_osx = skipif(sys.platform=='darwin',"This test does not run under OS X")
 
 
 skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
 
 skipknownfailure = skip('This test is known to fail')

IPython/testing/decorators_numpy.py

@@ -1,97 +1,97
 """Decorators for labeling test objects
 
 Decorators that merely return a modified version of the original
 function object are straightforward.  Decorators that return a new
 function object need to use
 nose.tools.make_decorator(original_function)(decorator) in returning
 the decorator, in order to preserve metadata such as function name,
 setup and teardown functions and so on - see nose.tools for more
 information.
 
 """
 
 def slow(t):
     """Labels a test as 'slow'.
 
     The exact definition of a slow test is obviously both subjective and
     hardware-dependent, but in general any individual test that requires more
     than a second or two should be labeled as slow (the whole suite consits of
     thousands of tests, so even a second is significant)."""
 
     t.slow = True
     return t
 
 def setastest(tf=True):
     ''' Signals to nose that this function is or is not a test
 
     Parameters
     ----------
     tf : bool
         If True specifies this is a test, not a test otherwise
 
     e.g
     >>> from numpy.testing.decorators import setastest
     >>> @setastest(False)
     ... def func_with_test_in_name(arg1, arg2): pass
     ...
     >>>
 
     This decorator cannot use the nose namespace, because it can be
     called from a non-test module. See also istest and nottest in
     nose.tools
 
     '''
     def set_test(t):
         t.__test__ = tf
         return t
     return set_test
 
 def skipif(skip_condition=True, msg=None):
     ''' Make function raise SkipTest exception if skip_condition is true
 
     Parameters
-    ---------
+    ----------
     skip_condition : bool or callable. 
  	Flag to determine whether to skip test.  If the condition is a 
  	callable, it is used at runtime to dynamically make the decision.  This 
  	is useful for tests that may require costly imports, to delay the cost 
  	until the test suite is actually executed.        
     msg : string
         Message to give on raising a SkipTest exception
 
    Returns
    -------
    decorator : function
        Decorator, which, when applied to a function, causes SkipTest
        to be raised when the skip_condition was True, and the function
        to be called normally otherwise.
 
     Notes
     -----
     You will see from the code that we had to further decorate the
     decorator with the nose.tools.make_decorator function in order to
     transmit function name, and various other metadata.
     '''
     if msg is None:
         msg = 'Test skipped due to test condition'
     def skip_decorator(f):
         # Local import to avoid a hard nose dependency and only incur the 
         # import time overhead at actual test-time. 
         import nose
         def skipper(*args, **kwargs):
             if skip_condition:
                 raise nose.SkipTest, msg
             else:
                 return f(*args, **kwargs)
         return nose.tools.make_decorator(f)(skipper)
     return skip_decorator
 
 def skipknownfailure(f):
     ''' Decorator to raise SkipTest for test known to fail
     '''
     # Local import to avoid a hard nose dependency and only incur the 
     # import time overhead at actual test-time. 
     import nose
     def skipper(*args, **kwargs):
         raise nose.SkipTest, 'This test is known to fail'
     return nose.tools.make_decorator(f)(skipper)

docs/Makefile

@@ -1,86 +1,91
 # Makefile for Sphinx documentation
 #
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
+SRCDIR        = source
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+ALLSPHINXOPTS   = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SRCDIR)
 
-.PHONY: help clean html web pickle htmlhelp latex changes linkcheck
+.PHONY: help clean html web pickle htmlhelp latex changes linkcheck api
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html      to make standalone HTML files"
 	@echo "  pickle    to make pickle files (usable by e.g. sphinx-web)"
 	@echo "  htmlhelp  to make HTML files and a HTML help project"
 	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 	@echo "  changes   to make an overview over all changed/added/deprecated items"
 	@echo "  linkcheck to check all external links for integrity"
 	@echo
 	@echo "Compound utility targets:"
 	@echo "pdf          latex and then runs the PDF generation"
 	@echo "all          html and pdf"
 	@echo "dist         all, and then puts the results in dist/"
 
 clean:
-	-rm -rf build/* dist/*
+	-rm -rf build/* dist/* $(SRCDIR)/api/generated
 
 pdf: latex
 	cd build/latex && make all-pdf
 
 all: html pdf
 
 dist: clean all
 	mkdir -p dist
 	ln build/latex/ipython.pdf dist/
 	cp -al build/html dist/
 	@echo "Build finished.  Final docs are in dist/"
 
-html:
+html: api
 	mkdir -p build/html build/doctrees
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html
 	@echo
 	@echo "Build finished. The HTML pages are in build/html."
 
+api:
+	python autogen_api.py
+	@echo "Build API docs finished."
+
 pickle:
 	mkdir -p build/pickle build/doctrees
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files or run"
 	@echo "  sphinx-web build/pickle"
 	@echo "to start the sphinx-web server."
 
 web: pickle
 
 htmlhelp:
 	mkdir -p build/htmlhelp build/doctrees
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in build/htmlhelp."
 
 latex:
 	mkdir -p build/latex build/doctrees
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in build/latex."
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
 	      "run these through (pdf)latex."
 
 changes:
 	mkdir -p build/changes build/doctrees
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes
 	@echo
 	@echo "The overview file is in build/changes."
 
 linkcheck:
 	mkdir -p build/linkcheck build/doctrees
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in build/linkcheck/output.txt."

docs/source/conf.py

@@ -1,187 +1,191
 # -*- coding: utf-8 -*-
 #
 # IPython documentation build configuration file.
 
 # NOTE: This file has been edited manually from the auto-generated one from
 # sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
 # needed, generate a scratch one and merge by hand any new fields needed.
 
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
 # The contents of this file are pickled, so don't put values in the namespace
 # that aren't pickleable (module imports are okay, they're removed automatically).
 #
 # All configuration values have a default value; values that are commented out
 # serve to show the default value.
 
 import sys, os
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
 sys.path.append(os.path.abspath('../sphinxext'))
 
 # Import support for ipython console session syntax highlighting (lives
 # in the sphinxext directory defined above)
 import ipython_console_highlighting
 
 # We load the ipython release info into a dict by explicit execution
 iprelease = {}
 execfile('../../IPython/Release.py',iprelease)
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc',
-              'inheritance_diagram', 'only_directives',
+              'sphinx.ext.doctest',
+              
+              'only_directives',
+              'inheritance_diagram',
               'ipython_console_highlighting', 
               # 'plot_directive', # disabled for now, needs matplotlib
+              'numpydoc',  # to preprocess docstrings
               ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
 # The suffix of source filenames.
 source_suffix = '.txt'
 
 # The master toctree document.
 master_doc = 'index'
 
 # General substitutions.
 project = 'IPython'
 copyright = '2008, The IPython Development Team'
 
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
 #
 # The full version, including alpha/beta/rc tags.
 release = iprelease['version']
 # The short X.Y version.
 version = '.'.join(release.split('.',2)[:2])
 
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
 today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
 exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
 #add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
 
 # Options for HTML output
 # -----------------------
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
 html_style = 'default.css'
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
 #html_title = None
 
 # The name of an image file (within the static path) to place at the top of
 # the sidebar.
 #html_logo = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
 #html_additional_pages = {}
 
 # If false, no module index is generated.
 #html_use_modindex = True
 
 # If true, the reST sources are included in the HTML build as _sources/<name>.
 #html_copy_source = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
 #html_use_opensearch = ''
 
 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
 # ------------------------
 
 # The paper size ('letter' or 'a4').
 latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
 latex_font_size = '11pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
                      ur"""The IPython Development Team""",
                      'manual'),
                     ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
 #latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
 #latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
 
 # Documents to append as an appendix to all manuals.
 #latex_appendices = []
 
 # If false, no module index is generated.
 #latex_use_modindex = True
 
 
 # Cleanup
 # -------
 # delete release info to avoid pickling errors from sphinx
 
 del iprelease

docs/source/index.txt

@@ -1,32 +1,33
 =====================
 IPython Documentation
 =====================
 
 .. htmlonly::
 
    :Release: |release|
    :Date: |today|
 
    Contents:
 
 .. toctree::
    :maxdepth: 2
 
    overview.txt
    install/index.txt
    interactive/index.txt
    parallel/index.txt
    config/index.txt
-   changes.txt
-   development/index.txt
    faq.txt
    history.txt
+   changes.txt
+   development/index.txt
+   api/index.txt
    license_and_copyright.txt
    credits.txt
 
 
 .. htmlonly::
 
   * :ref:`genindex`
   * :ref:`modindex`
   * :ref:`search`

docs/source/interactive/reference.txt

@@ -1,3162 +1,1599
 .. IPython documentation master file, created by sphinx-quickstart.py on Mon Mar 24 17:01:34 2008.
    You can adapt this file completely to your liking, but it should at least
    contain the root 'toctree' directive.
 
 =================
 IPython reference
 =================
 
 .. contents::
 
 .. _command_line_options:
 
 Command-line usage
 ==================
 
 You start IPython with the command::
 
     $ ipython [options] files
 
 If invoked with no options, it executes all the files listed in sequence
 and drops you into the interpreter while still acknowledging any options
 you may have set in your ipythonrc file. This behavior is different from
 standard Python, which when called as python -i will only execute one
 file and ignore your configuration setup.
 
 Please note that some of the configuration options are not available at
 the command line, simply because they are not practical here. Look into
 your ipythonrc configuration file for details on those. This file
 typically installed in the $HOME/.ipython directory. For Windows users,
 $HOME resolves to C:\\Documents and Settings\\YourUserName in most
 instances. In the rest of this text, we will refer to this directory as
 IPYTHONDIR.
 
 .. _Threading options:
 
 
 Special Threading Options
 -------------------------
 
 The following special options are ONLY valid at the beginning of the
 command line, and not later. This is because they control the initial-
 ization of ipython itself, before the normal option-handling mechanism
 is active.
 
     -gthread, -qthread, -q4thread, -wthread, -pylab: 
 	Only one of these can be given, and it can only be given as
 	the first option passed to IPython (it will have no effect in
 	any other position).  They provide threading support for the
 	GTK, Qt (versions 3 and 4) and WXPython toolkits, and for the
 	matplotlib library.
 
 	With any of the first four options, IPython starts running a
 	separate thread for the graphical toolkit's operation, so that
 	you can open and control graphical elements from within an
 	IPython command line, without blocking. All four provide
 	essentially the same functionality, respectively for GTK, Qt3,
 	Qt4 and WXWidgets (via their Python interfaces).
 
 	Note that with -wthread, you can additionally use the
 	-wxversion option to request a specific version of wx to be
 	used.  This requires that you have the wxversion Python module
 	installed, which is part of recent wxPython distributions.
 
 	If -pylab is given, IPython loads special support for the mat
 	plotlib library (http://matplotlib.sourceforge.net), allowing
 	interactive usage of any of its backends as defined in the
 	user's ~/.matplotlib/matplotlibrc file. It automatically
 	activates GTK, Qt or WX threading for IPyhton if the choice of
 	matplotlib backend requires it. It also modifies the %run
 	command to correctly execute (without blocking) any
 	matplotlib-based script which calls show() at the end.
 
     -tk
 	The -g/q/q4/wthread options, and -pylab (if matplotlib is
 	configured to use GTK, Qt3, Qt4 or WX), will normally block Tk
 	graphical interfaces. This means that when either GTK, Qt or WX
 	threading is active, any attempt to open a Tk GUI will result in a
 	dead window, and possibly cause the Python interpreter to crash.
 	An extra option, -tk, is available to address this issue. It can
 	only be given as a second option after any of the above (-gthread,
 	-wthread or -pylab).
 
 	If -tk is given, IPython will try to coordinate Tk threading
 	with GTK, Qt or WX. This is however potentially unreliable, and
 	you will have to test on your platform and Python configuration to
 	determine whether it works for you. Debian users have reported
 	success, apparently due to the fact that Debian builds all of Tcl,
 	Tk, Tkinter and Python with pthreads support. Under other Linux
 	environments (such as Fedora Core 2/3), this option has caused
 	random crashes and lockups of the Python interpreter. Under other
 	operating systems (Mac OSX and Windows), you'll need to try it to
 	find out, since currently no user reports are available.
     
 	There is unfortunately no way for IPython to determine at run time
 	whether -tk will work reliably or not, so you will need to do some
 	experiments before relying on it for regular work.
 
 
 
 Regular Options
 ---------------
 
 After the above threading options have been given, regular options can
 follow in any order. All options can be abbreviated to their shortest
 non-ambiguous form and are case-sensitive. One or two dashes can be
 used. Some options have an alternate short form, indicated after a ``|``.
 
 Most options can also be set from your ipythonrc configuration file. See
 the provided example for more details on what the options do. Options
 given at the command line override the values set in the ipythonrc file.
 
 All options with a [no] prepended can be specified in negated form
 (-nooption instead of -option) to turn the feature off.
 
     -help  print a help message and exit.
 
     -pylab
 	this can only be given as the first option passed to IPython
 	(it will have no effect in any other position). It adds
 	special support for the matplotlib library
 	(http://matplotlib.sourceforge.ne), allowing interactive usage
 	of any of its backends as defined in the user's .matplotlibrc
 	file.  It automatically activates GTK or WX threading for
 	IPyhton if the choice of matplotlib backend requires it. It
 	also modifies the %run command to correctly execute (without
 	blocking) any matplotlib-based script which calls show() at
 	the end. See `Matplotlib support`_ for more details.
 
     -autocall <val>
 	Make IPython automatically call any callable object even if you
 	didn't type explicit parentheses. For example, 'str 43' becomes
 	'str(43)' automatically. The value can be '0' to disable the feature,
 	'1' for smart autocall, where it is not applied if there are no more
 	arguments on the line, and '2' for full autocall, where all callable
 	objects are automatically called (even if no arguments are
 	present). The default is '1'.
 
     -[no]autoindent    
 	Turn automatic indentation on/off.
 
     -[no]automagic 
 	make magic commands automatic (without needing their first character
 	to be %). Type %magic at the IPython prompt for more information.
 
     -[no]autoedit_syntax 
 	When a syntax error occurs after editing a file, automatically
 	open the file to the trouble causing line for convenient
 	fixing.
 
     -[no]banner		Print the initial information banner (default on).
 
     -c <command>
 	execute the given command string. This is similar to the -c
 	option in the normal Python interpreter.
 
     -cache_size, cs <n> 
 	size of the output cache (maximum number of entries to hold in
 	memory). The default is 1000, you can change it permanently in your
 	config file. Setting it to 0 completely disables the caching system,
 	and the minimum value accepted is 20 (if you provide a value less than
 	20, it is reset to 0 and a warning is issued) This limit is defined
 	because otherwise you'll spend more time re-flushing a too small cache
 	than working.
 
     -classic, cl 
 	Gives IPython a similar feel to the classic Python
 	prompt.
 
     -colors <scheme> 
 	Color scheme for prompts and exception reporting. Currently
 	implemented: NoColor, Linux and LightBG.
 
     -[no]color_info 
 	IPython can display information about objects via a set of functions,
 	and optionally can use colors for this, syntax highlighting source
 	code and various other elements.  However, because this information is
 	passed through a pager (like 'less') and many pagers get confused with
 	color codes, this option is off by default. You can test it and turn
 	it on permanently in your ipythonrc file if it works for you. As a
 	reference, the 'less' pager supplied with Mandrake 8.2 works ok, but
 	that in RedHat 7.2 doesn't.
 
 	Test it and turn it on permanently if it works with your
 	system. The magic function %color_info allows you to toggle this
 	interactively for testing.
 
     -[no]debug 
 	Show information about the loading process. Very useful to pin down
 	problems with your configuration files or to get details about
 	session restores.
 
     -[no]deep_reload: 
 	IPython can use the deep_reload module which reloads changes in
 	modules recursively (it replaces the reload() function, so you don't
 	need to change anything to use it).  deep_reload() forces a full
 	reload of modules whose code may have changed, which the default
 	reload() function does not.
 
 	When deep_reload is off, IPython will use the normal reload(),
 	but deep_reload will still be available as dreload(). This
 	feature is off by default [which means that you have both
 	normal reload() and dreload()].
 
     -editor <name>
 	Which editor to use with the %edit command. By default,
 	IPython will honor your EDITOR environment variable (if not
 	set, vi is the Unix default and notepad the Windows one).
 	Since this editor is invoked on the fly by IPython and is
 	meant for editing small code snippets, you may want to use a
 	small, lightweight editor here (in case your default EDITOR is
 	something like Emacs).
 
     -ipythondir <name> 
 	name of your IPython configuration directory IPYTHONDIR. This
 	can also be specified through the environment variable
 	IPYTHONDIR.
 
     -log, l 
 	generate a log file of all input. The file is named
 	ipython_log.py in your current directory (which prevents logs
 	from multiple IPython sessions from trampling each other). You
 	can use this to later restore a session by loading your
 	logfile as a file to be executed with option -logplay (see
 	below).
 
     -logfile, lf <name>   specify the name of your logfile.
 
     -logplay, lp <name> 
 
       you can replay a previous log. For restoring a session as close as
       possible to the state you left it in, use this option (don't just run
       the logfile). With -logplay, IPython will try to reconstruct the
       previous working environment in full, not just execute the commands in
       the logfile.
 
       When a session is restored, logging is automatically turned on
       again with the name of the logfile it was invoked with (it is
       read from the log header). So once you've turned logging on for
       a session, you can quit IPython and reload it as many times as
       you want and it will continue to log its history and restore
       from the beginning every time.
       
       Caveats: there are limitations in this option. The history
       variables _i*,_* and _dh don't get restored properly. In the
       future we will try to implement full session saving by writing
       and retrieving a 'snapshot' of the memory state of IPython. But
       our first attempts failed because of inherent limitations of
       Python's Pickle module, so this may have to wait.
 
     -[no]messages   
 	Print messages which IPython collects about its startup
 	process (default on).
 
     -[no]pdb
 	Automatically call the pdb debugger after every uncaught
 	exception. If you are used to debugging using pdb, this puts
 	you automatically inside of it after any call (either in
 	IPython or in code called by it) which triggers an exception
 	which goes uncaught.
 
     -pydb
 	Makes IPython use the third party "pydb" package as debugger,
 	instead of pdb. Requires that pydb is installed.
 	
     -[no]pprint
 	ipython can optionally use the pprint (pretty printer) module
 	for displaying results. pprint tends to give a nicer display
 	of nested data structures. If you like it, you can turn it on
 	permanently in your config file (default off).
 
     -profile, p <name>
 
 	assume that your config file is ipythonrc-<name> or
 	ipy_profile_<name>.py (looks in current dir first, then in
 	IPYTHONDIR).  This is a quick way to keep and load multiple
 	config files for different tasks, especially if you use the
 	include option of config files. You can keep a basic
 	IPYTHONDIR/ipythonrc file and then have other 'profiles' which
 	include this one and load extra things for particular
 	tasks. For example:
 
 	1. $HOME/.ipython/ipythonrc : load basic things you always want.
 	2. $HOME/.ipython/ipythonrc-math : load (1) and basic math-related modules.
 	3. $HOME/.ipython/ipythonrc-numeric : load (1) and Numeric and plotting modules.
 
 	Since it is possible to create an endless loop by having
 	circular file inclusions, IPython will stop if it reaches 15
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
 
         Specify the string used for input prompts. Note that if you are using
 	numbered prompts, the number is represented with a '\#' in the
 	string. Don't forget to quote strings with spaces embedded in
 	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
 	discusses in detail all the available escapes to customize your
 	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
 	prompts. The special sequence '\D' is similar to '\#', but
 	with all digits replaced dots (so you can have your
 	continuation prompt aligned with your input prompt).  Default:
 	' .\D.:' (note three spaces at the start for alignment with
 	'In [\#]').
 
     -prompt_out,po <string> 
 	String used for output prompts, also uses numbers like
 	prompt_in1. Default: 'Out[\#]:'
 
     -quick   start in bare bones mode (no config file loaded).
 
     -rcfile <name>   
 	name of your IPython resource configuration file. Normally
 	IPython loads ipythonrc (from current directory) or
 	IPYTHONDIR/ipythonrc.
 
 	If the loading of your config file fails, IPython starts with
 	a bare bones configuration (no modules loaded at all).
 
     -[no]readline
 	use the readline library, which is needed to support name
 	completion and command history, among other things.  It is
 	enabled by default, but may cause problems for users of
 	X/Emacs in Python comint or shell buffers.
 
 	Note that X/Emacs 'eterm' buffers (opened with M-x term) support
 	IPython's readline and syntax coloring fine, only 'emacs' (M-x
 	shell and C-c !) buffers do not.
 
     -screen_length, sl <n>
 	number of lines of your screen. This is used to control
 	printing of very long strings. Strings longer than this number
 	of lines will be sent through a pager instead of directly
 	printed.
 
 	The default value for this is 0, which means IPython will
 	auto-detect your screen size every time it needs to print certain
 	potentially long strings (this doesn't change the behavior of the
 	'print' keyword, it's only triggered internally). If for some
 	reason this isn't working well (it needs curses support), specify
 	it yourself. Otherwise don't change the default.
 
     -separate_in, si <string>
 
 	separator before input prompts.
 	Default: '\n'
 
     -separate_out, so <string>  
         separator before output prompts.
         Default: nothing.
 
     -separate_out2, so2
 	separator after output prompts.
 	Default: nothing.
 	For these three options, use the value 0 to specify no separator.
 
     -nosep   
 	shorthand for '-SeparateIn 0 -SeparateOut 0 -SeparateOut2
 	0'. Simply removes all input/output separators.
 
     -upgrade 
 	allows you to upgrade your IPYTHONDIR configuration when you
 	install a new version of IPython. Since new versions may
 	include new command line options or example files, this copies
 	updated ipythonrc-type files. However, it backs up (with a
 	.old extension) all files which it overwrites so that you can
 	merge back any customizations you might have in your personal
 	files. Note that you should probably use %upgrade instead,
 	it's a safer alternative.
 
 
     -Version    print version information and exit.
 
     -wxversion <string> 
 	Select a specific version of wxPython (used in conjunction
 	with -wthread). Requires the wxversion module, part of recent
 	wxPython distributions
 
     -xmode <modename>
 
 	Mode for exception reporting.
 
 	Valid modes: Plain, Context and Verbose.
 
 	* Plain: similar to python's normal traceback printing.
 	* Context: prints 5 lines of context source code around each
 	  line in the traceback.
 	* Verbose: similar to Context, but additionally prints the
 	  variables currently visible where the exception happened
 	  (shortening their strings if too long). This can potentially be
 	  very slow, if you happen to have a huge data structure whose
 	  string representation is complex to compute. Your computer may
 	  appear to freeze for a while with cpu usage at 100%. If this
 	  occurs, you can cancel the traceback with Ctrl-C (maybe hitting it
 	  more than once).
 
 Interactive use
 ===============
 
 Warning: IPython relies on the existence of a global variable called
 _ip which controls the shell itself. If you redefine _ip to anything,
 bizarre behavior will quickly occur.
 
 Other than the above warning, IPython is meant to work as a drop-in
 replacement for the standard interactive interpreter. As such, any code
 which is valid python should execute normally under IPython (cases where
 this is not true should be reported as bugs). It does, however, offer
 many features which are not available at a standard python prompt. What
 follows is a list of these.
 
 
 Caution for Windows users
 -------------------------
 
 Windows, unfortunately, uses the '\' character as a path
 separator. This is a terrible choice, because '\' also represents the
 escape character in most modern programming languages, including
 Python. For this reason, using '/' character is recommended if you
 have problems with ``\``.  However, in Windows commands '/' flags
 options, so you can not use it for the root directory. This means that
 paths beginning at the root must be typed in a contrived manner like:
 ``%copy \opt/foo/bar.txt \tmp``
 
 .. _magic:
 
 Magic command system
 --------------------
 
 IPython will treat any line whose first character is a % as a special
 call to a 'magic' function. These allow you to control the behavior of
 IPython itself, plus a lot of system-type features. They are all
 prefixed with a % character, but parameters are given without
 parentheses or quotes.
 
 Example: typing '%cd mydir' (without the quotes) changes you working
 directory to 'mydir', if it exists.
 
 If you have 'automagic' enabled (in your ipythonrc file, via the command
 line option -automagic or with the %automagic function), you don't need
 to type in the % explicitly. IPython will scan its internal list of
 magic functions and call one if it exists. With automagic on you can
 then just type 'cd mydir' to go to directory 'mydir'. The automagic
 system has the lowest possible precedence in name searches, so defining
 an identifier with the same name as an existing magic function will
 shadow it for automagic use. You can still access the shadowed magic
 function by explicitly using the % character at the beginning of the line.
 
 An example (with automagic on) should clarify all this::
 
     In [1]: cd ipython # %cd is called by automagic
 
     /home/fperez/ipython
 
     In [2]: cd=1 # now cd is just a variable
 
     In [3]: cd .. # and doesn't work as a function anymore
 
     ------------------------------
 
         File "<console>", line 1
 
           cd ..
 
               ^
 
     SyntaxError: invalid syntax
 
     In [4]: %cd .. # but %cd always works
 
     /home/fperez
 
     In [5]: del cd # if you remove the cd variable
 
     In [6]: cd ipython # automagic can work again
 
     /home/fperez/ipython
 
 You can define your own magic functions to extend the system. The
 following example defines a new magic command, %impall::
 
     import IPython.ipapi 
 
     ip = IPython.ipapi.get()
 
     def doimp(self, arg):
 
         ip = self.api
 
         ip.ex("import %s; reload(%s); from %s import *" % (
 
         arg,arg,arg)
 
         )
 
     ip.expose_magic('impall', doimp)
 
 You can also define your own aliased names for magic functions. In your
-ipythonrc file, placing a line like:
+ipythonrc file, placing a line like::
 
-execute __IP.magic_cl = __IP.magic_clear
+    execute __IP.magic_cl = __IP.magic_clear
 
 will define %cl as a new name for %clear.
 
 Type %magic for more information, including a list of all available
 magic functions at any time and their docstrings. You can also type
 %magic_function_name? (see sec. 6.4 <#sec:dyn-object-info> for
 information on the '?' system) to get information about any particular
 magic function you are interested in.
 
+The API documentation for the :mod:`IPython.Magic` module contains the full
+docstrings of all currently available magic commands.
 
-Magic commands
---------------
-
-The rest of this section is automatically generated for each release
-from the docstrings in the IPython code. Therefore the formatting is
-somewhat minimal, but this method has the advantage of having
-information always in sync with the code.
-
-A list of all the magic commands available in IPython's default
-installation follows. This is similar to what you'll see by simply
-typing %magic at the prompt, but that will also give you information
-about magic commands you may have added as part of your personal
-customizations.
-
-.. magic_start
-
-**%Exit**::
-
-	Exit IPython without confirmation.
-
-**%Pprint**::
-
-	Toggle pretty printing on/off.
-
-**%alias**::
-
-	Define an alias for a system command.
-
-        '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
-
-        Then, typing 'alias_name params' will execute the system command 'cmd
-        params' (from your underlying operating system).
-
-        Aliases have lower precedence than magic functions and Python normal
-        variables, so if 'foo' is both a Python variable and an alias, the
-        alias can not be executed until 'del foo' removes the Python variable.
-
-        You can use the %l specifier in an alias definition to represent the
-        whole line when the alias is called.  For example:
-
-          In [2]: alias all echo "Input in brackets: <%l>"\
-          In [3]: all hello world\
-          Input in brackets: <hello world>
-
-        You can also define aliases with parameters using %s specifiers (one
-        per parameter):
-        
-          In [1]: alias parts echo first %s second %s\
-          In [2]: %parts A B\
-          first A second B\
-          In [3]: %parts A\
-          Incorrect number of arguments: 2 expected.\
-          parts is an alias to: 'echo first %s second %s'
-
-        Note that %l and %s are mutually exclusive.  You can only use one or
-        the other in your aliases.
-
-        Aliases expand Python variables just like system calls using ! or !! 
-        do: all expressions prefixed with '$' get expanded.  For details of
-        the semantic rules, see PEP-215:
-        http://www.python.org/peps/pep-0215.html.  This is the library used by
-        IPython for variable expansion.  If you want to access a true shell
-        variable, an extra $ is necessary to prevent its expansion by IPython:
-
-        In [6]: alias show echo\
-        In [7]: PATH='A Python string'\
-        In [8]: show $PATH\
-        A Python string\
-        In [9]: show $$PATH\
-        /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
-
-        You can use the alias facility to acess all of $PATH.  See the %rehash
-        and %rehashx functions, which automatically create aliases for the
-        contents of your $PATH.
-
-        If called with no parameters, %alias prints the current alias table.
-
-**%autocall**::
-
-	Make functions callable without having to type parentheses.
-
-        Usage:
-
-           %autocall [mode]
-
-        The mode can be one of: 0->Off, 1->Smart, 2->Full.  If not given, the
-        value is toggled on and off (remembering the previous state).
-
-        In more detail, these values mean:
-
-        0 -> fully disabled
-
-        1 -> active, but do not apply if there are no arguments on the line.
-
-        In this mode, you get:
-
-        In [1]: callable
-        Out[1]: <built-in function callable>
-
-        In [2]: callable 'hello'
-        ------> callable('hello')
-        Out[2]: False
-
-        2 -> Active always.  Even if no arguments are present, the callable
-        object is called:
-
-        In [4]: callable
-        ------> callable()
-
-        Note that even with autocall off, you can still use '/' at the start of
-        a line to treat the first argument on the command line as a function
-        and add parentheses to it:
-
-        In [8]: /str 43
-        ------> str(43)
-        Out[8]: '43'
-
-**%autoindent**::
-
-	Toggle autoindent on/off (if available).
-
-**%automagic**::
-
-	Make magic functions callable without having to type the initial %.
-
-        Without argumentsl toggles on/off (when off, you must call it as
-        %automagic, of course).  With arguments it sets the value, and you can
-        use any of (case insensitive):
-
-         - on,1,True: to activate
-
-         - off,0,False: to deactivate.
-
-        Note that magic functions have lowest priority, so if there's a
-        variable whose name collides with that of a magic fn, automagic won't
-        work for that function (you get the variable instead). However, if you
-        delete the variable (del var), the previously shadowed magic function
-        becomes visible to automagic again.
-
-**%bg**::
-
-	Run a job in the background, in a separate thread.
-
-        For example,
-
-          %bg myfunc(x,y,z=1)
-
-        will execute 'myfunc(x,y,z=1)' in a background thread.  As soon as the
-        execution starts, a message will be printed indicating the job
-        number.  If your job number is 5, you can use
-
-          myvar = jobs.result(5)  or  myvar = jobs[5].result
-
-        to assign this result to variable 'myvar'.
-
-        IPython has a job manager, accessible via the 'jobs' object.  You can
-        type jobs? to get more information about it, and use jobs.<TAB> to see
-        its attributes.  All attributes not starting with an underscore are
-        meant for public use.
-
-        In particular, look at the jobs.new() method, which is used to create
-        new jobs.  This magic %bg function is just a convenience wrapper
-        around jobs.new(), for expression-based jobs.  If you want to create a
-        new job with an explicit function object and arguments, you must call
-        jobs.new() directly.
-
-        The jobs.new docstring also describes in detail several important
-        caveats associated with a thread-based model for background job
-        execution.  Type jobs.new? for details.
-
-        You can check the status of all jobs with jobs.status().
-
-        The jobs variable is set by IPython into the Python builtin namespace.
-        If you ever declare a variable named 'jobs', you will shadow this
-        name.  You can either delete your global jobs variable to regain
-        access to the job manager, or make a new name and assign it manually
-        to the manager (stored in IPython's namespace).  For example, to
-        assign the job manager to the Jobs name, use:
-
-          Jobs = __builtins__.jobs
-
-**%bookmark**::
-
-	Manage IPython's bookmark system.
-
-        %bookmark <name>       - set bookmark to current dir
-        %bookmark <name> <dir> - set bookmark to <dir>
-        %bookmark -l           - list all bookmarks
-        %bookmark -d <name>    - remove bookmark
-        %bookmark -r           - remove all bookmarks
-
-        You can later on access a bookmarked folder with:
-          %cd -b <name>
-        or simply '%cd <name>' if there is no directory called <name> AND
-        there is such a bookmark defined.
-
-        Your bookmarks persist through IPython sessions, but they are
-        associated with each profile.
-
-**%cd**::
-
-	Change the current working directory.
-
-        This command automatically maintains an internal list of directories
-        you visit during your IPython session, in the variable _dh. The
-        command %dhist shows this history nicely formatted. You can also
-        do 'cd -<tab>' to see directory history conveniently.
-
-        Usage:
-
-          cd 'dir': changes to directory 'dir'.
-
-          cd -: changes to the last visited directory.
-
-          cd -<n>: changes to the n-th directory in the directory history.
-
-          cd -b <bookmark_name>: jump to a bookmark set by %bookmark
-             (note: cd <bookmark_name> is enough if there is no
-              directory <bookmark_name>, but a bookmark with the name exists.)
-              'cd -b <tab>' allows you to tab-complete bookmark names. 
-
-        Options:
-
-        -q: quiet.  Do not print the working directory after the cd command is
-        executed.  By default IPython's cd command does print this directory,
-        since the default prompts do not display path information.
-        
-        Note that !cd doesn't work for this purpose because the shell where
-        !command runs is immediately discarded after executing 'command'.
-
-**%clear**::
-
-	 Clear various data (e.g. stored history data)
-    
-    %clear out - clear output history
-    %clear in  - clear input history
-    %clear shadow_compress - Compresses shadow history (to speed up ipython)
-    %clear shadow_nuke - permanently erase all entries in shadow history
-    %clear dhist - clear dir history
-
-**%color_info**::
-
-	Toggle color_info.
-
-        The color_info configuration parameter controls whether colors are
-        used for displaying object details (by things like %psource, %pfile or
-        the '?' system). This function toggles this value with each call.
-
-        Note that unless you have a fairly recent pager (less works better
-        than more) in your system, using colored object information displays
-        will not work properly. Test it and see.
-
-**%colors**::
-
-	Switch color scheme for prompts, info system and exception handlers.
-
-        Currently implemented schemes: NoColor, Linux, LightBG.
-
-        Color scheme names are not case-sensitive.
-
-**%cpaste**::
-
-	Allows you to paste & execute a pre-formatted code block from clipboard
-        
-        You must terminate the block with '--' (two minus-signs) alone on the
-        line. You can also provide your own sentinel with '%paste -s %%' ('%%' 
-        is the new sentinel for this operation)
-        
-        The block is dedented prior to execution to enable execution of method
-        definitions. '>' and '+' characters at the beginning of a line are
-        ignored, to allow pasting directly from e-mails or diff files. The
-        executed block is also assigned to variable named 'pasted_block' for
-        later editing with '%edit pasted_block'.
-        
-        You can also pass a variable name as an argument, e.g. '%cpaste foo'.
-        This assigns the pasted block to variable 'foo' as string, without 
-        dedenting or executing it.
-        
-        Do not be alarmed by garbled output on Windows (it's a readline bug). 
-        Just press enter and type -- (and press enter again) and the block 
-        will be what was just pasted.
-        
-        IPython statements (magics, shell escapes) are not supported (yet).
-
-**%debug**::
-
-	Activate the interactive debugger in post-mortem mode.
-
-        If an exception has just occurred, this lets you inspect its stack
-        frames interactively.  Note that this will always work only on the last
-        traceback that occurred, so you must call this quickly after an
-        exception that you wish to inspect has fired, because if another one
-        occurs, it clobbers the previous one.
-
-        If you want IPython to automatically do this on every exception, see
-        the %pdb magic for more details.
-
-**%dhist**::
-
-	Print your history of visited directories.
-
-        %dhist       -> print full history\
-        %dhist n     -> print last n entries only\
-        %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\
-
-        This history is automatically maintained by the %cd command, and
-        always available as the global list variable _dh. You can use %cd -<n>
-        to go to directory number <n>.
-        
-        Note that most of time, you should view directory history by entering
-        cd -<TAB>.
-
-**%dirs**::
-
-	Return the current directory stack.
-
-**%doctest_mode**::
-
-	Toggle doctest mode on and off.
-
-        This mode allows you to toggle the prompt behavior between normal
-        IPython prompts and ones that are as similar to the default IPython
-        interpreter as possible.
-
-        It also supports the pasting of code snippets that have leading '>>>'
-        and '...' prompts in them.  This means that you can paste doctests from
-        files or docstrings (even if they have leading whitespace), and the
-        code will execute correctly.  You can then use '%history -tn' to see
-        the translated history without line numbers; this will give you the
-        input after removal of all the leading prompts and whitespace, which
-        can be pasted back into an editor.
-
-        With these features, you can switch into this mode easily whenever you
-        need to do testing and changes to doctests, without having to leave
-        your existing IPython session.
-
-**%ed**::
-
-	Alias to %edit.
-
-**%edit**::
-
-	Bring up an editor and execute the resulting code.
-
-        Usage:
-          %edit [options] [args]
-
-        %edit runs IPython's editor hook.  The default version of this hook is
-        set to call the __IPYTHON__.rc.editor command.  This is read from your
-        environment variable $EDITOR.  If this isn't found, it will default to
-        vi under Linux/Unix and to notepad under Windows.  See the end of this
-        docstring for how to change the editor hook.
-
-        You can also set the value of this editor via the command line option
-        '-editor' or in your ipythonrc file. This is useful if you wish to use
-        specifically for IPython an editor different from your typical default
-        (and for Windows users who typically don't set environment variables).
-
-        This command allows you to conveniently edit multi-line code right in
-        your IPython session.
-        
-        If called without arguments, %edit opens up an empty editor with a
-        temporary file and will execute the contents of this file when you
-        close it (don't forget to save it!).
-
-
-        Options:
-
-        -n <number>: open the editor at a specified line number.  By default,
-        the IPython editor hook uses the unix syntax 'editor +N filename', but
-        you can configure this by providing your own modified hook if your
-        favorite editor supports line-number specifications with a different
-        syntax.
-        
-        -p: this will call the editor with the same data as the previous time
-        it was used, regardless of how long ago (in your current session) it
-        was.
-
-        -r: use 'raw' input.  This option only applies to input taken from the
-        user's history.  By default, the 'processed' history is used, so that
-        magics are loaded in their transformed version to valid Python.  If
-        this option is given, the raw input as typed as the command line is
-        used instead.  When you exit the editor, it will be executed by
-        IPython's own processor.
-        
-        -x: do not execute the edited code immediately upon exit. This is
-        mainly useful if you are editing programs which need to be called with
-        command line arguments, which you can then do using %run.
-
-
-        Arguments:
-
-        If arguments are given, the following possibilites exist:
-
-        - The arguments are numbers or pairs of colon-separated numbers (like
-        1 4:8 9). These are interpreted as lines of previous input to be
-        loaded into the editor. The syntax is the same of the %macro command.
-
-        - If the argument doesn't start with a number, it is evaluated as a
-        variable and its contents loaded into the editor. You can thus edit
-        any string which contains python code (including the result of
-        previous edits).
-
-        - If the argument is the name of an object (other than a string),
-        IPython will try to locate the file where it was defined and open the
-        editor at the point where it is defined. You can use `%edit function`
-        to load an editor exactly at the point where 'function' is defined,
-        edit it and have the file be executed automatically.
-
-        If the object is a macro (see %macro for details), this opens up your
-        specified editor with a temporary file containing the macro's data.
-        Upon exit, the macro is reloaded with the contents of the file.
-
-        Note: opening at an exact line is only supported under Unix, and some
-        editors (like kedit and gedit up to Gnome 2.8) do not understand the
-        '+NUMBER' parameter necessary for this feature. Good editors like
-        (X)Emacs, vi, jed, pico and joe all do.
-
-        - If the argument is not found as a variable, IPython will look for a
-        file with that name (adding .py if necessary) and load it into the
-        editor. It will execute its contents with execfile() when you exit,
-        loading any code in the file into your interactive namespace.
-
-        After executing your code, %edit will return as output the code you
-        typed in the editor (except when it was an existing file). This way
-        you can reload the code in further invocations of %edit as a variable,
-        via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
-        the output.
-
-        Note that %edit is also available through the alias %ed.
-
-        This is an example of creating a simple function inside the editor and
-        then modifying it. First, start up the editor:
-
-        In [1]: ed\
-        Editing... done. Executing edited code...\
-        Out[1]: 'def foo():\n    print "foo() was defined in an editing session"\n'
-
-        We can then call the function foo():
-        
-        In [2]: foo()\
-        foo() was defined in an editing session
-
-        Now we edit foo.  IPython automatically loads the editor with the
-        (temporary) file where foo() was previously defined:
-        
-        In [3]: ed foo\
-        Editing... done. Executing edited code...
-
-        And if we call foo() again we get the modified version:
-        
-        In [4]: foo()\
-        foo() has now been changed!
-
-        Here is an example of how to edit a code snippet successive
-        times. First we call the editor:
-
-        In [8]: ed\
-        Editing... done. Executing edited code...\
-        hello\
-        Out[8]: "print 'hello'\n"
-
-        Now we call it again with the previous output (stored in _):
-
-        In [9]: ed _\
-        Editing... done. Executing edited code...\
-        hello world\
-        Out[9]: "print 'hello world'\n"
-
-        Now we call it with the output #8 (stored in _8, also as Out[8]):
-
-        In [10]: ed _8\
-        Editing... done. Executing edited code...\
-        hello again\
-        Out[10]: "print 'hello again'\n"
-
-
-        Changing the default editor hook:
-
-        If you wish to write your own editor hook, you can put it in a
-        configuration file which you load at startup time.  The default hook
-        is defined in the IPython.hooks module, and you can use that as a
-        starting example for further modifications.  That file also has
-        general instructions on how to set a new hook for use once you've
-        defined it.
-
-**%env**::
-
-	List environment variables.
-
-**%exit**::
-
-	Exit IPython, confirming if configured to do so.
-
-        You can configure whether IPython asks for confirmation upon exit by
-        setting the confirm_exit flag in the ipythonrc file.
-
-**%hist**::
-
-	Alternate name for %history.
-
-**%history**::
-
-	Print input history (_i<n> variables), with most recent last.
-    
-    %history       -> print at most 40 inputs (some may be multi-line)\
-    %history n     -> print at most n inputs\
-    %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\
-    
-    Each input's number <n> is shown, and is accessible as the
-    automatically generated variable _i<n>.  Multi-line statements are
-    printed starting at a new line for easy copy/paste.
-    
-
-    Options:
-
-      -n: do NOT print line numbers. This is useful if you want to get a
-      printout of many lines which can be directly pasted into a text
-      editor.
-
-      This feature is only available if numbered prompts are in use.
-
-      -t: (default) print the 'translated' history, as IPython understands it.
-      IPython filters your input and converts it all into valid Python source
-      before executing it (things like magics or aliases are turned into
-      function calls, for example). With this option, you'll see the native
-      history instead of the user-entered version: '%cd /' will be seen as
-      '_ip.magic("%cd /")' instead of '%cd /'.
-      
-      -r: print the 'raw' history, i.e. the actual commands you typed.
-      
-      -g: treat the arg as a pattern to grep for in (full) history.
-      This includes the "shadow history" (almost all commands ever written).
-      Use '%hist -g' to show full shadow history (may be very long).
-      In shadow history, every index nuwber starts with 0.
-
-      -f FILENAME: instead of printing the output to the screen, redirect it to
-       the given file.  The file is always overwritten, though IPython asks for
-       confirmation first if it already exists.
-
-**%logoff**::
-
-	Temporarily stop logging.
-
-        You must have previously started logging.
-
-**%logon**::
-
-	Restart logging.
-
-        This function is for restarting logging which you've temporarily
-        stopped with %logoff. For starting logging for the first time, you
-        must use the %logstart function, which allows you to specify an
-        optional log filename.
-
-**%logstart**::
-
-	Start logging anywhere in a session.
-
-        %logstart [-o|-r|-t] [log_name [log_mode]]
-
-        If no name is given, it defaults to a file named 'ipython_log.py' in your
-        current directory, in 'rotate' mode (see below).
-
-        '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
-        history up to that point and then continues logging.
-
-        %logstart takes a second optional parameter: logging mode. This can be one
-        of (note that the modes are given unquoted):\
-          append: well, that says it.\
-          backup: rename (if exists) to name~ and start name.\
-          global: single logfile in your home dir, appended to.\
-          over  : overwrite existing log.\
-          rotate: create rotating logs name.1~, name.2~, etc.
-
-        Options:
-
-          -o: log also IPython's output.  In this mode, all commands which
-          generate an Out[NN] prompt are recorded to the logfile, right after
-          their corresponding input line.  The output lines are always
-          prepended with a '#[Out]# ' marker, so that the log remains valid
-          Python code.
-
-          Since this marker is always the same, filtering only the output from
-          a log is very easy, using for example a simple awk call:
-
-            awk -F'#\[Out\]# ' '{if($2) {print $2}}' ipython_log.py
-
-          -r: log 'raw' input.  Normally, IPython's logs contain the processed
-          input, so that user lines are logged in their final form, converted
-          into valid Python.  For example, %Exit is logged as
-          '_ip.magic("Exit").  If the -r flag is given, all input is logged
-          exactly as typed, with no transformations applied.
-
-          -t: put timestamps before each input line logged (these are put in
-          comments).
-
-**%logstate**::
-
-	Print the status of the logging system.
-
-**%logstop**::
-
-	Fully stop logging and close log file.
-
-        In order to start logging again, a new %logstart call needs to be made,
-        possibly (though not necessarily) with a new filename, mode and other
-        options.
-
-**%lsmagic**::
-
-	List currently available magic functions.
-
-**%macro**::
-
-	Define a set of input lines as a macro for future re-execution.
-
-        Usage:\
-          %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
-
-        Options:
-        
-          -r: use 'raw' input.  By default, the 'processed' history is used,
-          so that magics are loaded in their transformed version to valid
-          Python.  If this option is given, the raw input as typed as the
-          command line is used instead.
-
-        This will define a global variable called `name` which is a string
-        made of joining the slices and lines you specify (n1,n2,... numbers
-        above) from your input history into a single string. This variable
-        acts like an automatic function which re-executes those lines as if
-        you had typed them. You just type 'name' at the prompt and the code
-        executes.
-
-        The notation for indicating number ranges is: n1-n2 means 'use line
-        numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
-        using the lines numbered 5,6 and 7.
-
-        Note: as a 'hidden' feature, you can also use traditional python slice
-        notation, where N:M means numbers N through M-1.
-
-        For example, if your history contains (%hist prints it):
-        
-          44: x=1\
-          45: y=3\
-          46: z=x+y\
-          47: print x\
-          48: a=5\
-          49: print 'x',x,'y',y\
-
-        you can create a macro with lines 44 through 47 (included) and line 49
-        called my_macro with:
-
-          In [51]: %macro my_macro 44-47 49
-
-        Now, typing `my_macro` (without quotes) will re-execute all this code
-        in one pass.
-
-        You don't need to give the line-numbers in order, and any given line
-        number can appear multiple times. You can assemble macros with any
-        lines from your input history in any order.
-
-        The macro is a simple object which holds its value in an attribute,
-        but IPython's display system checks for macros and executes them as
-        code instead of printing them when you type their name.
-
-        You can view a macro's contents by explicitly printing it with:
-        
-          'print macro_name'.
-
-        For one-off cases which DON'T contain magic function calls in them you
-        can obtain similar results by explicitly executing slices from your
-        input history with:
-
-          In [60]: exec In[44:48]+In[49]
-
-**%magic**::
-
-	Print information about the magic function system.
-
-**%mglob**::
-
-	    This program allows specifying filenames with "mglob" mechanism.
-    Supported syntax in globs (wilcard matching patterns)::
-    
-     *.cpp ?ellowo*                
-         - obvious. Differs from normal glob in that dirs are not included.
-           Unix users might want to write this as: "*.cpp" "?ellowo*"
-     rec:/usr/share=*.txt,*.doc    
-         - get all *.txt and *.doc under /usr/share, 
-           recursively
-     rec:/usr/share
-         - All files under /usr/share, recursively
-     rec:*.py
-         - All .py files under current working dir, recursively
-     foo                           
-         - File or dir foo
-     !*.bak readme*                   
-         - readme*, exclude files ending with .bak
-     !.svn/ !.hg/ !*_Data/ rec:.
-         - Skip .svn, .hg, foo_Data dirs (and their subdirs) in recurse.
-           Trailing / is the key, \ does not work!
-     dir:foo                       
-         - the directory foo if it exists (not files in foo)
-     dir:*                         
-         - all directories in current folder
-     foo.py bar.* !h* rec:*.py
-         - Obvious. !h* exclusion only applies for rec:*.py.
-           foo.py is *not* included twice.
-     @filelist.txt
-         - All files listed in 'filelist.txt' file, on separate lines.
-
-**%page**::
-
-	Pretty print the object and display it through a pager.
-        
-        %page [options] OBJECT
-
-        If no object is given, use _ (last output).
-        
-        Options:
-
-          -r: page str(object), don't pretty-print it.
-
-**%pdb**::
-
-	Control the automatic calling of the pdb interactive debugger.
-
-        Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
-        argument it works as a toggle.
-
-        When an exception is triggered, IPython can optionally call the
-        interactive pdb debugger after the traceback printout. %pdb toggles
-        this feature on and off.
-
-        The initial state of this feature is set in your ipythonrc
-        configuration file (the variable is called 'pdb').
-
-        If you want to just activate the debugger AFTER an exception has fired,
-        without having to type '%pdb on' and rerunning your code, you can use
-        the %debug magic.
-
-**%pdef**::
-
-	Print the definition header for any callable object.
-
-        If the object is a class, print the constructor information.
-
-**%pdoc**::
-
-	Print the docstring for an object.
-
-        If the given object is a class, it will print both the class and the
-        constructor docstrings.
-
-**%pfile**::
-
-	Print (or run through pager) the file where an object is defined.
-
-        The file opens at the line where the object definition begins. IPython
-        will honor the environment variable PAGER if set, and otherwise will
-        do its best to print the file in a convenient form.
-
-        If the given argument is not an object currently defined, IPython will
-        try to interpret it as a filename (automatically adding a .py extension
-        if needed). You can thus use %pfile as a syntax highlighting code
-        viewer.
-
-**%pinfo**::
-
-	Provide detailed information about an object.
-    
-        '%pinfo object' is just a synonym for object? or ?object.
-
-**%popd**::
-
-	Change to directory popped off the top of the stack.
-
-**%profile**::
-
-	Print your currently active IPyhton profile.
-
-**%prun**::
-
-	Run a statement through the python code profiler.
-
-        Usage:\
-          %prun [options] statement
-
-        The given statement (which doesn't require quote marks) is run via the
-        python profiler in a manner similar to the profile.run() function.
-        Namespaces are internally managed to work correctly; profile.run
-        cannot be used in IPython because it makes certain assumptions about
-        namespaces which do not hold under IPython.
-
-        Options:
-
-        -l <limit>: you can place restrictions on what or how much of the
-        profile gets printed. The limit value can be:
-
-          * A string: only information for function names containing this string
-          is printed.
-
-          * An integer: only these many lines are printed.
-
-          * A float (between 0 and 1): this fraction of the report is printed
-          (for example, use a limit of 0.4 to see the topmost 40% only).
-
-        You can combine several limits with repeated use of the option. For
-        example, '-l __init__ -l 5' will print only the topmost 5 lines of
-        information about class constructors.
-
-        -r: return the pstats.Stats object generated by the profiling. This
-        object has all the information about the profile in it, and you can
-        later use it for further analysis or in other functions.
-
-       -s <key>: sort profile by given key. You can provide more than one key
-        by using the option several times: '-s key1 -s key2 -s key3...'. The
-        default sorting key is 'time'.
-
-        The following is copied verbatim from the profile documentation
-        referenced below:
-
-        When more than one key is provided, additional keys are used as
-        secondary criteria when the there is equality in all keys selected
-        before them.
-        
-        Abbreviations can be used for any key names, as long as the
-        abbreviation is unambiguous.  The following are the keys currently
-        defined:
-
-                Valid Arg       Meaning\
-                  "calls"      call count\
-                  "cumulative" cumulative time\
-                  "file"       file name\
-                  "module"     file name\
-                  "pcalls"     primitive call count\
-                  "line"       line number\
-                  "name"       function name\
-                  "nfl"        name/file/line\
-                  "stdname"    standard name\
-                  "time"       internal time
-
-        Note that all sorts on statistics are in descending order (placing
-        most time consuming items first), where as name, file, and line number
-        searches are in ascending order (i.e., alphabetical). The subtle
-        distinction between "nfl" and "stdname" is that the standard name is a
-        sort of the name as printed, which means that the embedded line
-        numbers get compared in an odd way.  For example, lines 3, 20, and 40
-        would (if the file names were the same) appear in the string order
-        "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
-        line numbers.  In fact, sort_stats("nfl") is the same as
-        sort_stats("name", "file", "line").
-
-        -T <filename>: save profile results as shown on screen to a text
-        file. The profile is still shown on screen.
-
-        -D <filename>: save (via dump_stats) profile statistics to given
-        filename. This data is in a format understod by the pstats module, and
-        is generated by a call to the dump_stats() method of profile
-        objects. The profile is still shown on screen.
-
-        If you want to run complete programs under the profiler's control, use
-        '%run -p [prof_opts] filename.py [args to program]' where prof_opts
-        contains profiler specific options as described here.
-        
-        You can read the complete documentation for the profile module with:\
-          In [1]: import profile; profile.help()
-
-**%psearch**::
-
-	Search for object in namespaces by wildcard.
-
-        %psearch [options] PATTERN [OBJECT TYPE]
-
-        Note: ? can be used as a synonym for %psearch, at the beginning or at
-        the end: both a*? and ?a* are equivalent to '%psearch a*'.  Still, the
-        rest of the command line must be unchanged (options come first), so
-        for example the following forms are equivalent
-
-        %psearch -i a* function
-        -i a* function?
-        ?-i a* function
-
-        Arguments:
-        
-          PATTERN
-
-          where PATTERN is a string containing * as a wildcard similar to its
-          use in a shell.  The pattern is matched in all namespaces on the
-          search path. By default objects starting with a single _ are not
-          matched, many IPython generated objects have a single
-          underscore. The default is case insensitive matching. Matching is
-          also done on the attributes of objects and not only on the objects
-          in a module.
-
-          [OBJECT TYPE]
-
-          Is the name of a python type from the types module. The name is
-          given in lowercase without the ending type, ex. StringType is
-          written string. By adding a type here only objects matching the
-          given type are matched. Using all here makes the pattern match all
-          types (this is the default).
-
-        Options:
-
-          -a: makes the pattern match even objects whose names start with a
-          single underscore.  These names are normally ommitted from the
-          search.
-
-          -i/-c: make the pattern case insensitive/sensitive.  If neither of
-          these options is given, the default is read from your ipythonrc
-          file.  The option name which sets this value is
-          'wildcards_case_sensitive'.  If this option is not specified in your
-          ipythonrc file, IPython's internal default is to do a case sensitive
-          search.
-
-          -e/-s NAMESPACE: exclude/search a given namespace.  The pattern you
-          specifiy can be searched in any of the following namespaces:
-          'builtin', 'user', 'user_global','internal', 'alias', where
-          'builtin' and 'user' are the search defaults.  Note that you should
-          not use quotes when specifying namespaces.
-
-          'Builtin' contains the python module builtin, 'user' contains all
-          user data, 'alias' only contain the shell aliases and no python
-          objects, 'internal' contains objects used by IPython.  The
-          'user_global' namespace is only used by embedded IPython instances,
-          and it contains module-level globals.  You can add namespaces to the
-          search with -s or exclude them with -e (these options can be given
-          more than once).
-    
-        Examples:
-       
-        %psearch a*            -> objects beginning with an a
-        %psearch -e builtin a* -> objects NOT in the builtin space starting in a
-        %psearch a* function   -> all functions beginning with an a
-        %psearch re.e*         -> objects beginning with an e in module re
-        %psearch r*.e*         -> objects that start with e in modules starting in r
-        %psearch r*.* string   -> all strings in modules beginning with r
-
-        Case sensitve search:
-       
-        %psearch -c a*         list all object beginning with lower case a
-
-        Show objects beginning with a single _:
-       
-        %psearch -a _*         list objects beginning with a single underscore
-
-**%psource**::
-
-	Print (or run through pager) the source code for an object.
-
-**%pushd**::
-
-	Place the current dir on stack and change directory.
-        
-        Usage:\
-          %pushd ['dirname']
-
-**%pwd**::
-
-	Return the current working directory path.
-
-**%pycat**::
-
-	Show a syntax-highlighted file through a pager.
-
-        This magic is similar to the cat utility, but it will assume the file
-        to be Python source and will show it with syntax highlighting.
-
-**%quickref**::
-
-	 Show a quick reference sheet
-
-**%quit**::
-
-	Exit IPython, confirming if configured to do so (like %exit)
-
-**%r**::
-
-	Repeat previous input.
-
-        Note: Consider using the more powerfull %rep instead!
-        
-        If given an argument, repeats the previous command which starts with
-        the same string, otherwise it just repeats the previous input.
-
-        Shell escaped commands (with ! as first character) are not recognized
-        by this system, only pure python code and magic commands.
-
-**%rehashdir**::
-
-	 Add executables in all specified dirs to alias table
-     
-    Usage:
-
-    %rehashdir c:/bin;c:/tools
-      - Add all executables under c:/bin and c:/tools to alias table, in 
-      order to make them directly executable from any directory.
-        
-      Without arguments, add all executables in current directory.
-
-**%rehashx**::
-
-	Update the alias table with all executable files in $PATH.
-
-        This version explicitly checks that every entry in $PATH is a file
-        with execute access (os.X_OK), so it is much slower than %rehash.
-
-        Under Windows, it checks executability as a match agains a
-        '|'-separated string of extensions, stored in the IPython config
-        variable win_exec_ext.  This defaults to 'exe|com|bat'.
-        
-        This function also resets the root module cache of module completer,
-        used on slow filesystems.
-
-**%rep**::
-
-	 Repeat a command, or get command to input line for editing
-
-    - %rep (no arguments):
-    
-    Place a string version of last computation result (stored in the special '_'
-    variable) to the next input prompt. Allows you to create elaborate command
-    lines without using copy-paste::
-    
-        $ l = ["hei", "vaan"]       
-        $ "".join(l)        
-        ==> heivaan        
-        $ %rep        
-        $ heivaan_ <== cursor blinking    
-    
-    %rep 45
-    
-    Place history line 45 to next input prompt. Use %hist to find out the
-    number.
-    
-    %rep 1-4 6-7 3
-    
-    Repeat the specified lines immediately. Input slice syntax is the same as
-    in %macro and %save.
-    
-    %rep foo
-    
-    Place the most recent line that has the substring "foo" to next input.
-    (e.g. 'svn ci -m foobar').
-
-**%reset**::
-
-	Resets the namespace by removing all names defined by the user.
-
-        Input/Output history are left around in case you need them.
-
-**%run**::
-
-	Run the named file inside IPython as a program.
-
-        Usage:\
-          %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
-        
-        Parameters after the filename are passed as command-line arguments to
-        the program (put in sys.argv). Then, control returns to IPython's
-        prompt.
-
-        This is similar to running at a system prompt:\
-          $ python file args\
-        but with the advantage of giving you IPython's tracebacks, and of
-        loading all variables into your interactive namespace for further use
-        (unless -p is used, see below).
-
-        The file is executed in a namespace initially consisting only of
-        __name__=='__main__' and sys.argv constructed as indicated. It thus
-        sees its environment as if it were being run as a stand-alone program
-        (except for sharing global objects such as previously imported
-        modules). But after execution, the IPython interactive namespace gets
-        updated with all variables defined in the program (except for __name__
-        and sys.argv). This allows for very convenient loading of code for
-        interactive work, while giving each program a 'clean sheet' to run in.
-
-        Options:
-        
-        -n: __name__ is NOT set to '__main__', but to the running file's name
-        without extension (as python does under import).  This allows running
-        scripts and reloading the definitions in them without calling code
-        protected by an ' if __name__ == "__main__" ' clause.
-
-        -i: run the file in IPython's namespace instead of an empty one. This
-        is useful if you are experimenting with code written in a text editor
-        which depends on variables defined interactively.
-
-        -e: ignore sys.exit() calls or SystemExit exceptions in the script
-        being run.  This is particularly useful if IPython is being used to
-        run unittests, which always exit with a sys.exit() call.  In such
-        cases you are interested in the output of the test results, not in
-        seeing a traceback of the unittest module.
-
-        -t: print timing information at the end of the run.  IPython will give
-        you an estimated CPU time consumption for your script, which under
-        Unix uses the resource module to avoid the wraparound problems of
-        time.clock().  Under Unix, an estimate of time spent on system tasks
-        is also given (for Windows platforms this is reported as 0.0).
-
-        If -t is given, an additional -N<N> option can be given, where <N>
-        must be an integer indicating how many times you want the script to
-        run.  The final timing report will include total and per run results.
-
-        For example (testing the script uniq_stable.py):
-
-            In [1]: run -t uniq_stable
-
-            IPython CPU timings (estimated):\
-              User  :    0.19597 s.\
-              System:        0.0 s.\
-
-            In [2]: run -t -N5 uniq_stable
-
-            IPython CPU timings (estimated):\
-            Total runs performed: 5\
-              Times :      Total       Per run\
-              User  :   0.910862 s,  0.1821724 s.\
-              System:        0.0 s,        0.0 s.
-
-        -d: run your program under the control of pdb, the Python debugger.
-        This allows you to execute your program step by step, watch variables,
-        etc.  Internally, what IPython does is similar to calling:
-        
-          pdb.run('execfile("YOURFILENAME")')
-
-        with a breakpoint set on line 1 of your file.  You can change the line
-        number for this automatic breakpoint to be <N> by using the -bN option
-        (where N must be an integer).  For example:
-
-          %run -d -b40 myscript
-
-        will set the first breakpoint at line 40 in myscript.py.  Note that
-        the first breakpoint must be set on a line which actually does
-        something (not a comment or docstring) for it to stop execution.
-
-        When the pdb debugger starts, you will see a (Pdb) prompt.  You must
-        first enter 'c' (without qoutes) to start execution up to the first
-        breakpoint.
-
-        Entering 'help' gives information about the use of the debugger.  You
-        can easily see pdb's full documentation with "import pdb;pdb.help()"
-        at a prompt.
-
-        -p: run program under the control of the Python profiler module (which
-        prints a detailed report of execution times, function calls, etc).
-
-        You can pass other options after -p which affect the behavior of the
-        profiler itself. See the docs for %prun for details.
-
-        In this mode, the program's variables do NOT propagate back to the
-        IPython interactive namespace (because they remain in the namespace
-        where the profiler executes them).
-
-        Internally this triggers a call to %prun, see its documentation for
-        details on the options available specifically for profiling.
-
-        There is one special usage for which the text above doesn't apply:
-        if the filename ends with .ipy, the file is run as ipython script,
-        just as if the commands were written on IPython prompt.
-
-**%runlog**::
-
-	Run files as logs.
-
-        Usage:\
-          %runlog file1 file2 ...
-
-        Run the named files (treating them as log files) in sequence inside
-        the interpreter, and return to the prompt.  This is much slower than
-        %run because each line is executed in a try/except block, but it
-        allows running files with syntax errors in them.
-
-        Normally IPython will guess when a file is one of its own logfiles, so
-        you can typically use %run even for logs. This shorthand allows you to
-        force any file to be treated as a log file.
-
-**%save**::
-
-	Save a set of lines to a given filename.
-
-        Usage:\
-          %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
-
-        Options:
-        
-          -r: use 'raw' input.  By default, the 'processed' history is used,
-          so that magics are loaded in their transformed version to valid
-          Python.  If this option is given, the raw input as typed as the
-          command line is used instead.
-
-        This function uses the same syntax as %macro for line extraction, but
-        instead of creating a macro it saves the resulting string to the
-        filename you specify.
-
-        It adds a '.py' extension to the file if you don't do so yourself, and
-        it asks for confirmation before overwriting existing files.
-
-**%sc**::
-
-	Shell capture - execute a shell command and capture its output.
-
-        DEPRECATED. Suboptimal, retained for backwards compatibility.
-        
-        You should use the form 'var = !command' instead. Example:
-         
-         "%sc -l myfiles = ls ~" should now be written as
-            
-         "myfiles = !ls ~"
-         
-        myfiles.s, myfiles.l and myfiles.n still apply as documented
-        below.
-
-        --
-        %sc [options] varname=command
-
-        IPython will run the given command using commands.getoutput(), and
-        will then update the user's interactive namespace with a variable
-        called varname, containing the value of the call.  Your command can
-        contain shell wildcards, pipes, etc.
-
-        The '=' sign in the syntax is mandatory, and the variable name you
-        supply must follow Python's standard conventions for valid names.
-        
-        (A special format without variable name exists for internal use)
-
-        Options:
-
-          -l: list output.  Split the output on newlines into a list before
-          assigning it to the given variable.  By default the output is stored
-          as a single string.
-
-          -v: verbose.  Print the contents of the variable.
-
-        In most cases you should not need to split as a list, because the
-        returned value is a special type of string which can automatically
-        provide its contents either as a list (split on newlines) or as a
-        space-separated string.  These are convenient, respectively, either
-        for sequential processing or to be passed to a shell command.
-
-        For example:
-
-            # Capture into variable a
-            In [9]: sc a=ls *py
-
-            # a is a string with embedded newlines
-            In [10]: a
-            Out[10]: 'setup.py win32_manual_post_install.py'
-
-            # which can be seen as a list:
-            In [11]: a.l
-            Out[11]: ['setup.py', 'win32_manual_post_install.py']
-
-            # or as a whitespace-separated string:
-            In [12]: a.s
-            Out[12]: 'setup.py win32_manual_post_install.py'
-
-            # a.s is useful to pass as a single command line:
-            In [13]: !wc -l $a.s
-              146 setup.py
-              130 win32_manual_post_install.py
-              276 total
-
-            # while the list form is useful to loop over:
-            In [14]: for f in a.l:
-               ....:      !wc -l $f
-               ....:
-            146 setup.py
-            130 win32_manual_post_install.py
-
-        Similiarly, the lists returned by the -l option are also special, in
-        the sense that you can equally invoke the .s attribute on them to
-        automatically get a whitespace-separated string from their contents:
-
-            In [1]: sc -l b=ls *py
-
-            In [2]: b
-            Out[2]: ['setup.py', 'win32_manual_post_install.py']
-
-            In [3]: b.s
-            Out[3]: 'setup.py win32_manual_post_install.py'
-
-        In summary, both the lists and strings used for ouptut capture have
-        the following special attributes:
-
-            .l (or .list) : value as list.
-            .n (or .nlstr): value as newline-separated string.
-            .s (or .spstr): value as space-separated string.
-
-**%store**::
-
-	Lightweight persistence for python variables.
-
-    Example:
-    
-    ville@badger[~]|1> A = ['hello',10,'world']\
-    ville@badger[~]|2> %store A\
-    ville@badger[~]|3> Exit
-    
-    (IPython session is closed and started again...)
-    
-    ville@badger:~$ ipython -p pysh\
-    ville@badger[~]|1> print A
-    
-    ['hello', 10, 'world']
-    
-    Usage:
-    
-    %store          - Show list of all variables and their current values\
-    %store <var>    - Store the *current* value of the variable to disk\
-    %store -d <var> - Remove the variable and its value from storage\
-    %store -z       - Remove all variables from storage\
-    %store -r       - Refresh all variables from store (delete current vals)\
-    %store foo >a.txt  - Store value of foo to new file a.txt\
-    %store foo >>a.txt - Append value of foo to file a.txt\   
-    
-    It should be noted that if you change the value of a variable, you
-    need to %store it again if you want to persist the new value.
-    
-    Note also that the variables will need to be pickleable; most basic
-    python types can be safely %stored.
-    
-    Also aliases can be %store'd across sessions.
-
-**%sx**::
-
-	Shell execute - run a shell command and capture its output.
-
-        %sx command
-
-        IPython will run the given command using commands.getoutput(), and
-        return the result formatted as a list (split on '\n').  Since the
-        output is _returned_, it will be stored in ipython's regular output
-        cache Out[N] and in the '_N' automatic variables.
-
-        Notes:
-
-        1) If an input line begins with '!!', then %sx is automatically
-        invoked.  That is, while:
-          !ls
-        causes ipython to simply issue system('ls'), typing
-          !!ls
-        is a shorthand equivalent to:
-          %sx ls
-        
-        2) %sx differs from %sc in that %sx automatically splits into a list,
-        like '%sc -l'.  The reason for this is to make it as easy as possible
-        to process line-oriented shell output via further python commands.
-        %sc is meant to provide much finer control, but requires more
-        typing.
-
-        3) Just like %sc -l, this is a list with special attributes:
-
-          .l (or .list) : value as list.
-          .n (or .nlstr): value as newline-separated string.
-          .s (or .spstr): value as whitespace-separated string.
-
-        This is very useful when trying to use such lists as arguments to
-        system commands.
-
-**%system_verbose**::
-
-	Set verbose printing of system calls.
-
-        If called without an argument, act as a toggle
-
-**%time**::
-
-	Time execution of a Python statement or expression.
-
-        The CPU and wall clock times are printed, and the value of the
-        expression (if any) is returned.  Note that under Win32, system time
-        is always reported as 0, since it can not be measured.
-
-        This function provides very basic timing functionality.  In Python
-        2.3, the timeit module offers more control and sophistication, so this
-        could be rewritten to use it (patches welcome).
-        
-        Some examples:
-
-          In [1]: time 2**128
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00
-          Out[1]: 340282366920938463463374607431768211456L
-
-          In [2]: n = 1000000
-
-          In [3]: time sum(range(n))
-          CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
-          Wall time: 1.37
-          Out[3]: 499999500000L
-
-          In [4]: time print 'hello world'
-          hello world
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00
-
-          Note that the time needed by Python to compile the given expression
-          will be reported if it is more than 0.1s.  In this example, the
-          actual exponentiation is done by Python at compilation time, so while
-          the expression can take a noticeable amount of time to compute, that
-          time is purely due to the compilation:
-
-          In [5]: time 3**9999;
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00 s
-
-          In [6]: time 3**999999;
-          CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
-          Wall time: 0.00 s
-          Compiler : 0.78 s
-
-**%timeit**::
-
-	Time execution of a Python statement or expression
-
-        Usage:\
-          %timeit [-n<N> -r<R> [-t|-c]] statement
-
-        Time execution of a Python statement or expression using the timeit
-        module.
-
-        Options:
-        -n<N>: execute the given statement <N> times in a loop. If this value
-        is not given, a fitting value is chosen. 
-        
-        -r<R>: repeat the loop iteration <R> times and take the best result.
-        Default: 3
-        
-        -t: use time.time to measure the time, which is the default on Unix.
-        This function measures wall time.
-        
-        -c: use time.clock to measure the time, which is the default on
-        Windows and measures wall time. On Unix, resource.getrusage is used
-        instead and returns the CPU user time.
-
-        -p<P>: use a precision of <P> digits to display the timing result.
-        Default: 3
-
-        
-        Examples:\
-          In [1]: %timeit pass
-          10000000 loops, best of 3: 53.3 ns per loop
-
-          In [2]: u = None
-
-          In [3]: %timeit u is None
-          10000000 loops, best of 3: 184 ns per loop
-
-          In [4]: %timeit -r 4 u == None
-          1000000 loops, best of 4: 242 ns per loop
-
-          In [5]: import time
-
-          In [6]: %timeit -n1 time.sleep(2)
-          1 loops, best of 3: 2 s per loop
-          
-
-        The times reported by %timeit will be slightly higher than those
-        reported by the timeit.py script when variables are accessed. This is
-        due to the fact that %timeit executes the statement in the namespace
-        of the shell, compared with timeit.py, which uses a single setup
-        statement to import function or create variables. Generally, the bias
-        does not matter as long as results from timeit.py are not mixed with
-        those from %timeit.
-
-**%unalias**::
-
-	Remove an alias
-
-**%upgrade**::
-
-	 Upgrade your IPython installation
-        
-        This will copy the config files that don't yet exist in your 
-        ipython dir from the system config dir. Use this after upgrading 
-        IPython if you don't wish to delete your .ipython dir.
-
-        Call with -nolegacy to get rid of ipythonrc* files (recommended for
-        new users)
-
-**%which**::
-
-	 %which <cmd> => search PATH for files matching cmd. Also scans aliases.
-
-    Traverses PATH and prints all files (not just executables!) that match the
-    pattern on command line. Probably more useful in finding stuff
-    interactively than 'which', which only prints the first matching item.
-    
-    Also discovers and expands aliases, so you'll see what will be executed
-    when you call an alias.
-    
-    Example:
-    
-    [~]|62> %which d
-    d -> ls -F --color=auto
-      == c:\cygwin\bin\ls.exe
-    c:\cygwin\bin\d.exe
-    
-    [~]|64> %which diff*
-    diff3 -> diff3
-      == c:\cygwin\bin\diff3.exe
-    diff -> diff
-      == c:\cygwin\bin\diff.exe
-    c:\cygwin\bin\diff.exe
-    c:\cygwin\bin\diff3.exe
-
-**%who**::
-
-	Print all interactive variables, with some minimal formatting.
-
-        If any arguments are given, only variables whose type matches one of
-        these are printed.  For example:
-
-          %who function str
-
-        will only list functions and strings, excluding all other types of
-        variables.  To find the proper type names, simply use type(var) at a
-        command line to see how python prints type names.  For example:
-
-          In [1]: type('hello')\
-          Out[1]: <type 'str'>
-
-        indicates that the type name for strings is 'str'.
-
-        %who always excludes executed names loaded through your configuration
-        file and things which are internal to IPython.
-
-        This is deliberate, as typically you may load many modules and the
-        purpose of %who is to show you only what you've manually defined.
-
-**%who_ls**::
-
-	Return a sorted list of all interactive variables.
-
-        If arguments are given, only variables of types matching these
-        arguments are returned.
-
-**%whos**::
-
-	Like %who, but gives some extra information about each variable.
-
-        The same type filtering of %who can be applied here.
-
-        For all variables, the type is printed. Additionally it prints:
-        
-          - For {},[],(): their length.
-
-          - For numpy and Numeric arrays, a summary with shape, number of
-          elements, typecode and size in memory.
-
-          - Everything else: a string representation, snipping their middle if
-          too long.
-
-**%xmode**::
-
-	Switch modes for the exception handlers.
-
-        Valid modes: Plain, Context and Verbose.
-
-        If called without arguments, acts as a toggle.
-
-.. magic_end
 
 Access to the standard Python help
 ----------------------------------
 
 As of Python 2.1, a help system is available with access to object docstrings
 and the Python manuals. Simply type 'help' (no quotes) to access it. You can
 also type help(object) to obtain information about a given object, and
 help('keyword') for information on a keyword. As noted :ref:`here
 <accessing_help>`, you need to properly configure your environment variable
 PYTHONDOCS for this feature to work correctly.
 
 .. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
 
 Typing ?word or word? prints detailed information about an object. If
 certain strings in the object are too long (docstrings, code, etc.) they
 get snipped in the center for brevity. This system gives access variable
 types and values, full source code for any object (if available),
 function prototypes and other useful information.
 
 Typing ??word or word?? gives access to the full information without
 snipping long strings. Long strings are sent to the screen through the
 less pager if longer than the screen and printed otherwise. On systems
 lacking the less command, IPython uses a very basic internal pager.
 
 The following magic functions are particularly useful for gathering
 information about your working environment. You can get more details by
 typing %magic or querying them individually (use %function_name? with or
 without the %), this is just a summary:
 
     * **%pdoc <object>**: Print (or run through a pager if too long) the
       docstring for an object. If the given object is a class, it will
       print both the class and the constructor docstrings.
     * **%pdef <object>**: Print the definition header for any callable
       object. If the object is a class, print the constructor information.
     * **%psource <object>**: Print (or run through a pager if too long)
       the source code for an object.
     * **%pfile <object>**: Show the entire source file where an object was
       defined via a pager, opening it at the line where the object
       definition begins.
     * **%who/%whos**: These functions give information about identifiers
       you have defined interactively (not things you loaded or defined
       in your configuration files). %who just prints a list of
       identifiers and %whos prints a table with some basic details about
       each identifier.
 
 Note that the dynamic object information functions (?/??, %pdoc, %pfile,
 %pdef, %psource) give you access to documentation even on things which
 are not really defined as separate identifiers. Try for example typing
 {}.get? or after doing import os, type os.path.abspath??.
 
 
 .. _readline:
 
 Readline-based features
 -----------------------
 
 These features require the GNU readline library, so they won't work if
 your Python installation lacks readline support. We will first describe
 the default behavior IPython uses, and then how to change it to suit
 your preferences.
 
 
 Command line completion
 +++++++++++++++++++++++
 
 At any time, hitting TAB will complete any available python commands or
 variable names, and show you a list of the possible completions if
 there's no unambiguous one. It will also complete filenames in the
 current directory if no python names match what you've typed so far.
 
 
 Search command history
 ++++++++++++++++++++++
 
 IPython provides two ways for searching through previous input and thus
 reduce the need for repetitive typing:
 
    1. Start typing, and then use Ctrl-p (previous,up) and Ctrl-n
       (next,down) to search through only the history items that match
       what you've typed so far. If you use Ctrl-p/Ctrl-n at a blank
       prompt, they just behave like normal arrow keys.
    2. Hit Ctrl-r: opens a search prompt. Begin typing and the system
       searches your history for lines that contain what you've typed so
       far, completing as much as it can.
 
 
 Persistent command history across sessions
 ++++++++++++++++++++++++++++++++++++++++++
 
 IPython will save your input history when it leaves and reload it next
 time you restart it. By default, the history file is named
 $IPYTHONDIR/history, but if you've loaded a named profile,
 '-PROFILE_NAME' is appended to the name. This allows you to keep
 separate histories related to various tasks: commands related to
 numerical work will not be clobbered by a system shell history, for
 example.
 
 
 Autoindent
 ++++++++++
 
 IPython can recognize lines ending in ':' and indent the next line,
 while also un-indenting automatically after 'raise' or 'return'.
 
 This feature uses the readline library, so it will honor your ~/.inputrc
 configuration (or whatever file your INPUTRC variable points to). Adding
 the following lines to your .inputrc file can make indenting/unindenting
 more convenient (M-i indents, M-u unindents)::
 
     $if Python 
     "\M-i": "    "  
     "\M-u": "\d\d\d\d"  
     $endif
 
 Note that there are 4 spaces between the quote marks after "M-i" above.
 
 Warning: this feature is ON by default, but it can cause problems with
 the pasting of multi-line indented code (the pasted code gets
 re-indented on each line). A magic function %autoindent allows you to
 toggle it on/off at runtime. You can also disable it permanently on in
 your ipythonrc file (set autoindent 0).
 
 
 Customizing readline behavior
 +++++++++++++++++++++++++++++
 
 All these features are based on the GNU readline library, which has an
 extremely customizable interface. Normally, readline is configured via a
 file which defines the behavior of the library; the details of the
 syntax for this can be found in the readline documentation available
 with your system or on the Internet. IPython doesn't read this file (if
 it exists) directly, but it does support passing to readline valid
 options via a simple interface. In brief, you can customize readline by
 setting the following options in your ipythonrc configuration file (note
 that these options can not be specified at the command line):
 
     * **readline_parse_and_bind**: this option can appear as many times as
       you want, each time defining a string to be executed via a
       readline.parse_and_bind() command. The syntax for valid commands
       of this kind can be found by reading the documentation for the GNU
       readline library, as these commands are of the kind which readline
       accepts in its configuration file.
     * **readline_remove_delims**: a string of characters to be removed
       from the default word-delimiters list used by readline, so that
       completions may be performed on strings which contain them. Do not
       change the default value unless you know what you're doing.
     * **readline_omit__names**: when tab-completion is enabled, hitting
       <tab> after a '.' in a name will complete all attributes of an
       object, including all the special methods whose names include
       double underscores (like __getitem__ or __class__). If you'd
       rather not see these names by default, you can set this option to
       1. Note that even when this option is set, you can still see those
       names by explicitly typing a _ after the period and hitting <tab>:
       'name._<tab>' will always complete attribute names starting with '_'.
       
       This option is off by default so that new users see all
       attributes of any objects they are dealing with.
 
 You will find the default values along with a corresponding detailed
 explanation in your ipythonrc file.
 
 
 Session logging and restoring
 -----------------------------
 
 You can log all input from a session either by starting IPython with the
 command line switches -log or -logfile (see :ref:`here <command_line_options>`)
 or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
 restoring the state of a previous session. This feature is not quite
 perfect, but can still be useful in many cases.
 
 The log files can also be used as a way to have a permanent record of
 any code you wrote while experimenting. Log files are regular text files
 which you can later open in your favorite text editor to extract code or
 to 'clean them up' before using them to replay a session.
 
 The %logstart function for activating logging in mid-session is used as
 follows:
 
 %logstart [log_name [log_mode]]
 
 If no name is given, it defaults to a file named 'log' in your
 IPYTHONDIR directory, in 'rotate' mode (see below).
 
 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
 history up to that point and then continues logging.
 
 %logstart takes a second optional parameter: logging mode. This can be
 one of (note that the modes are given unquoted):
 
     * [over:] overwrite existing log_name.
     * [backup:] rename (if exists) to log_name~ and start log_name.
     * [append:] well, that says it.
     * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
 
 The %logoff and %logon functions allow you to temporarily stop and
 resume logging to a file which had previously been started with
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
 .. _system_shell_access:
 
 System shell access
 -------------------
 
 Any input line beginning with a ! character is passed verbatim (minus
 the !, of course) to the underlying operating system. For example,
 typing !ls will run 'ls' in the current directory.
 
 Manual capture of command output
 --------------------------------
 
 If the input line begins with two exclamation marks, !!, the command is
 executed but its output is captured and returned as a python list, split
 on newlines. Any output sent by the subprocess to standard error is
 printed separately, so that the resulting list only captures standard
 output. The !! syntax is a shorthand for the %sx magic command.
 
 Finally, the %sc magic (short for 'shell capture') is similar to %sx,
 but allowing more fine-grained control of the capture details, and
 storing the result directly into a named variable. The direct use of 
 %sc is now deprecated, and you should ise the ``var = !cmd`` syntax
 instead.
 
 IPython also allows you to expand the value of python variables when
 making system calls. Any python variable or expression which you prepend
 with $ will get expanded before the system call is made::
 
     In [1]: pyvar='Hello world' 
     In [2]: !echo "A python variable: $pyvar"  
     A python variable: Hello world
 
 If you want the shell to actually see a literal $, you need to type it
 twice::
 
     In [3]: !echo "A system variable: $$HOME"  
     A system variable: /home/fperez
 
 You can pass arbitrary expressions, though you'll need to delimit them
 with {} if there is ambiguity as to the extent of the expression::
 
     In [5]: x=10 
     In [6]: y=20 
     In [13]: !echo $x+y  
     10+y  
     In [7]: !echo ${x+y}  
     30
 
 Even object attributes can be expanded::
 
     In [12]: !echo $sys.argv  
     [/home/fperez/usr/bin/ipython]
 
 
 System command aliases
 ----------------------
 
 The %alias magic function and the alias option in the ipythonrc
 configuration file allow you to define magic functions which are in fact
 system shell commands. These aliases can have parameters.
 
 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
 Then, typing '%alias_name params' will execute the system command 'cmd
 params' (from your underlying operating system).
 
 You can also define aliases with parameters using %s specifiers (one per
 parameter). The following example defines the %parts function as an
 alias to the command 'echo first %s second %s' where each %s will be
 replaced by a positional parameter to the call to %parts::
 
     In [1]: alias parts echo first %s second %s 
     In [2]: %parts A B 
     first A second B  
     In [3]: %parts A  
     Incorrect number of arguments: 2 expected.  
     parts is an alias to: 'echo first %s second %s'
 
 If called with no parameters, %alias prints the table of currently
 defined aliases.
 
 The %rehash/rehashx magics allow you to load your entire $PATH as
 ipython aliases. See their respective docstrings (or sec. 6.2
 <#sec:magic> for further details).
 
 
 .. _dreload:
 
 Recursive reload
 ----------------
 
 The dreload function does a recursive reload of a module: changes made
 to the module since you imported will actually be available without
 having to exit.
 
 
 Verbose and colored exception traceback printouts
 -------------------------------------------------
 
 IPython provides the option to see very detailed exception tracebacks,
 which can be especially useful when debugging large programs. You can
 run any Python file with the %run function to benefit from these
 detailed tracebacks. Furthermore, both normal and verbose tracebacks can
 be colored (if your terminal supports it) which makes them much easier
 to parse visually.
 
 See the magic xmode and colors functions for details (just type %magic).
 
 These features are basically a terminal version of Ka-Ping Yee's cgitb
 module, now part of the standard Python library.
 
 
 .. _input_caching:
 
 Input caching system
 --------------------
 
 IPython offers numbered prompts (In/Out) with input and output caching.
 All input is saved and can be retrieved as variables (besides the usual
 arrow key recall).
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 _i: stores previous input. _ii: next previous. _iii: next-next previous.
 _ih : a list of all input _ih[n] is the input from line n and this list
 is aliased to the global variable In. If you overwrite In with a
 variable of your own, you can remake the assignment to the internal list
 with a simple 'In=_ih'.
 
 Additionally, global variables named _i<n> are dynamically created (<n>
 being the prompt counter), such that
 _i<n> == _ih[<n>] == In[<n>].
 
 For example, what you typed at prompt 14 is available as _i14, _ih[14]
 and In[14].
 
 This allows you to easily cut and paste multi line interactive prompts
 by printing them out: they print like a clean string, without prompt
 characters. You can also manipulate them like regular variables (they
 are strings), modify or exec them (typing 'exec _i9' will re-execute the
 contents of input prompt 9, 'exec In[9:14]+In[18]' will re-execute lines
 9 through 13 and line 18).
 
 You can also re-execute multiple lines of input easily by using the
 magic %macro function (which automates the process and allows
 re-execution without having to type 'exec' every time). The macro system
 also allows you to re-execute previous lines which include magic
 function calls (which require special processing). Type %macro? or see
 sec. 6.2 <#sec:magic> for more details on the macro system.
 
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
 .. _output_caching:
 
 Output caching system
 ---------------------
 
 For output that is returned from actions, a system similar to the input
 cache exists but using _ instead of _i. Only actions that produce a
 result (NOT assignments, for example) are cached. If you are familiar
 with Mathematica, IPython's _ variables behave exactly like
 Mathematica's % variables.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 
     * [_] (a single underscore) : stores previous output, like Python's
       default interpreter.
     * [__] (two underscores): next previous.
     * [___] (three underscores): next-next previous.
 
 Additionally, global variables named _<n> are dynamically created (<n>
 being the prompt counter), such that the result of output <n> is always
 available as _<n> (don't use the angle brackets, just the number, e.g.
 _21).
 
 These global variables are all stored in a global dictionary (not a
 list, since it only has entries for lines which returned a result)
 available under the names _oh and Out (similar to _ih and In). So the
 output from line 12 can be obtained as _12, Out[12] or _oh[12]. If you
 accidentally overwrite the Out variable you can recover it by typing
 'Out=_oh' at the prompt.
 
 This system obviously can potentially put heavy memory demands on your
 system, since it prevents Python's garbage collector from removing any
 previously computed results. You can control how many results are kept
 in memory with the option (at the command line or in your ipythonrc
 file) cache_size. If you set it to 0, the whole system is completely
 disabled and the prompts revert to the classic '>>>' of normal Python.
 
 
 Directory history
 -----------------
 
 Your history of visited directories is kept in the global list _dh, and
 the magic %cd command can be used to go to any entry in that list. The
 %dhist command allows you to view this history. do ``cd -<TAB`` to
 conventiently view the directory history.
 
 
 Automatic parentheses and quotes
 --------------------------------
 
 These features were adapted from Nathan Gray's LazyPython. They are
 meant to allow less typing for common situations.
 
 
 Automatic parentheses
 ---------------------
 
 Callable objects (i.e. functions, methods, etc) can be invoked like this
 (notice the commas between the arguments)::
 
     >>> callable_ob arg1, arg2, arg3
 
 and the input will be translated to this::
 
     -> callable_ob(arg1, arg2, arg3)
 
 You can force automatic parentheses by using '/' as the first character
 of a line. For example::
 
     >>> /globals # becomes 'globals()'
 
 Note that the '/' MUST be the first character on the line! This won't work::
 
     >>> print /globals # syntax error
 
 In most cases the automatic algorithm should work, so you should rarely
 need to explicitly invoke /. One notable exception is if you are trying
 to call a function with a list of tuples as arguments (the parenthesis
 will confuse IPython)::
 
     In [1]: zip (1,2,3),(4,5,6) # won't work
 
 but this will work::
 
     In [2]: /zip (1,2,3),(4,5,6) 
     ---> zip ((1,2,3),(4,5,6))  
     Out[2]= [(1, 4), (2, 5), (3, 6)]
 
 IPython tells you that it has altered your command line by displaying
 the new command line preceded by ->. e.g.::
 
     In [18]: callable list 
     ----> callable (list)
 
 
 Automatic quoting
 -----------------
 
 You can force automatic quoting of a function's arguments by using ','
 or ';' as the first character of a line. For example::
 
     >>> ,my_function /home/me # becomes my_function("/home/me")
 
 If you use ';' instead, the whole argument is quoted as a single string
 (while ',' splits on whitespace)::
 
     >>> ,my_function a b c # becomes my_function("a","b","c")
 
     >>> ;my_function a b c # becomes my_function("a b c")
 
 Note that the ',' or ';' MUST be the first character on the line! This
 won't work::
 
     >>> x = ,my_function /home/me # syntax error
 
 IPython as your default Python environment
 ==========================================
 
 Python honors the environment variable PYTHONSTARTUP and will execute at
 startup the file referenced by this variable. If you put at the end of
 this file the following two lines of code::
 
     import IPython 
     IPython.Shell.IPShell().mainloop(sys_exit=1)
 
 then IPython will be your working environment anytime you start Python.
 The sys_exit=1 is needed to have IPython issue a call to sys.exit() when
 it finishes, otherwise you'll be back at the normal Python '>>>'
 prompt.
 
 This is probably useful to developers who manage multiple Python
 versions and don't want to have correspondingly multiple IPython
 versions. Note that in this mode, there is no way to pass IPython any
 command-line options, as those are trapped first by Python itself.
 
 .. _Embedding:
 
 Embedding IPython
 =================
 
 It is possible to start an IPython instance inside your own Python
 programs. This allows you to evaluate dynamically the state of your
 code, operate with your variables, analyze them, etc. Note however that
 any changes you make to values while in the shell do not propagate back
 to the running code, so it is safe to modify your values because you
 won't break your code in bizarre ways by doing so.
 
 This feature allows you to easily have a fully functional python
 environment for doing object introspection anywhere in your code with a
 simple function call. In some cases a simple print statement is enough,
 but if you need to do more detailed analysis of a code fragment this
 feature can be very valuable.
 
 It can also be useful in scientific computing situations where it is
 common to need to do some automatic, computationally intensive part and
 then stop to look at data, plots, etc.
 Opening an IPython instance will give you full access to your data and
 functions, and you can resume program execution once you are done with
 the interactive part (perhaps to stop again later, as many times as
 needed).
 
 The following code snippet is the bare minimum you need to include in
 your Python programs for this to work (detailed examples follow later)::
 
     from IPython.Shell import IPShellEmbed
 
     ipshell = IPShellEmbed()
 
     ipshell() # this call anywhere in your program will start IPython
 
 You can run embedded instances even in code which is itself being run at
 the IPython interactive prompt with '%run <filename>'. Since it's easy
 to get lost as to where you are (in your top-level IPython or in your
 embedded one), it's a good idea in such cases to set the in/out prompts
 to something different for the embedded instances. The code examples
 below illustrate this.
 
 You can also have multiple IPython instances in your program and open
 them separately, for example with different options for data
 presentation. If you close and open the same instance multiple times,
 its prompt counters simply continue from each execution to the next.
 
 Please look at the docstrings in the Shell.py module for more details on
 the use of this system.
 
 The following sample file illustrating how to use the embedding
 functionality is provided in the examples directory as example-embed.py.
 It should be fairly self-explanatory::
 
 
     #!/usr/bin/env python
 
     """An example of how to embed an IPython shell into a running program.
 
     Please see the documentation in the IPython.Shell module for more details.
 
     The accompanying file example-embed-short.py has quick code fragments for
     embedding which you can cut and paste in your code once you understand how
     things work.
 
     The code in this file is deliberately extra-verbose, meant for learning."""
 
     # The basics to get you going:
 
     # IPython sets the __IPYTHON__ variable so you can know if you have nested
     # copies running.
 
     # Try running this code both at the command line and from inside IPython (with
     # %run example-embed.py)
     try:
         __IPYTHON__
     except NameError:
         nested = 0
         args = ['']
     else:
         print "Running nested copies of IPython."
         print "The prompts for the nested copy have been modified"
         nested = 1
         # what the embedded instance will see as sys.argv:
         args = ['-pi1','In <\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
 
     # Now create an instance of the embeddable shell. The first argument is a
     # string with options exactly as you would type them if you were starting
     # IPython at the system command line. Any parameters you want to define for
     # configuration can thus be specified here.
     ipshell = IPShellEmbed(args,
                            banner = 'Dropping into IPython',
                            exit_msg = 'Leaving Interpreter, back to program.')
 
     # Make a second instance, you can have as many as you want.
     if nested:
         args[1] = 'In2<\\#>'
     else:
         args = ['-pi1','In2<\\#>: ','-pi2','   .\\D.: ',
                 '-po','Out<\\#>: ','-nosep']
     ipshell2 = IPShellEmbed(args,banner = 'Second IPython instance.')
 
     print '\nHello. This is printed from the main controller program.\n'
 
     # You can then call ipshell() anywhere you need it (with an optional
     # message):
     ipshell('***Called from top level. '
             'Hit Ctrl-D to exit interpreter and continue program.\n'
             'Note that if you use %kill_embedded, you can fully deactivate\n'
             'This embedded instance so it will never turn on again')
 
     print '\nBack in caller program, moving along...\n'
 
     #---------------------------------------------------------------------------
     # More details:
 
     # IPShellEmbed instances don't print the standard system banner and
     # messages. The IPython banner (which actually may contain initialization
     # messages) is available as <instance>.IP.BANNER in case you want it.
 
     # IPShellEmbed instances print the following information everytime they
     # start:
 
     # - A global startup banner.
 
     # - A call-specific header string, which you can use to indicate where in the
     # execution flow the shell is starting.
 
     # They also print an exit message every time they exit.
 
     # Both the startup banner and the exit message default to None, and can be set
     # either at the instance constructor or at any other time with the
     # set_banner() and set_exit_msg() methods.
 
     # The shell instance can be also put in 'dummy' mode globally or on a per-call
     # basis. This gives you fine control for debugging without having to change
     # code all over the place.
 
     # The code below illustrates all this.
 
 
     # This is how the global banner and exit_msg can be reset at any point
     ipshell.set_banner('Entering interpreter - New Banner')
     ipshell.set_exit_msg('Leaving interpreter - New exit_msg')
 
     def foo(m):
         s = 'spam'
         ipshell('***In foo(). Try @whos, or print s or m:')
         print 'foo says m = ',m
 
     def bar(n):
         s = 'eggs'
         ipshell('***In bar(). Try @whos, or print s or n:')
         print 'bar says n = ',n
 
     # Some calls to the above functions which will trigger IPython:
     print 'Main program calling foo("eggs")\n'
     foo('eggs')
 
     # The shell can be put in 'dummy' mode where calls to it silently return. This
     # allows you, for example, to globally turn off debugging for a program with a
     # single call.
     ipshell.set_dummy_mode(1)
     print '\nTrying to call IPython which is now "dummy":'
     ipshell()
     print 'Nothing happened...'
     # The global 'dummy' mode can still be overridden for a single call
     print '\nOverriding dummy mode manually:'
     ipshell(dummy=0)
 
     # Reactivate the IPython shell
     ipshell.set_dummy_mode(0)
 
     print 'You can even have multiple embedded instances:'
     ipshell2()
 
     print '\nMain program calling bar("spam")\n'
     bar('spam')
 
     print 'Main program finished. Bye!'
 
     #********************** End of file <example-embed.py> ***********************
 
 Once you understand how the system functions, you can use the following
 code fragments in your programs which are ready for cut and paste::
 
 
     """Quick code snippets for embedding IPython into other programs.
 
     See example-embed.py for full details, this file has the bare minimum code for
     cut and paste use once you understand how to use the system."""
 
     #---------------------------------------------------------------------------
     # This code loads IPython but modifies a few things if it detects it's running
     # embedded in another IPython session (helps avoid confusion)
 
     try:
         __IPYTHON__
     except NameError:
         argv = ['']
         banner = exit_msg = ''
     else:
         # Command-line options for IPython (a list like sys.argv)
         argv = ['-pi1','In <\\#>:','-pi2','   .\\D.:','-po','Out<\\#>:']
         banner = '*** Nested interpreter ***'
         exit_msg = '*** Back in main IPython ***'
 
     # First import the embeddable shell class
     from IPython.Shell import IPShellEmbed
     # Now create the IPython shell instance. Put ipshell() anywhere in your code
     # where you want it to open.
     ipshell = IPShellEmbed(argv,banner=banner,exit_msg=exit_msg)
 
     #---------------------------------------------------------------------------
     # This code will load an embeddable IPython shell always with no changes for
     # nested embededings.
 
     from IPython.Shell import IPShellEmbed
     ipshell = IPShellEmbed()
     # Now ipshell() will open IPython anywhere in the code.
 
     #---------------------------------------------------------------------------
     # This code loads an embeddable shell only if NOT running inside
     # IPython. Inside IPython, the embeddable shell variable ipshell is just a
     # dummy function.
 
     try:
         __IPYTHON__
     except NameError:
         from IPython.Shell import IPShellEmbed
         ipshell = IPShellEmbed()
         # Now ipshell() will open IPython anywhere in the code
     else:
         # Define a dummy ipshell() so the same code doesn't crash inside an
         # interactive IPython
         def ipshell(): pass
 
     #******************* End of file <example-embed-short.py> ********************
 
 Using the Python debugger (pdb)
 ===============================
 
 Running entire programs via pdb
 -------------------------------
 
 pdb, the Python debugger, is a powerful interactive debugger which
 allows you to step through code, set breakpoints, watch variables,
 etc.  IPython makes it very easy to start any script under the control
 of pdb, regardless of whether you have wrapped it into a 'main()'
 function or not. For this, simply type '%run -d myscript' at an
 IPython prompt. See the %run command's documentation (via '%run?' or
 in Sec. magic_ for more details, including how to control where pdb
 will stop execution first.
 
 For more information on the use of the pdb debugger, read the included
 pdb.doc file (part of the standard Python distribution). On a stock
 Linux system it is located at /usr/lib/python2.3/pdb.doc, but the
 easiest way to read it is by using the help() function of the pdb module
 as follows (in an IPython prompt):
 
 In [1]: import pdb 
 In [2]: pdb.help()
 
 This will load the pdb.doc document in a file viewer for you automatically.
 
 
 Automatic invocation of pdb on exceptions
 -----------------------------------------
 
 IPython, if started with the -pdb option (or if the option is set in
 your rc file) can call the Python pdb debugger every time your code
 triggers an uncaught exception. This feature
 can also be toggled at any time with the %pdb magic command. This can be
 extremely useful in order to find the origin of subtle bugs, because pdb
 opens up at the point in your code which triggered the exception, and
 while your program is at this point 'dead', all the data is still
 available and you can walk up and down the stack frame and understand
 the origin of the problem.
 
 Furthermore, you can use these debugging facilities both with the
 embedded IPython mode and without IPython at all. For an embedded shell
 (see sec. Embedding_), simply call the constructor with
 '-pdb' in the argument string and automatically pdb will be called if an
 uncaught exception is triggered by your code.
 
 For stand-alone use of the feature in your programs which do not use
 IPython at all, put the following lines toward the top of your 'main'
 routine::
 
     import sys,IPython.ultraTB  
     sys.excepthook = IPython.ultraTB.FormattedTB(mode='Verbose',
     color_scheme='Linux', call_pdb=1)
 
 The mode keyword can be either 'Verbose' or 'Plain', giving either very
 detailed or normal tracebacks respectively. The color_scheme keyword can
 be one of 'NoColor', 'Linux' (default) or 'LightBG'. These are the same
 options which can be set in IPython with -colors and -xmode.
 
 This will give any of your programs detailed, colored tracebacks with
 automatic invocation of pdb.
 
 
 Extensions for syntax processing
 ================================
 
 This isn't for the faint of heart, because the potential for breaking
 things is quite high. But it can be a very powerful and useful feature.
 In a nutshell, you can redefine the way IPython processes the user input
 line to accept new, special extensions to the syntax without needing to
 change any of IPython's own code.
 
 In the IPython/Extensions directory you will find some examples
 supplied, which we will briefly describe now. These can be used 'as is'
 (and both provide very useful functionality), or you can use them as a
 starting point for writing your own extensions.
 
 
 Pasting of code starting with '>>> ' or '... '
 ----------------------------------------------
 
 In the python tutorial it is common to find code examples which have
 been taken from real python sessions. The problem with those is that all
 the lines begin with either '>>> ' or '... ', which makes it impossible
 to paste them all at once. One must instead do a line by line manual
 copying, carefully removing the leading extraneous characters.
 
 This extension identifies those starting characters and removes them
 from the input automatically, so that one can paste multi-line examples
 directly into IPython, saving a lot of time. Please look at the file
 InterpreterPasteInput.py in the IPython/Extensions directory for details
 on how this is done.
 
 IPython comes with a special profile enabling this feature, called
 tutorial. Simply start IPython via 'ipython -p tutorial' and the feature
 will be available. In a normal IPython session you can activate the
 feature by importing the corresponding module with:
 In [1]: import IPython.Extensions.InterpreterPasteInput
 
 The following is a 'screenshot' of how things work when this extension
 is on, copying an example from the standard tutorial::
 
     IPython profile: tutorial 
 
     *** Pasting of code with ">>>" or "..." has been enabled. 
 
     In [1]: >>> def fib2(n): # return Fibonacci series up to n 
        ...: ...     """Return a list containing the Fibonacci series up to
     n.""" 
        ...: ...     result = [] 
        ...: ...     a, b = 0, 1 
        ...: ...     while b < n: 
        ...: ...         result.append(b)    # see below 
        ...: ...         a, b = b, a+b 
        ...: ...     return result 
        ...: 
 
     In [2]: fib2(10) 
     Out[2]: [1, 1, 2, 3, 5, 8]
 
 Note that as currently written, this extension does not recognize
 IPython's prompts for pasting. Those are more complicated, since the
 user can change them very easily, they involve numbers and can vary in
 length. One could however extract all the relevant information from the
 IPython instance and build an appropriate regular expression. This is
 left as an exercise for the reader.
 
 
 Input of physical quantities with units
 ---------------------------------------
 
 The module PhysicalQInput allows a simplified form of input for physical
 quantities with units. This file is meant to be used in conjunction with
 the PhysicalQInteractive module (in the same directory) and
 Physics.PhysicalQuantities from Konrad Hinsen's ScientificPython
 (http://dirac.cnrs-orleans.fr/ScientificPython/).
 
 The Physics.PhysicalQuantities module defines PhysicalQuantity objects,
 but these must be declared as instances of a class. For example, to
 define v as a velocity of 3 m/s, normally you would write::
 
     In [1]: v = PhysicalQuantity(3,'m/s')
 
 Using the PhysicalQ_Input extension this can be input instead as: 
 In [1]: v = 3 m/s
 which is much more convenient for interactive use (even though it is
 blatantly invalid Python syntax).
 
 The physics profile supplied with IPython (enabled via 'ipython -p
 physics') uses these extensions, which you can also activate with:
 
 from math import * # math MUST be imported BEFORE PhysicalQInteractive 
 from IPython.Extensions.PhysicalQInteractive import * 
 import IPython.Extensions.PhysicalQInput
 
 
 Threading support
 =================
 
 WARNING: The threading support is still somewhat experimental, and it
 has only seen reasonable testing under Linux. Threaded code is
 particularly tricky to debug, and it tends to show extremely
 platform-dependent behavior. Since I only have access to Linux machines,
 I will have to rely on user's experiences and assistance for this area
 of IPython to improve under other platforms.
 
 IPython, via the -gthread , -qthread, -q4thread and -wthread options
 (described in Sec. `Threading options`_), can run in
 multithreaded mode to support pyGTK, Qt3, Qt4 and WXPython applications
 respectively. These GUI toolkits need to control the python main loop of
 execution, so under a normal Python interpreter, starting a pyGTK, Qt3,
 Qt4 or WXPython application will immediately freeze the shell.
 
 IPython, with one of these options (you can only use one at a time),
 separates the graphical loop and IPython's code execution run into
 different threads. This allows you to test interactively (with %run, for
 example) your GUI code without blocking.
 
 A nice mini-tutorial on using IPython along with the Qt Designer
 application is available at the SciPy wiki:
 http://www.scipy.org/Cookbook/Matplotlib/Qt_with_IPython_and_Designer.
 
 
 Tk issues
 ---------
 
 As indicated in Sec. `Threading options`_, a special -tk option is
 provided to try and allow Tk graphical applications to coexist
 interactively with WX, Qt or GTK ones. Whether this works at all,
 however, is very platform and configuration dependent. Please
 experiment with simple test cases before committing to using this
 combination of Tk and GTK/Qt/WX threading in a production environment.
 
 
 I/O pitfalls
 ------------
 
 Be mindful that the Python interpreter switches between threads every
 $N$ bytecodes, where the default value as of Python 2.3 is $N=100.$ This
 value can be read by using the sys.getcheckinterval() function, and it
 can be reset via sys.setcheckinterval(N). This switching of threads can
 cause subtly confusing effects if one of your threads is doing file I/O.
 In text mode, most systems only flush file buffers when they encounter a
 '\n'. An instruction as simple as:: 
 
   print >> filehandle, ''hello world''
 
 actually consists of several bytecodes, so it is possible that the
 newline does not reach your file before the next thread switch.
 Similarly, if you are writing to a file in binary mode, the file won't
 be flushed until the buffer fills, and your other thread may see
 apparently truncated files.
 
 For this reason, if you are using IPython's thread support and have (for
 example) a GUI application which will read data generated by files
 written to from the IPython thread, the safest approach is to open all
 of your files in unbuffered mode (the third argument to the file/open
 function is the buffering value)::
 
   filehandle = open(filename,mode,0)
 
 This is obviously a brute force way of avoiding race conditions with the
 file buffering. If you want to do it cleanly, and you have a resource
 which is being shared by the interactive IPython loop and your GUI
 thread, you should really handle it with thread locking and
 syncrhonization properties. The Python documentation discusses these.
 
 .. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
 
 IPython ships with a basic system for running scripts interactively in
 sections, useful when presenting code to audiences. A few tags embedded
 in comments (so that the script remains valid Python code) divide a file
 into separate blocks, and the demo can be run one block at a time, with
 IPython printing (with syntax highlighting) the block before executing
 it, and returning to the interactive prompt after each block. The
 interactive namespace is updated after each block is run with the
 contents of the demo's namespace.
 
 This allows you to show a piece of code, run it and then execute
 interactively commands based on the variables just created. Once you
 want to continue, you simply execute the next block of the demo. The
 following listing shows the markup necessary for dividing a script into
 sections for execution as a demo::
 
 
     """A simple interactive demo to illustrate the use of IPython's Demo class.
 
     Any python script can be run as a demo, but that does little more than showing
     it on-screen, syntax-highlighted in one shot.  If you add a little simple
     markup, you can stop at specified intervals and return to the ipython prompt,
     resuming execution later.
     """
 
     print 'Hello, welcome to an interactive IPython demo.'
     print 'Executing this block should require confirmation before proceeding,'
     print 'unless auto_all has been set to true in the demo object'
 
     # The mark below defines a block boundary, which is a point where IPython will
     # stop execution and return to the interactive prompt.
     # Note that in actual interactive execution, 
     # <demo> --- stop ---
 
     x = 1
     y = 2
 
     # <demo> --- stop ---
 
     # the mark below makes this block as silent
     # <demo> silent
 
     print 'This is a silent block, which gets executed but not printed.'
 
     # <demo> --- stop ---
     # <demo> auto
     print 'This is an automatic block.'
     print 'It is executed without asking for confirmation, but printed.'
     z = x+y
 
     print 'z=',x
 
     # <demo> --- stop ---
     # This is just another normal block.
     print 'z is now:', z
 
     print 'bye!'
 
 In order to run a file as a demo, you must first make a Demo object out
 of it. If the file is named myscript.py, the following code will make a
 demo::
 
     from IPython.demo import Demo
 
     mydemo = Demo('myscript.py')
 
 This creates the mydemo object, whose blocks you run one at a time by
 simply calling the object with no arguments. If you have autocall active
 in IPython (the default), all you need to do is type::
 
     mydemo 
 
 and IPython will call it, executing each block. Demo objects can be
 restarted, you can move forward or back skipping blocks, re-execute the
 last block, etc. Simply use the Tab key on a demo object to see its
 methods, and call '?' on them to see their docstrings for more usage
 details. In addition, the demo module itself contains a comprehensive
 docstring, which you can access via::
 
     from IPython import demo
 
     demo?
 
 Limitations: It is important to note that these demos are limited to
 fairly simple uses. In particular, you can not put division marks in
 indented code (loops, if statements, function definitions, etc.)
 Supporting something like this would basically require tracking the
 internal execution state of the Python interpreter, so only top-level
 divisions are allowed. If you want to be able to open an IPython
 instance at an arbitrary point in a program, you can use IPython's
 embedding facilities, described in detail in Sec. 9
 
 
 .. _Matplotlib support:
 
 Plotting with matplotlib
 ========================
 
 The matplotlib library (http://matplotlib.sourceforge.net
 http://matplotlib.sourceforge.net) provides high quality 2D plotting for
 Python. Matplotlib can produce plots on screen using a variety of GUI
 toolkits, including Tk, GTK and WXPython. It also provides a number of
 commands useful for scientific computing, all with a syntax compatible
 with that of the popular Matlab program.
 
 IPython accepts the special option -pylab (see :ref:`here
 <command_line_options>`). This configures it to support matplotlib, honoring
 the settings in the .matplotlibrc file. IPython will detect the user's choice
 of matplotlib GUI backend, and automatically select the proper threading model
 to prevent blocking. It also sets matplotlib in interactive mode and modifies
 %run slightly, so that any matplotlib-based script can be executed using %run
 and the final show() command does not block the interactive shell.
 
 The -pylab option must be given first in order for IPython to configure its
 threading mode. However, you can still issue other options afterwards. This
 allows you to have a matplotlib-based environment customized with additional
 modules using the standard IPython profile mechanism (see :ref:`here
 <profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
 ipythonrc-myprofile after configuring matplotlib.

docs/sphinxext/inheritance_diagram.py

@@ -1,423 +1,407
 """
 Defines a docutils directive for inserting inheritance diagrams.
 
 Provide the directive with one or more classes or modules (separated
 by whitespace).  For modules, all of the classes in that module will
 be used.
 
 Example::
 
    Given the following classes:
 
    class A: pass
    class B(A): pass
    class C(A): pass
    class D(B, C): pass
    class E(B): pass
 
    .. inheritance-diagram: D E
 
    Produces a graph like the following:
 
                A
               / \
              B   C
             / \ /
            E   D
 
 The graph is inserted as a PNG+image map into HTML and a PDF in
 LaTeX.
 """
 
 import inspect
 import os
 import re
 import subprocess
 try:
     from hashlib import md5
 except ImportError:
     from md5 import md5
 
 from docutils.nodes import Body, Element
-from docutils.writers.html4css1 import HTMLTranslator
-from sphinx.latexwriter import LaTeXTranslator
 from docutils.parsers.rst import directives
 from sphinx.roles import xfileref_role
 
+def my_import(name):
+    """Module importer - taken from the python documentation.
+
+    This function allows importing names with dots in them."""
+    
+    mod = __import__(name)
+    components = name.split('.')
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod
+
 class DotException(Exception):
     pass
 
 class InheritanceGraph(object):
     """
     Given a list of classes, determines the set of classes that
     they inherit from all the way to the root "object", and then
     is able to generate a graphviz dot graph from them.
     """
     def __init__(self, class_names, show_builtins=False):
         """
         *class_names* is a list of child classes to show bases from.
 
         If *show_builtins* is True, then Python builtins will be shown
         in the graph.
         """
         self.class_names = class_names
         self.classes = self._import_classes(class_names)
         self.all_classes = self._all_classes(self.classes)
         if len(self.all_classes) == 0:
             raise ValueError("No classes found for inheritance diagram")
         self.show_builtins = show_builtins
 
     py_sig_re = re.compile(r'''^([\w.]*\.)?    # class names
                            (\w+)  \s* $        # optionally arguments
                            ''', re.VERBOSE)
 
     def _import_class_or_module(self, name):
         """
         Import a class using its fully-qualified *name*.
         """
         try:
             path, base = self.py_sig_re.match(name).groups()
         except:
             raise ValueError(
                 "Invalid class or module '%s' specified for inheritance diagram" % name)
         fullname = (path or '') + base
         path = (path and path.rstrip('.'))
         if not path:
             path = base
-        if not path:
-            raise ValueError(
-                "Invalid class or module '%s' specified for inheritance diagram" % name)
         try:
             module = __import__(path, None, None, [])
+            # We must do an import of the fully qualified name.  Otherwise if a
+            # subpackage 'a.b' is requested where 'import a' does NOT provide
+            # 'a.b' automatically, then 'a.b' will not be found below.  This
+            # second call will force the equivalent of 'import a.b' to happen
+            # after the top-level import above.
+            my_import(fullname)
+            
         except ImportError:
             raise ValueError(
                 "Could not import class or module '%s' specified for inheritance diagram" % name)
 
         try:
             todoc = module
             for comp in fullname.split('.')[1:]:
                 todoc = getattr(todoc, comp)
         except AttributeError:
             raise ValueError(
                 "Could not find class or module '%s' specified for inheritance diagram" % name)
 
         # If a class, just return it
         if inspect.isclass(todoc):
             return [todoc]
         elif inspect.ismodule(todoc):
             classes = []
             for cls in todoc.__dict__.values():
                 if inspect.isclass(cls) and cls.__module__ == todoc.__name__:
                     classes.append(cls)
             return classes
         raise ValueError(
             "'%s' does not resolve to a class or module" % name)
 
     def _import_classes(self, class_names):
         """
         Import a list of classes.
         """
         classes = []
         for name in class_names:
             classes.extend(self._import_class_or_module(name))
         return classes
 
     def _all_classes(self, classes):
         """
         Return a list of all classes that are ancestors of *classes*.
         """
         all_classes = {}
 
         def recurse(cls):
             all_classes[cls] = None
             for c in cls.__bases__:
                 if c not in all_classes:
                     recurse(c)
 
         for cls in classes:
             recurse(cls)
 
         return all_classes.keys()
 
     def class_name(self, cls, parts=0):
         """
         Given a class object, return a fully-qualified name.  This
         works for things I've tested in matplotlib so far, but may not
         be completely general.
         """
         module = cls.__module__
         if module == '__builtin__':
             fullname = cls.__name__
         else:
             fullname = "%s.%s" % (module, cls.__name__)
         if parts == 0:
             return fullname
         name_parts = fullname.split('.')
         return '.'.join(name_parts[-parts:])
 
     def get_all_class_names(self):
         """
         Get all of the class names involved in the graph.
         """
         return [self.class_name(x) for x in self.all_classes]
 
     # These are the default options for graphviz
     default_graph_options = {
         "rankdir": "LR",
         "size": '"8.0, 12.0"'
         }
     default_node_options = {
         "shape": "box",
         "fontsize": 10,
         "height": 0.25,
         "fontname": "Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",
         "style": '"setlinewidth(0.5)"'
         }
     default_edge_options = {
         "arrowsize": 0.5,
         "style": '"setlinewidth(0.5)"'
         }
 
     def _format_node_options(self, options):
         return ','.join(["%s=%s" % x for x in options.items()])
     def _format_graph_options(self, options):
         return ''.join(["%s=%s;\n" % x for x in options.items()])
 
     def generate_dot(self, fd, name, parts=0, urls={},
                      graph_options={}, node_options={},
                      edge_options={}):
         """
         Generate a graphviz dot graph from the classes that
         were passed in to __init__.
 
         *fd* is a Python file-like object to write to.
 
         *name* is the name of the graph
 
         *urls* is a dictionary mapping class names to http urls
 
         *graph_options*, *node_options*, *edge_options* are
         dictionaries containing key/value pairs to pass on as graphviz
         properties.
         """
         g_options = self.default_graph_options.copy()
         g_options.update(graph_options)
         n_options = self.default_node_options.copy()
         n_options.update(node_options)
         e_options = self.default_edge_options.copy()
         e_options.update(edge_options)
 
         fd.write('digraph %s {\n' % name)
         fd.write(self._format_graph_options(g_options))
 
         for cls in self.all_classes:
             if not self.show_builtins and cls in __builtins__.values():
                 continue
 
             name = self.class_name(cls, parts)
 
             # Write the node
             this_node_options = n_options.copy()
             url = urls.get(self.class_name(cls))
             if url is not None:
                 this_node_options['URL'] = '"%s"' % url
             fd.write('  "%s" [%s];\n' %
                      (name, self._format_node_options(this_node_options)))
 
             # Write the edges
             for base in cls.__bases__:
                 if not self.show_builtins and base in __builtins__.values():
                     continue
 
                 base_name = self.class_name(base, parts)
                 fd.write('  "%s" -> "%s" [%s];\n' %
                          (base_name, name,
                           self._format_node_options(e_options)))
         fd.write('}\n')
 
     def run_dot(self, args, name, parts=0, urls={},
                 graph_options={}, node_options={}, edge_options={}):
         """
         Run graphviz 'dot' over this graph, returning whatever 'dot'
         writes to stdout.
 
         *args* will be passed along as commandline arguments.
 
         *name* is the name of the graph
 
         *urls* is a dictionary mapping class names to http urls
 
         Raises DotException for any of the many os and
         installation-related errors that may occur.
         """
         try:
             dot = subprocess.Popen(['dot'] + list(args),
                                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                    close_fds=True)
         except OSError:
             raise DotException("Could not execute 'dot'.  Are you sure you have 'graphviz' installed?")
         except ValueError:
             raise DotException("'dot' called with invalid arguments")
         except:
             raise DotException("Unexpected error calling 'dot'")
 
         self.generate_dot(dot.stdin, name, parts, urls, graph_options,
                           node_options, edge_options)
         dot.stdin.close()
         result = dot.stdout.read()
         returncode = dot.wait()
         if returncode != 0:
             raise DotException("'dot' returned the errorcode %d" % returncode)
         return result
 
 class inheritance_diagram(Body, Element):
     """
     A docutils node to use as a placeholder for the inheritance
     diagram.
     """
     pass
 
-def inheritance_diagram_directive_run(class_names, options, state):
+def inheritance_diagram_directive(name, arguments, options, content, lineno,
+                                  content_offset, block_text, state,
+                                  state_machine):
     """
     Run when the inheritance_diagram directive is first encountered.
     """
     node = inheritance_diagram()
 
+    class_names = arguments
+
     # Create a graph starting with the list of classes
     graph = InheritanceGraph(class_names)
 
     # Create xref nodes for each target of the graph's image map and
     # add them to the doc tree so that Sphinx can resolve the
     # references to real URLs later.  These nodes will eventually be
     # removed from the doctree after we're done with them.
     for name in graph.get_all_class_names():
         refnodes, x = xfileref_role(
             'class', ':class:`%s`' % name, name, 0, state)
         node.extend(refnodes)
     # Store the graph object so we can use it to generate the
     # dot file later
     node['graph'] = graph
     # Store the original content for use as a hash
     node['parts'] = options.get('parts', 0)
     node['content'] = " ".join(class_names)
     return [node]
 
 def get_graph_hash(node):
     return md5(node['content'] + str(node['parts'])).hexdigest()[-10:]
 
 def html_output_graph(self, node):
     """
     Output the graph for HTML.  This will insert a PNG with clickable
     image map.
     """
     graph = node['graph']
     parts = node['parts']
 
     graph_hash = get_graph_hash(node)
     name = "inheritance%s" % graph_hash
-    png_path = os.path.join('_static', name + ".png")
-
-    path = '_static'
-    source = self.document.attributes['source']
-    count = source.split('/doc/')[-1].count('/')
-    for i in range(count):
-        if os.path.exists(path): break
-        path = '../'+path
-    path = '../'+path #specifically added for matplotlib
+    path = '_images'
+    dest_path = os.path.join(setup.app.builder.outdir, path)
+    if not os.path.exists(dest_path):
+        os.makedirs(dest_path)
+    png_path = os.path.join(dest_path, name + ".png")
+    path = setup.app.builder.imgpath
 
     # Create a mapping from fully-qualified class names to URLs.
     urls = {}
     for child in node:
         if child.get('refuri') is not None:
             urls[child['reftitle']] = child.get('refuri')
         elif child.get('refid') is not None:
             urls[child['reftitle']] = '#' + child.get('refid')
 
     # These arguments to dot will save a PNG file to disk and write
     # an HTML image map to stdout.
     image_map = graph.run_dot(['-Tpng', '-o%s' % png_path, '-Tcmapx'],
                               name, parts, urls)
     return ('<img src="%s/%s.png" usemap="#%s" class="inheritance"/>%s' %
             (path, name, name, image_map))
 
 def latex_output_graph(self, node):
     """
     Output the graph for LaTeX.  This will insert a PDF.
     """
     graph = node['graph']
     parts = node['parts']
 
     graph_hash = get_graph_hash(node)
     name = "inheritance%s" % graph_hash
-    pdf_path = os.path.join('_static', name + ".pdf")
+    dest_path = os.path.abspath(os.path.join(setup.app.builder.outdir, '_images'))
+    if not os.path.exists(dest_path):
+        os.makedirs(dest_path)
+    pdf_path = os.path.abspath(os.path.join(dest_path, name + ".pdf"))
 
     graph.run_dot(['-Tpdf', '-o%s' % pdf_path],
                   name, parts, graph_options={'size': '"6.0,6.0"'})
-    return '\\includegraphics{../../%s}' % pdf_path
+    return '\n\\includegraphics{%s}\n\n' % pdf_path
 
 def visit_inheritance_diagram(inner_func):
     """
     This is just a wrapper around html/latex_output_graph to make it
     easier to handle errors and insert warnings.
     """
     def visitor(self, node):
         try:
             content = inner_func(self, node)
         except DotException, e:
             # Insert the exception as a warning in the document
             warning = self.document.reporter.warning(str(e), line=node.line)
             warning.parent = node
             node.children = [warning]
         else:
             source = self.document.attributes['source']
             self.body.append(content)
             node.children = []
     return visitor
 
 def do_nothing(self, node):
     pass
 
-options_spec = {
-    'parts': directives.nonnegative_int
-    }
-
-# Deal with the old and new way of registering directives
-try:
-    from docutils.parsers.rst import Directive
-except ImportError:
-    from docutils.parsers.rst.directives import _directives
-    def inheritance_diagram_directive(name, arguments, options, content, lineno,
-                                      content_offset, block_text, state,
-                                      state_machine):
-        return inheritance_diagram_directive_run(arguments, options, state)
-    inheritance_diagram_directive.__doc__ = __doc__
-    inheritance_diagram_directive.arguments = (1, 100, 0)
-    inheritance_diagram_directive.options = options_spec
-    inheritance_diagram_directive.content = 0
-    _directives['inheritance-diagram'] = inheritance_diagram_directive
-else:
-    class inheritance_diagram_directive(Directive):
-        has_content = False
-        required_arguments = 1
-        optional_arguments = 100
-        final_argument_whitespace = False
-        option_spec = options_spec
-
-        def run(self):
-            return inheritance_diagram_directive_run(
-                self.arguments, self.options, self.state)
-    inheritance_diagram_directive.__doc__ = __doc__
-
-    directives.register_directive('inheritance-diagram',
-                                  inheritance_diagram_directive)
-
 def setup(app):
-    app.add_node(inheritance_diagram)
-
-    HTMLTranslator.visit_inheritance_diagram = \
-        visit_inheritance_diagram(html_output_graph)
-    HTMLTranslator.depart_inheritance_diagram = do_nothing
-
-    LaTeXTranslator.visit_inheritance_diagram = \
-        visit_inheritance_diagram(latex_output_graph)
-    LaTeXTranslator.depart_inheritance_diagram = do_nothing
+    setup.app = app
+    setup.confdir = app.confdir
+
+    app.add_node(
+        inheritance_diagram,
+        latex=(visit_inheritance_diagram(latex_output_graph), do_nothing),
+        html=(visit_inheritance_diagram(html_output_graph), do_nothing))
+    app.add_directive(
+        'inheritance-diagram', inheritance_diagram_directive,
+        False, (1, 100, 0), parts = directives.nonnegative_int)

docs/sphinxext/ipython_console_highlighting.py

@@ -1,75 +1,98
+"""reST directive for syntax-highlighting ipython interactive sessions.
+"""
+
+#-----------------------------------------------------------------------------
+# Needed modules
+
+# Standard library
+import re
+
+# Third party
 from pygments.lexer import Lexer, do_insertions
-from pygments.lexers.agile import PythonConsoleLexer, PythonLexer, \
-    PythonTracebackLexer
+from pygments.lexers.agile import (PythonConsoleLexer, PythonLexer, 
+                                   PythonTracebackLexer)
 from pygments.token import Comment, Generic
+
 from sphinx import highlighting
-import re
 
+
+#-----------------------------------------------------------------------------
+# Global constants
 line_re = re.compile('.*?\n')
 
+#-----------------------------------------------------------------------------
+# Code begins - classes and functions
+
 class IPythonConsoleLexer(Lexer):
     """
     For IPython console output or doctests, such as:
 
-    Tracebacks are not currently supported.
-
     .. sourcecode:: ipython
 
       In [1]: a = 'foo'
 
       In [2]: a
       Out[2]: 'foo'
 
       In [3]: print a
       foo
 
       In [4]: 1 / 0
+
+    Notes:
+
+      - Tracebacks are not currently supported.
+
+      - It assumes the default IPython prompts, not customized ones.
     """
+    
     name = 'IPython console session'
     aliases = ['ipython']
     mimetypes = ['text/x-ipython-console']
     input_prompt = re.compile("(In \[[0-9]+\]: )|(   \.\.\.+:)")
     output_prompt = re.compile("(Out\[[0-9]+\]: )|(   \.\.\.+:)")
     continue_prompt = re.compile("   \.\.\.+:")
     tb_start = re.compile("\-+")
 
     def get_tokens_unprocessed(self, text):
         pylexer = PythonLexer(**self.options)
         tblexer = PythonTracebackLexer(**self.options)
 
         curcode = ''
         insertions = []
         for match in line_re.finditer(text):
             line = match.group()
             input_prompt = self.input_prompt.match(line)
             continue_prompt = self.continue_prompt.match(line.rstrip())
             output_prompt = self.output_prompt.match(line)
             if line.startswith("#"):
                 insertions.append((len(curcode),
                                    [(0, Comment, line)]))
             elif input_prompt is not None:
                 insertions.append((len(curcode),
                                    [(0, Generic.Prompt, input_prompt.group())]))
                 curcode += line[input_prompt.end():]
             elif continue_prompt is not None:
                 insertions.append((len(curcode),
                                    [(0, Generic.Prompt, continue_prompt.group())]))
                 curcode += line[continue_prompt.end():]
             elif output_prompt is not None:
                 insertions.append((len(curcode),
                                    [(0, Generic.Output, output_prompt.group())]))
                 curcode += line[output_prompt.end():]
             else:
                 if curcode:
                     for item in do_insertions(insertions,
                                               pylexer.get_tokens_unprocessed(curcode)):
                         yield item
                         curcode = ''
                         insertions = []
                 yield match.start(), Generic.Output, line
         if curcode:
             for item in do_insertions(insertions,
                                       pylexer.get_tokens_unprocessed(curcode)):
                 yield item
 
+#-----------------------------------------------------------------------------
+# Register the extension as a valid pygments lexer
 highlighting.lexers['ipython'] = IPythonConsoleLexer()