main branch resync

IPython/external/argparse.py

@@ -0,0 +1,1867 b''
+# -*- coding: utf-8 -*-
+
+# Copyright � 2006 Steven J. Bethard <[e-mail]>.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted under the terms of the 3-clause BSD
+# license. No warranty expressed or implied.
+# For details, see the accompanying file LICENSE.txt.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+* handles both optional and positional arguments
+* produces highly informative usage messages
+* supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file:
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    HelpFormatter, RawDescriptionHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default, while
+        RawDescriptionHelpFormatter tells the parser not to perform any
+        line-wrapping on description text.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '0.8.0'
+
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = '==PARSER=='
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format:
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+    
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+    
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+        
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join(func(*args) for func, args in self.items)
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty            
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, optionals, positionals, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, optionals, positionals, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations            
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+                    
+            # update the maximum item length
+            invocation_length = max(len(s) for s in invocations)
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list        
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+
+    def format_help(self):
+        help = self._root_section.format_help() % dict(prog=self._prog)
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join(part
+                       for part in part_strings
+                       if part and part is not SUPPRESS)
+
+    def _format_usage(self, usage, optionals, positionals, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if no optionals or positionals are available, usage is just prog
+        if usage is None and not optionals and not positionals:
+            usage = '%(prog)s'
+            
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            usage = '%(prog)s' % dict(prog=self._prog)
+                
+            # determine width of "usage: PROG" and width of text
+            prefix_width = len(prefix) + len(usage) + 1
+            prefix_indent = self._current_indent + prefix_width
+            text_width = self._width - self._current_indent
+
+            # put them on one line if they're short enough
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals)
+            if prefix_width + len(action_usage) + 1 < text_width:
+                usage = '%s %s' % (usage, action_usage)
+            
+            # if they're long, wrap optionals and positionals individually
+            else:
+                optional_usage = format(optionals)
+                positional_usage = format(positionals)
+                indent = ' ' * prefix_indent
+
+                # usage is made of PROG, optionals and positionals
+                parts = [usage, ' ']
+                
+                # options always get added right after PROG
+                if optional_usage:
+                    parts.append(_textwrap.fill(
+                        optional_usage, text_width,
+                        initial_indent=indent,
+                        subsequent_indent=indent).lstrip())
+
+                # if there were options, put arguments on the next line
+                # otherwise, start them right after PROG
+                if positional_usage:
+                    part = _textwrap.fill(
+                        positional_usage, text_width,
+                        initial_indent=indent,
+                        subsequent_indent=indent).lstrip()
+                    if optional_usage:
+                        part = '\n' + indent + part
+                    parts.append(part)
+                usage = ''.join(parts)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions):
+        parts = []
+        for action in actions:
+            if action.help is SUPPRESS:
+                continue
+
+            # produce all arg strings        
+            if not action.option_strings:
+                parts.append(self._format_args(action, action.dest))
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required
+                if not action.required:
+                    part = '[%s]' % part
+                parts.append(part)
+
+        return ' '.join(parts)
+
+    def _format_text(self, text):
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string            
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            return self._format_metavar(action, action.dest)
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _format_metavar(self, action, default_metavar):
+        if action.metavar is not None:
+            name = action.metavar
+        elif action.choices is not None:
+            choice_strs = (str(choice) for choice in action.choices)
+            name = '{%s}' % ','.join(choice_strs)
+        else:
+            name = default_metavar
+        return name
+
+    def _format_args(self, action, default_metavar):
+        name = self._format_metavar(action, default_metavar)
+        if action.nargs is None:
+            result = name
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % name
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % (name, name)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % (name, name)
+        elif action.nargs is PARSER:
+            result = '%s ...' % name
+        else:
+            result = ' '.join([name] * action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name, value in params.items():
+            if value is SUPPRESS:
+                del params[name]
+        if params.get('choices') is not None:
+            choices_str = ', '.join(str(c) for c in params['choices'])
+            params['choices'] = choices_str
+        return action.help % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    
+    def _fill_text(self, text, width, indent):
+        return ''.join(indent + line for line in text.splitlines(True))
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+# =====================
+# Options and Arguments
+# =====================
+
+class ArgumentError(Exception):
+    """ArgumentError(message, argument)
+
+    Raised whenever there was an error creating or using an argument
+    (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+    
+    def __init__(self, argument, message):
+        if argument.option_strings:
+            self.argument_name =  '/'.join(argument.option_strings)
+        elif argument.metavar not in (None, SUPPRESS):
+            self.argument_name = argument.metavar
+        elif argument.dest not in (None, SUPPRESS):
+            self.argument_name = argument.dest
+        else:
+            self.argument_name = None
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Action(*strings, **options)
+
+    Action objects hold the information necessary to convert a
+    set of command-line arguments (possibly including an initial option
+    string) into the desired Python object(s).
+
+    Keyword Arguments:
+
+    option_strings -- A list of command-line option strings which
+        should be associated with this action.
+    
+    dest -- The name of the attribute to hold the created object(s)
+    
+    nargs -- The number of command-line arguments that should be consumed.
+        By default, one argument will be consumed and a single value will
+        be produced.  Other values include:
+            * N (an integer) consumes N arguments (and produces a list)
+            * '?' consumes zero or one arguments
+            * '*' consumes zero or more arguments (and produces a list)
+            * '+' consumes one or more arguments (and produces a list)
+        Note that the difference between the default and nargs=1 is that
+        with the default, a single value will be produced, while with
+        nargs=1, a list containing a single value will be produced.
+
+    const -- The value to be produced if the option is specified and the
+        option uses an action that takes no values.
+
+    default -- The value to be produced if the option is not specified.
+
+    type -- The type which the command-line arguments should be converted
+        to, should be one of 'string', 'int', 'float', 'complex' or a
+        callable object that accepts a single string argument. If None,
+        'string' is assumed.
+        
+    choices -- A container of values that should be allowed. If not None,
+        after a command-line argument has been converted to the appropriate
+        type, an exception will be raised if it is not a member of this
+        collection.
+
+    required -- True if the action must always be specified at the command
+        line. This is only meaningful for optional command-line arguments.
+
+    help -- The help string describing the argument.
+
+    metavar -- The name to be used for the option's argument with the help
+        string. If None, the 'dest' value will be used as the name.
+    """
+
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar'
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+class _StoreAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs must be > 0')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+class _StoreConstAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+class _StoreTrueAction(_StoreConstAction):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+class _StoreFalseAction(_StoreConstAction):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+    
+class _AppendAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs must be > 0')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        _ensure_value(namespace, self.dest, []).append(values)
+
+class _AppendConstAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        _ensure_value(namespace, self.dest, []).append(self.const)
+
+class _CountAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+class _HelpAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+class _VersionAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_version()
+        parser.exit()
+        
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+        def __init__(self, name, help):
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=name, help=help)
+
+        
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+        
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = {}
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser        
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            tup = parser_name, ', '.join(self._name_parser_map)
+            msg = _('unknown parser %r (choices: %s)' % tup)
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        parser.parse_args(arg_strings, namespace)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+    mode -- A string indicating how the file is to be opened. Accepts the
+        same values as the builtin open() function.
+    bufsize -- The file's desired buffer size. Accepts the same values as
+        the builtin open() function.
+    """    
+    def __init__(self, mode='r', bufsize=None):
+        self._mode = mode
+        self._bufsize = bufsize
+    
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if self._mode == 'r':
+                return _sys.stdin
+            elif self._mode == 'w':
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r' % self._mode)
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        if self._bufsize:
+            return open(string, self._mode, self._bufsize)
+        else:
+            return open(string, self._mode)
+
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+
+    def __init__(self, **kwargs):
+        for name, value in kwargs.iteritems():
+            setattr(self, name, value)
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class _ActionsContainer(object):
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+        
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._optional_actions_list = []
+        self._positional_actions_list = []
+        self._positional_actions_full_list = []
+        self._option_strings = {}
+
+        # defaults storage
+        self._defaults = {}
+
+    # ====================
+    # Registration methods
+    # ====================
+
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default settings methods
+    # ==================================
+
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action_list in [self._option_strings.values(),
+                            self._positional_actions_full_list]:
+            for action in action_list:
+                if action.dest in kwargs:
+                    action.default = kwargs[action.dest]
+
+    # =======================
+    # Adding argument actions
+    # =======================
+
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+            
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        action = action_class(**kwargs)
+        return self._add_action(action)
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to optional or positional list
+        if action.option_strings:
+            self._optional_actions_list.append(action)
+        else:
+            self._positional_actions_list.append(action)
+            self._positional_actions_full_list.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_strings[option_string] = action
+
+        # return the created action
+        return action
+
+    def _add_container_actions(self, container):
+        for action in container._optional_actions_list:
+            self._add_action(action)
+        for action in container._positional_actions_list:
+            self._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+        
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on one-or-fewer-character option strings
+            if len(option_string) < 2:
+                msg = _('invalid option string %r: '
+                        'must be at least two characters long')
+                raise ValueError(msg % option_string)
+
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                msg = _('invalid option string %r: '
+                        'must start with a character %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
+
+            # error on strings that are all prefix characters
+            if not (set(option_string) - set(self.prefix_chars)):
+                msg = _('invalid option string %r: '
+                        'must contain characters other than %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)            
+            if option_string[0] in self.prefix_chars:
+                if option_string[1] in self.prefix_chars:
+                    long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option        
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_strings:
+                confl_optional = self._option_strings[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = _('conflicting option string(s): %s')
+        conflict_string = ', '.join(option_string
+                                    for option_string, action
+                                    in conflicting_actions)
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options        
+        for option_string, action in conflicting_actions:
+            
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_strings.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._optional_actions_list.remove(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+        
+        self.title = title
+        self._registries = container._registries
+        self._positional_actions_full_list = container._positional_actions_full_list
+        self._option_strings = container._option_strings
+        self._defaults = container._defaults
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.add_help = add_help
+
+        self._argument_group_class = _ArgumentGroup
+        self._has_subparsers = False
+        self._argument_groups = []
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        if self.add_help:
+            self.add_argument(
+                '-h', '--help', action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                '-v', '--version', action='version', default=SUPPRESS,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+|-\d*.\d+$')
+                
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+
+    def add_argument_group(self, *args, **kwargs):
+        group = self._argument_group_class(self, *args, **kwargs)
+        self._argument_groups.append(group)
+        return group
+
+    def add_subparsers(self, **kwargs):
+        if self._has_subparsers:
+            self.error(_('cannot have multiple subparser arguments'))
+        
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            formatter.add_usage(self.usage, [],
+                                self._get_positional_actions(), '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._positional_actions_list.append(action)
+        self._positional_actions_full_list.append(action)
+        self._has_subparsers = True
+
+        # return the created parsers action
+        return action
+
+    def _add_container_actions(self, container):
+        super(ArgumentParser, self)._add_container_actions(container)
+        try:
+            groups = container._argument_groups
+        except AttributeError:
+            pass
+        else:
+            for group in groups:
+                new_group = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+                new_group._add_container_actions(group)
+
+    def _get_optional_actions(self):
+        actions = []
+        actions.extend(self._optional_actions_list)
+        for argument_group in self._argument_groups:
+            actions.extend(argument_group._optional_actions_list)
+        return actions
+
+    def _get_positional_actions(self):
+        return list(self._positional_actions_full_list)
+
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+
+    def parse_args(self, args=None, namespace=None):
+        # args default to the system args
+        if args is None:
+            args = _sys.argv[1:]
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+            
+        # add any action defaults that aren't present
+        optional_actions = self._get_optional_actions()
+        positional_actions = self._get_positional_actions()
+        for action in optional_actions + positional_actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        default = action.default
+                        if isinstance(action.default, basestring):
+                            default = self._get_value(action, default)
+                        setattr(namespace, action.dest, default)
+
+        # add any parser defaults that aren't present
+        for dest, value in self._defaults.iteritems():
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, value)
+            
+        # parse the arguments and exit if there are any errors
+        try:
+            result = self._parse_args(args, namespace)
+        except ArgumentError, err:
+            self.error(str(err))
+
+        # make sure all required optionals are present
+        for action in self._get_optional_actions():
+            if action.required:
+                if getattr(result, action.dest, None) is None:
+                    opt_strs = '/'.join(action.option_strings)
+                    msg = _('option %s is required' % opt_strs)
+                    self.error(msg)
+
+        # return the parsed arguments                    
+        return result
+
+    def _parse_args(self, arg_strings, namespace):            
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        def take_action(action, argument_strings, option_string=None):
+            argument_values = self._get_values(action, argument_strings)
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)            
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+            
+            # determine the optional action and parse any explicit
+            # argument out of the option string
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # loop because single-dash options can be chained
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, raise an error
+                if action is None:
+                    self.error(_('no such option: %s') % option_string)
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        parse_optional = self._parse_optional
+                        for char in self.prefix_chars:
+                            option_string = char + explicit_arg
+                            option_tuple = parse_optional(option_string)
+                            if option_tuple[0] is not None:
+                                break
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                        # set the action, etc. for the next loop iteration
+                        action, option_string, explicit_arg = option_tuple
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+                
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+            
+            # consume any Positionals preceding the next option
+            next_option_string_index = min(
+                index
+                for index in option_string_indices
+                if index >= start_index)
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were unparseable
+            # arguments
+            if start_index not in option_string_indices:
+                msg = _('extra arguments found: %s')
+                extras = arg_strings[start_index:next_option_string_index]
+                self.error(msg % ' '.join(extras))
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were too
+        # many supplied
+        if stop_index != len(arg_strings):
+            extras = arg_strings[stop_index:]
+            self.error(_('extra arguments found: %s') % ' '.join(extras))
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # return the updated namespace
+        return namespace
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match        
+        if match is None:
+            nargs_errors = {
+                None:_('expected one argument'),
+                OPTIONAL:_('expected at most one argument'),
+                ONE_OR_MORE:_('expected at least one argument')
+            }
+            default = _('expected %s argument(s)') % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in xrange(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join(self._get_nargs_pattern(action)
+                              for action in actions_slice)
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend(len(string) for string in match.groups())
+                break
+
+        # return the list of arg string counts
+        return result
+    
+    def _parse_optional(self, arg_string):
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+        
+        # if it's just dashes, it was meant to be positional
+        if not arg_string.strip('-'):
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_strings:
+            action = self._option_strings[arg_string]
+            return action, arg_string, None
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = []
+        prefix_tuples = self._get_option_prefix_tuples(arg_string)
+        for option_string in self._option_strings:
+            for option_prefix, explicit_arg in prefix_tuples:
+                if option_string.startswith(option_prefix):
+                    action = self._option_strings[option_string]
+                    tup = action, option_string, explicit_arg
+                    option_tuples.append(tup)
+                    break
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join(opt_str for _, opt_str, _ in option_tuples)
+            tup = arg_string, options
+            self.error(_('ambiguous option: %s could match %s') % tup)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        if self._negative_number_matcher.match(arg_string):
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_prefix_tuples(self, option_string):
+        result = []
+        
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            tup = option_prefix, explicit_arg
+            result.append(tup)
+
+        # option strings starting with a single prefix character are
+        # split at all indices
+        else:
+            for first_index, char in enumerate(option_string):
+                if char not in self.prefix_chars:
+                    break
+            for i in xrange(len(option_string), first_index, -1):
+                tup = option_string[:i], option_string[i:] or None
+                result.append(tup)
+
+        # return the collected prefix tuples
+        return result                
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+        
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs is PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers            
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed        
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER args, strip out '--'
+        if action.nargs is not PARSER:
+            arg_strings = [s for s in arg_strings if s != '--']
+        
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, basestring):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+        
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs is PARSER:
+            value = list(self._get_value(action, v) for v in arg_strings)
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = list(self._get_value(action, v) for v in arg_strings)
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value            
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+        
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # TypeErrors or ValueErrors indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = _('invalid %s value: %r')
+            raise ArgumentError(action, msg % (name, arg_string))
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            tup = value, ', '.join(map(repr, action.choices))
+            msg = _('invalid choice: %r (choose from %s)') % tup
+            raise ArgumentError(action, msg)
+
+        
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage,
+                            self._get_optional_actions(),
+                            self._get_positional_actions())
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage,
+                            self._get_optional_actions(),
+                            self._get_positional_actions())
+
+        # description        
+        formatter.add_text(self.description)
+
+        # positionals
+        formatter.start_section(_('positional arguments'))
+        formatter.add_arguments(self._positional_actions_list)
+        formatter.end_section()
+
+        # optionals
+        formatter.start_section(_('optional arguments'))
+        formatter.add_arguments(self._optional_actions_list)
+        formatter.end_section()
+
+        # user-defined groups
+        for argument_group in self._argument_groups:
+            formatter.start_section(argument_group.title)
+            formatter.add_text(argument_group.description)
+            formatter.add_arguments(argument_group._positional_actions_list)
+            formatter.add_arguments(argument_group._optional_actions_list)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+
+    def print_usage(self, file=None):
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+
+    # ===============
+    # Exiting methods
+    # ===============
+
+    def exit(self, status=0, message=None):
+        if message:
+            _sys.stderr.write(message)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+        
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        self.exit(2, _('%s: error: %s\n') % (self.prog, message))

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -0,0 +1,155 b''
+# encoding: utf-8
+
+"""This file contains unittests for the frontendbase module."""
+
+__docformat__ = "restructuredtext en"
+
+#---------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                         
+#                                                                           
+#  Distributed under the terms of the BSD License.  The full license is in  
+#  the file COPYING, distributed as part of this software.                  
+#---------------------------------------------------------------------------
+                                                                            
+#---------------------------------------------------------------------------
+# Imports                                                                   
+#---------------------------------------------------------------------------
+
+import unittest
+
+try:
+    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+    from IPython.frontend import frontendbase 
+    from IPython.kernel.engineservice import EngineService
+except ImportError:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+from IPython.testing.decorators import skip
+
+class FrontEndCallbackChecker(AsyncFrontEndBase):
+    """FrontEndBase subclass for checking callbacks"""
+    def __init__(self, engine=None, history=None):
+        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
+                                                    history=history)
+        self.updateCalled = False
+        self.renderResultCalled = False
+        self.renderErrorCalled = False
+   
+    def update_cell_prompt(self, result, blockID=None):
+        self.updateCalled = True
+        return result
+   
+    def render_result(self, result):
+        self.renderResultCalled = True
+        return result
+    
+    
+    def render_error(self, failure):
+        self.renderErrorCalled = True
+        return failure
+    
+
+
+    
+class TestAsyncFrontendBase(unittest.TestCase):
+    def setUp(self):
+        """Setup the EngineService and FrontEndBase"""
+        
+        self.fb = FrontEndCallbackChecker(engine=EngineService())
+    
+    def test_implements_IFrontEnd(self):
+        assert(frontendbase.IFrontEnd.implementedBy(
+                                    AsyncFrontEndBase))
+    
+    def test_is_complete_returns_False_for_incomplete_block(self):
+        """"""
+        
+        block = """def test(a):"""
+        
+        assert(self.fb.is_complete(block) == False)
+    
+    def test_is_complete_returns_True_for_complete_block(self):
+        """"""
+        
+        block = """def test(a): pass"""
+            
+        assert(self.fb.is_complete(block))
+        
+        block = """a=3"""
+        
+        assert(self.fb.is_complete(block))
+    
+    def test_blockID_added_to_result(self):
+        block = """3+3"""
+        
+        d = self.fb.execute(block, blockID='TEST_ID')
+        
+        d.addCallback(self.checkBlockID, expected='TEST_ID')
+    
+    def test_blockID_added_to_failure(self):
+        block = "raise Exception()"
+        
+        d = self.fb.execute(block,blockID='TEST_ID')
+        d.addErrback(self.checkFailureID, expected='TEST_ID')
+    
+    def checkBlockID(self, result, expected=""):
+        assert(result['blockID'] == expected)
+    
+    
+    def checkFailureID(self, failure, expected=""):
+        assert(failure.blockID == expected)
+    
+    
+    def test_callbacks_added_to_execute(self):
+        """test that
+        update_cell_prompt
+        render_result
+        
+        are added to execute request
+        """
+        
+        d = self.fb.execute("10+10")
+        d.addCallback(self.checkCallbacks)
+    
+    def checkCallbacks(self, result):
+        assert(self.fb.updateCalled)
+        assert(self.fb.renderResultCalled)
+    
+    @skip("This test fails and lead to an unhandled error in a Deferred.")
+    def test_error_callback_added_to_execute(self):
+        """test that render_error called on execution error"""
+        
+        d = self.fb.execute("raise Exception()")
+        d.addCallback(self.checkRenderError)
+    
+    def checkRenderError(self, result):
+        assert(self.fb.renderErrorCalled)
+    
+    def test_history_returns_expected_block(self):
+        """Make sure history browsing doesn't fail"""
+        
+        blocks = ["a=1","a=2","a=3"]
+        for b in blocks:
+            d = self.fb.execute(b)
+        
+        # d is now the deferred for the last executed block
+        d.addCallback(self.historyTests, blocks)
+        
+    
+    def historyTests(self, result, blocks):
+        """historyTests"""
+        
+        assert(len(blocks) >= 3)
+        assert(self.fb.get_history_previous("") == blocks[-2])
+        assert(self.fb.get_history_previous("") == blocks[-3])
+        assert(self.fb.get_history_next() == blocks[-2])
+    
+    
+    def test_history_returns_none_at_startup(self):
+        """test_history_returns_none_at_startup"""
+        
+        assert(self.fb.get_history_previous("")==None)
+        assert(self.fb.get_history_next()==None)
+    
+    

IPython/kernel/core/tests/test_interpreter.py

@@ -0,0 +1,26 b''
+# encoding: utf-8
+
+"""This file contains unittests for the interpreter.py module."""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                           
+#                                                                             
+#  Distributed under the terms of the BSD License.  The full license is in    
+#  the file COPYING, distributed as part of this software.                    
+#-----------------------------------------------------------------------------
+                                                                              
+#-----------------------------------------------------------------------------
+# Imports                                                                     
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.core.interpreter import Interpreter
+
+def test_unicode():
+    """ Test unicode handling with the interpreter.
+    """
+    i = Interpreter()
+    i.execute_python(u'print "ù"')
+    i.execute_python('print "ù"')
+

IPython/testing/iptest.py

@@ -0,0 +1,53 b''
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""IPython Test Suite Runner.
+"""
+
+import sys
+import warnings
+
+from nose.core import TestProgram
+import nose.plugins.builtin
+
+from IPython.testing.plugin.ipdoctest import IPythonDoctest
+
+def main():
+    """Run the IPython test suite.
+    """
+
+    warnings.filterwarnings('ignore', 
+        'This will be removed soon.  Use IPython.testing.util instead')
+
+    
+    # construct list of plugins, omitting the existing doctest plugin
+    plugins = [IPythonDoctest()]
+    for p in nose.plugins.builtin.plugins:
+        plug = p()
+        if plug.name == 'doctest':
+            continue
+
+        #print 'adding plugin:',plug.name  # dbg
+        plugins.append(plug)
+
+    argv = sys.argv + ['--doctest-tests','--doctest-extension=txt',
+                       '--detailed-errors',
+                       
+                       # We add --exe because of setuptools' imbecility (it
+                       # blindly does chmod +x on ALL files).  Nose does the
+                       # right thing and it tries to avoid executables,
+                       # setuptools unfortunately forces our hand here.  This
+                       # has been discussed on the distutils list and the
+                       # setuptools devs refuse to fix this problem!
+                       '--exe',
+                       ]
+
+    has_ip = False
+    for arg in sys.argv:
+        if 'IPython' in arg:
+            has_ip = True
+            break
+
+    if not has_ip:
+        argv.append('IPython')
+
+    TestProgram(argv=argv,plugins=plugins)

IPython/testing/tests/test_decorators.py

@@ -0,0 +1,161 b''
+"""Tests for the decorators we've created for IPython.
+"""
+
+# Module imports
+# Std lib
+import inspect
+import sys
+
+# Third party
+import nose.tools as nt
+
+# Our own
+from IPython.testing import decorators as dec
+
+
+#-----------------------------------------------------------------------------
+# Utilities
+
+# Note: copied from OInspect, kept here so the testing stuff doesn't create
+# circular dependencies and is easier to reuse.
+def getargspec(obj):
+    """Get the names and default values of a function's arguments.
+
+    A tuple of four things is returned: (args, varargs, varkw, defaults).
+    'args' is a list of the argument names (it may contain nested lists).
+    'varargs' and 'varkw' are the names of the * and ** arguments or None.
+    'defaults' is an n-tuple of the default values of the last n arguments.
+
+    Modified version of inspect.getargspec from the Python Standard
+    Library."""
+
+    if inspect.isfunction(obj):
+        func_obj = obj
+    elif inspect.ismethod(obj):
+        func_obj = obj.im_func
+    else:
+        raise TypeError, 'arg is not a Python function'
+    args, varargs, varkw = inspect.getargs(func_obj.func_code)
+    return args, varargs, varkw, func_obj.func_defaults
+
+#-----------------------------------------------------------------------------
+# Testing functions
+
[e-mail]
+def test_deliberately_broken():
+    """A deliberately broken test - we want to skip this one."""
+    1/0
+
[e-mail]('foo')
+def test_deliberately_broken2():
+    """Another deliberately broken test - we want to skip this one."""
+    1/0
+
+
+# Verify that we can correctly skip the doctest for a function at will, but
+# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
+def doctest_bad(x,y=1,**k):
+    """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    print 'x:',x
+    print 'y:',y
+    print 'k:',k
+
+
+def call_doctest_bad():
+    """Check that we can still call the decorated functions.
+    
+    >>> doctest_bad(3,y=4)
+    x: 3
+    y: 4
+    k: {}
+    """
+    pass
+
+
+def test_skip_dt_decorator():
+    """Doctest-skipping decorator should preserve the docstring.
+    """
+    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
+    check = """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    # Fetch the docstring from doctest_bad after decoration.
+    val = doctest_bad.__doc__
+    
+    assert check==val,"doctest_bad docstrings don't match"
+
+# Doctest skipping should work for class methods too
+class foo(object):
+    """Foo
+
+    Example:
+
+    >>> 1+1
+    2
+    """
+
+    @dec.skip_doctest
+    def __init__(self,x):
+        """Make a foo.
+
+        Example:
+
+        >>> f = foo(3)
+        junk
+        """
+        print 'Making a foo.'
+        self.x = x
+        
+    @dec.skip_doctest
+    def bar(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        >>> f.bar(0)
+        boom!
+        >>> 1/0
+        bam!
+        """
+        return 1/y
+
+    def baz(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        Making a foo.
+        >>> f.baz(3)
+        True
+        """
+        return self.x==y
+
+
+
+def test_skip_dt_decorator2():
+    """Doctest-skipping decorator should preserve function signature.
+    """
+    # Hardcoded correct answer
+    dtargs = (['x', 'y'], None, 'k', (1,))
+    # Introspect out the value
+    dtargsr = getargspec(doctest_bad)
+    assert dtargsr==dtargs, \
+           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
+
+
[e-mail]_linux
+def test_linux():
+    nt.assert_not_equals(sys.platform,'linux2',"This test can't run under linux")
+
[e-mail]_win32
+def test_win32():
+    nt.assert_not_equals(sys.platform,'win32',"This test can't run under windows")
+
[e-mail]_osx
+def test_osx():
+    nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")

IPython/tests/test_genutils.py

@@ -0,0 +1,32 b''
+# encoding: utf-8
+
+"""Tests for genutils.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython import genutils
+
+
+def test_get_home_dir():
+    """Make sure we can get the home directory."""
+    home_dir = genutils.get_home_dir()
+
+def test_get_ipython_dir():
+    """Make sure we can get the ipython directory."""
+    ipdir = genutils.get_ipython_dir()
+
+def test_get_security_dir():
+    """Make sure we can get the ipython/security directory."""
+    sdir = genutils.get_security_dir()
+    
  No newline at end of file

IPython/tests/test_magic.py

@@ -0,0 +1,21 b''
+""" Tests for various magic functions
+
+Needs to be run by nose (to make ipython session available)
+
+"""
+def test_rehashx():
+    # clear up everything
+    _ip.IP.alias_table.clear()
+    del _ip.db['syscmdlist']
+    
+    _ip.magic('rehashx')
+    # Practically ALL ipython development systems will have more than 10 aliases
+
+    assert len(_ip.IP.alias_table) > 10
+    for key, val in _ip.IP.alias_table.items():
+        # we must strip dots from alias names
+        assert '.' not in key
+
+    # rehashx must fill up syscmdlist
+    scoms = _ip.db['syscmdlist']
+    assert len(scoms) > 10

docs/examples/core/ipython-sh.desktop

@@ -0,0 +1,22 b''
+# If you want ipython to appear in a linux app launcher ("start menu"), install this by doing:
+# sudo desktop-file-install ipython-sh.desktop
+
+[Desktop Entry]
+Comment=Perform shell-like tasks in interactive ipython session
+Exec=ipython -p sh
+GenericName[en_US]=IPython shell mode
+GenericName=IPython shell mode
+Icon=gnome-netstatus-idle
+MimeType=
+Name[en_US]=ipython-sh
+Name=ipython-sh
+Path=
+Categories=Development;Utility;
+StartupNotify=false
+Terminal=true
+TerminalOptions=
+Type=Application
+X-DBUS-ServiceName=
+X-DBUS-StartupType=none
+X-KDE-SubstituteUID=false
+X-KDE-Username=

docs/examples/core/ipython.desktop

@@ -0,0 +1,22 b''
+# If you want ipython to appear in a linux app launcher ("start menu"), install this by doing:
+# sudo desktop-file-install ipython.desktop
+
+[Desktop Entry]
+Comment=Enhanced interactive Python shell
+Exec=ipython
+GenericName[en_US]=IPython
+GenericName=IPython
+Icon=gnome-netstatus-idle
+MimeType=
+Name[en_US]=ipython
+Name=ipython
+Path=
+Categories=Development;Utility;
+StartupNotify=false
+Terminal=true
+TerminalOptions=
+Type=Application
+X-DBUS-ServiceName=
+X-DBUS-StartupType=none
+X-KDE-SubstituteUID=false
+X-KDE-Username=

docs/source/attic/parallel_task_old.txt

@@ -0,0 +1,240 b''
+.. _paralleltask:
+
+==========================
+The IPython task interface
+==========================
+
+.. contents::
+
+The ``Task`` interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the ``MultiEngine`` interface, in the ``Task`` interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful. Best of all the user can use both of these interfaces at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the ``Task`` interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign Tasks to engines explicitly.
+
+Starting the IPython controller and engines
+===========================================
+
+To follow along with this tutorial, the user will need to start the IPython
+controller and four IPython engines. The simplest way of doing this is to
+use the ``ipcluster`` command::
+
+	$ ipcluster -n 4
+	
+For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+
+The magic here is that this single controller and set of engines is running both the MultiEngine and ``Task`` interfaces simultaneously.
+
+QuickStart Task Farming
+=======================
+
+First, a quick example of how to start running the most basic Tasks.
+The first step is to import the IPython ``client`` module and then create a ``TaskClient`` instance::
+    
+    In [1]: from IPython.kernel import client
+    
+    In [2]: tc = client.TaskClient()
+
+Then the user wrap the commands the user want to run in Tasks::
+
+    In [3]: tasklist = []
+    In [4]: for n in range(1000):
+        ...     tasklist.append(client.Task("a = %i"%n, pull="a"))
+
+The first argument of the ``Task`` constructor is a string, the command to be executed.  The most important optional keyword argument is ``pull``, which can be a string or list of strings, and it specifies the variable names to be saved as results of the ``Task``.
+
+Next, the user need to submit the Tasks to the ``TaskController`` with the ``TaskClient``::
+
+    In [5]: taskids = [ tc.run(t) for t in tasklist ]
+
+This will give the user a list of the TaskIDs used by the controller to keep track of the Tasks and their results.  Now at some point the user are going to want to get those results back.  The ``barrier`` method allows the user to wait for the Tasks to finish running::
+
+    In [6]: tc.barrier(taskids)
+
+This command will block until all the Tasks in ``taskids`` have finished.  Now, the user probably want to look at the user's results::
+
+    In [7]: task_results = [ tc.get_task_result(taskid) for taskid in taskids ]
+
+Now the user have a list of ``TaskResult`` objects, which have the actual result as a dictionary, but also keep track of some useful metadata about the ``Task``::
+
+    In [8]: tr = ``Task``_results[73]
+    
+    In [9]: tr
+    Out[9]: ``TaskResult``[ID:73]:{'a':73}
+    
+    In [10]: tr.engineid
+    Out[10]: 1
+    
+    In [11]: tr.submitted, tr.completed, tr.duration
+    Out[11]: ("2008/03/08 03:41:42", "2008/03/08 03:41:44", 2.12345)
+
+The actual results are stored in a dictionary, ``tr.results``, and a namespace object ``tr.ns`` which accesses the result keys by attribute::
+
+    In [12]: tr.results['a']
+    Out[12]: 73
+    
+    In [13]: tr.ns.a
+    Out[13]: 73
+
+That should cover the basics of running simple Tasks.  There are several more powerful things the user can do with Tasks covered later.  The most useful probably being using a ``MutiEngineClient`` interface to initialize all the engines with the import dependencies necessary to run the user's Tasks.
+
+There are many options for running and managing Tasks. The best way to learn further about the ``Task`` interface is to study the examples in ``docs/examples``. If the user do so and learn a lots about this interface, we encourage the user to expand this documentation about the ``Task`` system.
+
+Overview of the Task System
+===========================
+
+The user's view of the ``Task`` system has three basic objects:  The ``TaskClient``, the ``Task``, and the ``TaskResult``.  The names of these three objects well indicate their role.
+
+The ``TaskClient`` is the user's ``Task`` farming connection to the IPython cluster.  Unlike the ``MultiEngineClient``, the ``TaskControler`` handles all the scheduling and distribution of work, so the ``TaskClient`` has no notion of engines, it just submits Tasks and requests their results.  The Tasks are described as ``Task`` objects, and their results are wrapped in ``TaskResult`` objects.  Thus, there are very few necessary methods for the user to manage.
+
+Inside the task system is a Scheduler object, which assigns tasks to workers.  The default scheduler is a simple FIFO queue.  Subclassing the Scheduler should be easy, just implementing your own priority system.
+
+The TaskClient
+==============
+
+The ``TaskClient`` is the object the user use to connect to the ``Controller`` that is managing the user's Tasks.  It is the analog of the ``MultiEngineClient`` for the standard IPython multiplexing interface.  As with all client interfaces, the first step is to import the IPython Client Module::
+
+    In [1]: from IPython.kernel import client
+
+Just as with the ``MultiEngineClient``, the user create the ``TaskClient`` with a tuple, containing the ip-address and port of the ``Controller``.  the ``client`` module conveniently has the default address of the ``Task`` interface of the controller.  Creating a default ``TaskClient`` object would be done with this::
+
+    In [2]: tc = client.TaskClient(client.default_task_address)
+
+or, if the user want to specify a non default location of the ``Controller``, the user can specify explicitly::
+
+    In [3]: tc = client.TaskClient(("192.168.1.1", 10113))
+
+As discussed earlier, the ``TaskClient`` only has a few basic methods.
+
+ * ``tc.run(task)``
+    ``run`` is the method by which the user submits Tasks.  It takes exactly one argument, a ``Task`` object.  All the advanced control of ``Task`` behavior is handled by properties of the ``Task`` object, rather than the submission command, so they will be discussed later in the `Task`_ section.  ``run`` returns an integer, the ``Task``ID by which the ``Task`` and its results can be tracked and retrieved::
+    
+        In [4]: ``Task``ID = tc.run(``Task``)
+    
+ * ``tc.get_task_result(taskid, block=``False``)``
+    ``get_task_result`` is the method by which results are retrieved.  It takes a single integer argument, the ``Task``ID`` of the result the user wish to retrieve.  ``get_task_result`` also takes a keyword argument ``block``.  ``block`` specifies whether the user actually want to wait for the result.  If ``block`` is false, as it is by default, ``get_task_result`` will return immediately.  If the ``Task`` has completed, it will return the ``TaskResult`` object for that ``Task``.  But if the ``Task`` has not completed, it will return ``None``.  If the user specify ``block=``True``, then ``get_task_result`` will wait for the ``Task`` to complete, and always return the ``TaskResult`` for the requested ``Task``.
+ * ``tc.barrier(taskid(s))``
+    ``barrier`` is a synchronization method.  It takes exactly one argument, a ``Task``ID or list of taskIDs.  ``barrier`` will block until all the specified Tasks have completed.  In practice, a barrier is often called between the ``Task`` submission section of the code and the result gathering section::
+        
+        In [5]: taskIDs = [ tc.run(``Task``) for ``Task`` in myTasks ]
+        
+        In [6]: tc.get_task_result(taskIDs[-1]) is None
+        Out[6]: ``True``
+        
+        In [7]: tc.barrier(``Task``ID)
+        
+        In [8]: results = [ tc.get_task_result(tid) for tid in taskIDs ]
+        
+ * ``tc.queue_status(verbose=``False``)``
+    ``queue_status`` is a method for querying the state of the ``TaskControler``.  ``queue_status`` returns a dict of the form::
+        
+        {'scheduled': Tasks that have been submitted but yet run
+         'pending'  : Tasks that are currently running
+         'succeeded': Tasks that have completed successfully
+         'failed'   : Tasks that have finished with a failure
+        }
+    
+    if @verbose is not specified (or is ``False``), then the values of the dict are integers - the number of Tasks in each state.  if @verbose is ``True``, then each element in the dict is a list of the taskIDs in that state::
+    
+        In [8]: tc.queue_status()
+        Out[8]: {'scheduled': 4,
+                'pending'  : 2,
+                'succeeded': 5,
+                'failed'   : 1
+                }
+        
+        In [9]: tc.queue_status(verbose=True)
+        Out[9]: {'scheduled': [8,9,10,11],
+                'pending'  : [6,7],
+                'succeeded': [0,1,2,4,5],
+                'failed'   : [3]
+                }
+        
+ * ``tc.abort(taskid)``
+    ``abort`` allows the user to abort Tasks that have already been submitted.  ``abort`` will always return immediately.  If the ``Task`` has completed, ``abort`` will raise an ``IndexError ``Task`` Already Completed``.  An obvious case for ``abort`` would be where the user submits a long-running ``Task`` with a number of retries (see ``Task``_ section for how to specify retries) in an interactive session, but realizes there has been a typo.  The user can then abort the ``Task``, preventing certain failures from cluttering up the queue.  It can also be used for parallel search-type problems, where only one ``Task`` will give the solution, so once the user find the solution, the user would want to abort all remaining Tasks to prevent wasted work.
+ * ``tc.spin()``
+    ``spin`` simply triggers the scheduler in the ``TaskControler``.  Under most normal circumstances, this will do nothing.  The primary known usage case involves the ``Task`` dependency (see `Dependencies`_).  The dependency is a function of an Engine's ``properties``, but changing the ``properties`` via the ``MutliEngineClient`` does not trigger a reschedule event.  The main example case for this requires the following event sequence:
+     * ``engine`` is available, ``Task`` is submitted, but ``engine`` does not have ``Task``'s dependencies.
+     * ``engine`` gets necessary dependencies while no new Tasks are submitted or completed.
+     * now ``engine`` can run ``Task``, but a ``Task`` event is required for the ``TaskControler`` to try scheduling ``Task`` again.
+     
+    ``spin`` is just an empty ping method to ensure that the Controller has scheduled all available Tasks, and should not be needed under most normal circumstances.
+
+That covers the ``TaskClient``, a simple interface to the cluster.  With this, the user can submit jobs (and abort if necessary), request their results, synchronize on arbitrary subsets of jobs.
+
+.. _task: The Task Object
+
+The Task Object
+===============
+
+The ``Task`` is the basic object for describing a job.  It can be used in a very simple manner, where the user just specifies a command string to be executed as the ``Task``.  The usage of this first argument is exactly the same as the ``execute`` method of the ``MultiEngine`` (in fact, ``execute`` is called to run the code)::
+
+    In [1]: t = client.Task("a = str(id)")
+    
+This ``Task`` would run, and store the string representation of the ``id`` element in ``a`` in each worker's namespace, but it is fairly useless because the user does not know anything about the state of the ``worker`` on which it ran at the time of retrieving results.  It is important that each ``Task`` not expect the state of the ``worker`` to persist after the ``Task`` is completed.  
+There are many different situations for using ``Task`` Farming, and the ``Task`` object has many attributes for use in customizing the ``Task`` behavior.  All of a ``Task``'s attributes may be specified in the constructor, through keyword arguments, or after ``Task`` construction through attribute assignment.
+
+Data Attributes
+***************
+It is likely that the user may want to move data around before or after executing the ``Task``.  We provide methods of sending data to initialize the worker's namespace, and specifying what data to bring back as the ``Task``'s results.
+
+ * pull = []
+    The obvious case is as above, where ``t`` would execute and store the result of ``myfunc`` in ``a``, it is likely that the user would want to bring ``a`` back to their namespace.  This is done through the ``pull`` attribute.  ``pull`` can be a string or list of strings, and it specifies the names of variables to be retrieved.  The ``TaskResult`` object retrieved by ``get_task_result`` will have a dictionary of keys and values, and the ``Task``'s ``pull`` attribute determines what goes into it::
+        
+        In [2]: t = client.Task("a = str(id)", pull = "a")
+        
+        In [3]: t = client.Task("a = str(id)", pull = ["a", "id"])
+        
+ * push = {}
+    A user might also want to initialize some data into the namespace before the code part of the ``Task`` is run.  Enter ``push``.  ``push`` is a dictionary of key/value pairs to be loaded from the user's namespace into the worker's immediately before execution::
+
+        In [4]: t = client.Task("a = f(submitted)", push=dict(submitted=time.time()), pull="a")
+
+push and pull result directly in calling an ``engine``'s ``push`` and ``pull`` methods before and after ``Task`` execution respectively, and thus their api is the same.
+
+Namespace Cleaning
+******************
+When a user is running a large number of Tasks, it is likely that the namespace of the worker's could become cluttered.  Some Tasks might be sensitive to clutter, while others might be known to cause namespace pollution.  For these reasons, Tasks have two boolean attributes for cleaning up the namespace.
+
+ * ``clear_after``
+    if clear_after is specified ``True``, the worker on which the ``Task`` was run will be reset (via ``engine.reset``) upon completion of the ``Task``.  This can be useful for both Tasks that produce clutter or Tasks whose intermediate data one might wish to be kept private::
+    
+        In [5]: t = client.Task("a = range(1e10)", pull = "a",clear_after=True)
+        
+    
+ * ``clear_before``
+    as one might guess, clear_before is identical to ``clear_after``, but it takes place before the ``Task`` is run.  This ensures that the ``Task`` runs on a fresh worker::
+
+        In [6]: t = client.Task("a = globals()", pull = "a",clear_before=True)
+
+Of course, a user can both at the same time, ensuring that all workers are clear except when they are currently running a job.  Both of these default to ``False``.
+
+Fault Tolerance
+***************
+It is possible that Tasks might fail, and there are a variety of reasons this could happen.  One might be that the worker it was running on disconnected, and there was nothing wrong with the ``Task`` itself.  With the fault tolerance attributes of the ``Task``, the user can specify how many times to resubmit the ``Task``, and what to do if it never succeeds.
+
+ * ``retries``
+    ``retries`` is an integer, specifying the number of times a ``Task`` is to be retried.  It defaults to zero.  It is often a good idea for this number to be 1 or 2, to protect the ``Task`` from disconnecting engines, but not a large number.  If a ``Task`` is failing 100 times, there is probably something wrong with the ``Task``.  The canonical bad example:
+    
+        In [7]: t = client.Task("os.kill(os.getpid(), 9)", retries=99)
+        
+    This would actually take down 100 workers.
+    
+ * ``recovery_task``
+    ``recovery_task`` is another ``Task`` object, to be run in the event of the original ``Task`` still failing after running out of retries.  Since ``recovery_task`` is another ``Task`` object, it can have its own ``recovery_task``.  The chain of Tasks is limitless, except loops are not allowed (that would be bad!).
+
+Dependencies
+************
+Dependencies are the most powerful part of the ``Task`` farming system, because it allows the user to do some classification of the workers, and guide the ``Task`` distribution without meddling with the controller directly.  It makes use of two objects - the ``Task``'s ``depend`` attribute, and the engine's ``properties``.  See the `MultiEngine`_ reference for how to use engine properties.  The engine properties api exists for extending IPython, allowing conditional execution and new controllers that make decisions based on properties of its engines.  Currently the ``Task`` dependency is the only internal use of the properties api.
+
+.. _MultiEngine: ./parallel_multiengine
+
+The ``depend`` attribute of a ``Task`` must be a function of exactly one argument, the worker's properties dictionary, and it should return ``True`` if the ``Task`` should be allowed to run on the worker and ``False`` if not.  The usage in the controller is fault tolerant, so exceptions raised by ``Task.depend`` will be ignored and functionally equivalent to always returning ``False``.  Tasks`` with invalid ``depend`` functions will never be assigned to a worker::
+
+    In [8]: def dep(properties):
+        ...     return properties["RAM"] > 2**32 # have at least 4GB
+    In [9]: t = client.Task("a = bigfunc()", depend=dep)
+    
+It is important to note that assignment of values to the properties dict is done entirely by the user, either locally (in the engine) using the EngineAPI, or remotely, through the ``MultiEngineClient``'s get/set_properties methods.
+
+
+
+
+
+    

docs/source/development/config_blueprint.txt

@@ -0,0 +1,33 b''
+=========================================
+Notes on the IPython configuration system
+=========================================
+
+This document has some random notes on the configuration system.
+
+To start, an IPython process needs:
+
+* Configuration files
+* Command line options
+* Additional files (FURL files, extra scripts, etc.)
+
+It feeds these things into the core logic of the process, and as output,
+produces:
+
+* Log files
+* Security files
+
+There are a number of things that complicate this:
+
+* A process may need to be started on a different host that doesn't have
+  any of the config files or additional files.  Those files need to be
+  moved over and put in a staging area.  The process then needs to be told
+  about them.
+* The location of the output files should somehow be set by config files or
+  command line options.
+* Our config files are very hierarchical, but command line options are flat,
+  making it difficult to relate command line options to config files.
+* Some processes (like ipcluster and the daemons) have to manage the input and
+  output files for multiple different subprocesses, each possibly on a
+  different host.  Ahhhh!
+* Our configurations are not singletons.  A given user will likely have
+  many different configurations for different clusters.

docs/source/install/install.txt

@@ -0,0 +1,217 b''
+Overview
+========
+
+This document describes the steps required to install IPython.  IPython is organized into a number of subpackages, each of which has its own dependencies.  All of the subpackages come with IPython, so you don't need to download and install them separately.  However, to use a given subpackage, you will need to install all of its dependencies.  
+
+
+Please let us know if you have problems installing IPython or any of its
+dependencies. IPython requires Python version 2.4 or greater.  We have not tested
+IPython with the upcoming 2.6 or 3.0 versions.
+
+.. warning::
+
+	IPython will not work with Python 2.3 or below.
+
+Some of the installation approaches use the :mod:`setuptools` package and its :command:`easy_install` command line program.  In many scenarios, this provides the most simple method of installing IPython and its dependencies.  It is not required though.  More information about :mod:`setuptools` can be found on its website.
+
+More general information about installing Python packages can be found in Python's documentation at http://www.python.org/doc/.
+
+Quickstart
+==========
+
+If you have :mod:`setuptools` installed and you are on OS X or Linux (not Windows), the following will download and install IPython *and* the main optional dependencies::
+
+    $ easy_install ipython[kernel,security,test]
+
+This will get Twisted, zope.interface and Foolscap, which are needed for IPython's parallel computing features as well as the nose package, which will enable you to run IPython's test suite.  To run IPython's test suite, use the :command:`iptest` command::
+
+    $ iptest
+
+Read on for more specific details and instructions for Windows.
+
+Installing IPython itself
+=========================
+
+Given a properly built Python, the basic interactive IPython shell will work with no external dependencies.  However, some Python distributions (particularly on Windows and OS X), don't come with a working :mod:`readline` module.  The IPython shell will work without :mod:`readline`, but will lack many features that users depend on, such as tab completion and command line editing.  See below for details of how to make sure you have a working :mod:`readline`.
+
+Installation using easy_install
+-------------------------------
+
+If you have :mod:`setuptools` installed, the easiest way of getting IPython is to simple use :command:`easy_install`::
+
+	$ easy_install ipython
+
+That's it.
+
+Installation from source
+------------------------
+
+If you don't want to use :command:`easy_install`, or don't have it installed, just grab the latest stable build of IPython from `here <http://ipython.scipy.org/dist/>`_.  Then do the following::
+
+	$ tar -xzf ipython.tar.gz
+	$ cd ipython
+	$ python setup.py install
+
+If you are installing to a location (like ``/usr/local``) that requires higher permissions, you may need to run the last command with :command:`sudo`.
+
+Windows
+-------
+
+There are a few caveats for Windows users.  The main issue is that a basic ``python setup.py install`` approach won't create ``.bat`` file or Start Menu shortcuts, which most users want.  To get an installation with these, there are two choices:
+
+1.  Install using :command:`easy_install`.
+
+2.  Install using our binary ``.exe`` Windows installer, which can be found at `here <http://ipython.scipy.org/dist/>`_
+
+3.  Install from source, but using :mod:`setuptools` (``python setupegg.py install``).
+
+Installing the development version
+----------------------------------
+
+It is also possible to install the development version of IPython from our `Bazaar <http://bazaar-vcs.org/>`_ source code
+repository.  To do this you will need to have Bazaar installed on your system.  Then just do::
+
+	$ bzr branch lp:ipython
+	$ cd ipython
+	$ python setup.py install
+
+Again, this last step on Windows won't create ``.bat`` files or Start Menu shortcuts, so you will have to use one of the other approaches listed above.
+
+Some users want to be able to follow the development branch as it changes.  If you have :mod:`setuptools` installed, this is easy. Simply replace the last step by::
+
+	$ python setupegg.py develop
+
+This creates links in the right places and installs the command line script to the appropriate places.  Then, if you want to update your IPython at any time, just do::
+
+	$ bzr pull
+
+Basic optional dependencies
+===========================
+
+There are a number of basic optional dependencies that most users will want to get.  These are:
+
+* readline (for command line editing, tab completion, etc.)
+* nose (to run the IPython test suite)
+* pexpect (to use things like irunner)
+
+If you are comfortable installing these things yourself, have at it, otherwise read on for more details.
+
+readline
+--------
+
+In principle, all Python distributions should come with a working :mod:`readline` module.  But, reality is not quite that simple.  There are two common situations where you won't have a working :mod:`readline` module:
+
+* If you are using the built-in Python on Mac OS X.
+
+* If you are running Windows, which doesn't have a :mod:`readline` module.
+
+On OS X, the built-in Python doesn't not have :mod:`readline` because of license issues.  Starting with OS X 10.5 (Leopard), Apple's built-in Python has a BSD-licensed not-quite-compatible readline replacement. As of IPython 0.9, many of the issues related to the differences between readline and libedit have been resolved.  For many users, libedit may be sufficient.
+
+Most users on OS X will want to get the full :mod:`readline` module.  To get a working :mod:`readline` module, just do (with :mod:`setuptools` installed)::
+
+	$ easy_install readline
+
+.. note:
+
+    Other Python distributions on OS X (such as fink, MacPorts and the 
+    official python.org binaries) already have readline installed so 
+    you don't have to do this step.
+
+If needed, the readline egg can be build and installed from source (see the wiki page at http://ipython.scipy.org/moin/InstallationOSXLeopard).
+
+On Windows, you will need the PyReadline module. PyReadline is a separate,
+Windows only implementation of readline that uses native Windows calls through
+:mod:`ctypes`. The easiest way of installing PyReadline is you use the binary
+installer available `here <http://ipython.scipy.org/dist/>`_. The
+:mod:`ctypes` module, which comes with Python 2.5 and greater, is required by
+PyReadline. It is available for Python 2.4 at
+http://python.net/crew/theller/ctypes.
+
+nose
+----
+
+To run the IPython test suite you will need the :mod:`nose` package.  Nose provides a great way of sniffing out and running all of the IPython tests.  The simplest way of getting nose, is to use :command:`easy_install`::
+
+	$ easy_install nose
+
+Another way of getting this is to do::
+
+    $ easy_install ipython[test]
+
+For more installation options, see the `nose website <http://somethingaboutorange.com/mrl/projects/nose/>`_.  Once you have nose installed, you can run IPython's test suite using the iptest command::
+
+	$ iptest
+
+
+pexpect
+-------
+
+The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's :command:`irunner` script.  On Unix platforms (including OS X), just do::
+
+	$ easy_install pexpect
+
+Windows users are out of luck as pexpect does not run there.
+
+Dependencies for IPython.kernel (parallel computing)
+====================================================
+
+The IPython kernel provides a nice architecture for parallel computing.  The main focus of this architecture is on interactive parallel computing.  These features require a number of additional packages:
+
+* zope.interface (yep, we use interfaces)
+* Twisted (asynchronous networking framework)
+* Foolscap (a nice, secure network protocol)
+* pyOpenSSL (security for network connections)
+
+On a Unix style platform (including OS X), if you want to use :mod:`setuptools`, you can just do::
+
+    $ easy_install ipython[kernel]    # the first three
+    $ easy_install ipython[security]  # pyOpenSSL
+
+zope.interface and Twisted
+--------------------------
+
+Twisted [Twisted]_ and zope.interface [ZopeInterface]_ are used for networking related things. On Unix
+style platforms (including OS X), the simplest way of getting the these is to
+use :command:`easy_install`::
+
+	$ easy_install zope.interface
+	$ easy_install Twisted
+
+Of course, you can also download the source tarballs from the `Twisted website <twistedmatrix.org>`_ and the `zope.interface page at PyPI <http://pypi.python.org/pypi/zope.interface>`_ and do the usual ``python setup.py install`` if you prefer.
+
+Windows is a bit different.  For zope.interface and Twisted, simply get the latest binary ``.exe`` installer from the Twisted website.  This installer includes both zope.interface and Twisted and should just work.
+
+Foolscap
+--------
+
+Foolscap [Foolscap]_ uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features.
+
+On all platforms a simple::
+
+    $ easy_install foolscap
+
+should work.  You can also download the source tarballs from the `Foolscap website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install`` if you prefer.
+
+pyOpenSSL
+---------
+
+IPython requires an older version of pyOpenSSL [pyOpenSSL]_ (0.6 rather than the current 0.7).  There are a couple of options for getting this:
+
+1.  Most Linux distributions have packages for pyOpenSSL.
+2.  The built-in Python 2.5 on OS X 10.5 already has it installed.
+3.  There are source tarballs on the pyOpenSSL website.  On Unix-like
+    platforms, these can be built using ``python seutp.py install``.
+4.  There is also a binary ``.exe`` Windows installer on the `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_.
+
+Dependencies for IPython.frontend (the IPython GUI)
+===================================================
+
+wxPython
+--------
+
+Starting with IPython 0.9, IPython has a new IPython.frontend package that has a nice wxPython based IPython GUI.  As you would expect, this GUI requires wxPython.  Most Linux distributions have wxPython packages available and the built-in Python on OS X comes with wxPython preinstalled.  For Windows, a binary installer is available on the `wxPython website <http://www.wxpython.org/>`_.
+
+.. [Twisted] Twisted matrix.  http://twistedmatrix.org
+.. [ZopeInterface] http://pypi.python.org/pypi/zope.interface
+.. [Foolscap] Foolscap network protocol.  http://foolscap.lothar.com/trac
+.. [pyOpenSSL] pyOpenSSL.  http://pyopenssl.sourceforge.net
  No newline at end of file

docs/source/parallel/parallel_process.txt

@@ -0,0 +1,251 b''
+.. _parallel_process:
+
+===========================================
+Starting the IPython controller and engines
+===========================================
+
+To use IPython for parallel computing, you need to start one instance of
+the controller and one or more instances of the engine. The controller
+and each engine can run on different machines or on the same machine.
+Because of this, there are many different possibilities.
+
+Broadly speaking, there are two ways of going about starting a controller and engines:
+
+* In an automated manner using the :command:`ipcluster` command.
+* In a more manual way using the :command:`ipcontroller` and 
+  :command:`ipengine` commands.
+
+This document describes both of these methods.  We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases.
+
+General considerations
+======================
+
+Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines.  These things come up no matter which method you use to start your IPython cluster.
+
+Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``.  The following steps are then required:
+
+1. Start the controller on ``host0`` by running :command:`ipcontroller` on
+   ``host0``.
+2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the 
+   controller from ``host0`` to hosts ``host1``-``hostn``.
+3. Start the engines on hosts ``host1``-``hostn`` by running 
+   :command:`ipengine`.  This command has to be told where the FURL file 
+   (:file:`ipcontroller-engine.furl`) is located.
+
+At this point, the controller and engines will be connected. By default, the
+FURL files created by the controller are put into the
+:file:`~/.ipython/security` directory. If the engines share a filesystem with
+the controller, step 2 can be skipped as the engines will automatically look
+at that location.
+
+The final step required required to actually use the running controller from a
+client is to move the FURL files :file:`ipcontroller-mec.furl` and
+:file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will
+be run.  If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically.  Otherwise, the full path to them has to be passed to the client's constructor.
+
+Using :command:`ipcluster`
+==========================
+
+The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations:
+
+1. When the controller and engines are all run on localhost. This is useful
+   for testing or running on a multicore computer.
+2. When engines are started using the :command:`mpirun` command that comes 
+   with most MPI [MPI]_ implementations
+3. When engines are started using the PBS [PBS]_ batch system.
+
+.. note::
+
+    It is also possible for advanced users to add support to
+    :command:`ipcluster` for starting controllers and engines using other
+    methods (like Sun's Grid Engine for example).
+
+.. note::
+
+    Currently :command:`ipcluster` requires that the
+    :file:`~/.ipython/security` directory live on a shared filesystem that is
+    seen by both the controller and engines. If you don't have a shared file
+    system you will need to use :command:`ipcontroller` and
+    :command:`ipengine` directly.
+
+Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller`
+and :command:`ipengine` to perform the steps described above.
+
+Using :command:`ipcluster` in local mode
+----------------------------------------
+
+To start one controller and 4 engines on localhost, just do::
+
+    $ ipcluster local -n 4
+
+To see other command line options for the local mode, do::
+
+    $ ipcluster local -h
+
+Using :command:`ipcluster` in mpirun mode
+-----------------------------------------
+
+The mpirun mode is useful if you:
+
+1. Have MPI installed.
+2. Your systems are configured to use the :command:`mpirun` command to start 
+   processes.
+
+If these are satisfied, you can start an IPython cluster using::
+
+    $ ipcluster mpirun -n 4
+
+This does the following:
+
+1. Starts the IPython controller on current host.
+2. Uses :command:`mpirun` to start 4 engines.
+
+On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`.  However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting.  The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option::
+
+    $ ipcluster mpirun -n 4 --mpi=mpi4py
+
+Unfortunately, even this won't work for some MPI implementations.  If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time.  Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use.  However, this custom Python executable approach will not work with :command:`ipcluster` currently.
+
+Additional command line options for this mode can be found by doing::
+
+    $ ipcluster mpirun -h
+
+More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
+
+
+Using :command:`ipcluster` in PBS mode
+--------------------------------------
+
+The PBS mode uses the Portable Batch System [PBS]_ to start the engines.  To use this mode, you first need to create a PBS script template that will be used to start the engines.  Here is a sample PBS script template:
+
+.. sourcecode:: bash
+
+    #PBS -N ipython
+    #PBS -j oe
+    #PBS -l walltime=00:10:00
+    #PBS -l nodes=${n/4}:ppn=4
+    #PBS -q parallel
+
+    cd $$PBS_O_WORKDIR
+    export PATH=$$HOME/usr/local/bin
+    export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages
+    /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine
+
+There are a few important points about this template:
+
+1. This template will be rendered at runtime using IPython's :mod:`Itpl` 
+   template engine.
+
+2. Instead of putting in the actual number of engines, use the notation
+   ``${n}`` to indicate the number of engines to be started. You can also uses
+   expressions like ``${n/4}`` in the template to indicate the number of 
+   nodes.
+
+3. Because ``$`` is a special character used by the template engine, you must  
+   escape any ``$`` by using ``$$``.  This is important when referring to 
+   environment variables in the template.
+
+4. Any options to :command:`ipengine` should be given in the batch script 
+   template.
+
+5. Depending on the configuration of you system, you may have to set 
+   environment variables in the script template.
+
+Once you have created such a script, save it with a name like :file:`pbs.template`.  Now you are ready to start your job::
+
+    $ ipcluster pbs -n 128 --pbs-script=pbs.template
+
+Additional command line options for this mode can be found by doing::
+
+    $ ipcluster pbs -h
+
+Using the :command:`ipcontroller` and :command:`ipengine` commands
+==================================================================
+
+It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines.  This approach gives you full control over all aspects of the startup process.
+
+Starting the controller and engine on your local machine
+--------------------------------------------------------
+
+To use :command:`ipcontroller` and :command:`ipengine` to start things on your
+local machine, do the following.
+
+First start the controller::
+
+	$ ipcontroller
+	
+Next, start however many instances of the engine you want using (repeatedly) the command::
+
+	$ ipengine
+
+The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython.
+
+.. warning:: 
+	
+	The order of the above operations is very important.  You *must*
+ 	start the controller before the engines, since the engines connect 
+	to the controller as they get started.
+
+.. note::
+
+    On some platforms (OS X), to put the controller and engine into the
+    background you may need to give these commands in the form ``(ipcontroller
+    &)`` and ``(ipengine &)`` (with the parentheses) for them to work
+    properly.
+
+Starting the controller and engines on different hosts
+------------------------------------------------------
+
+When the controller and engines are running on different hosts, things are
+slightly more complicated, but the underlying ideas are the same:
+
+1. Start the controller on a host using :command:`ipcontroller`.
+2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run.
+3. Use :command:`ipengine` on the engine's hosts to start the engines.
+
+The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located.  There are two ways you can do this:
+
+* Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security`
+  directory on the engine's host, where it will be found automatically.
+* Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file``
+  flag.
+
+The ``--furl-file`` flag works like this::
+
+    $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl
+
+.. note::
+    
+    If the controller's and engine's hosts all have a shared file system  
+    (:file:`~./ipython/security` is the same on all of them), then things
+    will just work!
+
+Make FURL files persistent
+---------------------------
+
+At fist glance it may seem that that managing the FURL files is a bit annoying.  Going back to the house and key analogy, copying the FURL around each time you start the controller is like having to make a new key every time you want to unlock the door and enter your house.  As with your house, you want to be able to create the key (or FURL file) once, and then simply use it at any point in the future.
+
+This is possible.  The only thing you have to do is decide what ports the controller will listen on for the engines and clients.  This is done as follows::
+
+    $ ipcontroller -r --client-port=10101 --engine-port=10102
+
+Then, just copy the furl files over the first time and you are set.  You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports.
+
+.. note::
+
+    You may ask the question: what ports does the controller listen on if you
+    don't tell is to use specific ones? The default is to use high random port
+    numbers. We do this for two reasons: i) to increase security through
+    obscurity and ii) to multiple controllers on a given host to start and
+    automatically use different ports.
+
+Log files
+---------
+
+All of the components of IPython have log files associated with them.
+These log files can be extremely useful in debugging problems with
+IPython and can be found in the directory :file:`~/.ipython/log`.  Sending
+the log files to us will often help us to debug any problems.
+
+
+.. [PBS] Portable Batch System.  http://www.openpbs.org/

docs/source/parallel/parallel_security.txt

@@ -0,0 +1,363 b''
+.. _parallelsecurity:
+
+===========================
+Security details of IPython
+===========================
+
+IPython's :mod:`IPython.kernel` package exposes the full power of the Python
+interpreter over a TCP/IP network for the purposes of parallel computing. This
+feature brings up the important question of IPython's security model. This
+document gives details about this model and how it is implemented in IPython's
+architecture.
+
+Processs and network topology
+=============================
+
+To enable parallel computing, IPython has a number of different processes that
+run. These processes are discussed at length in the IPython documentation and
+are summarized here:
+
+* The IPython *engine*.  This process is a full blown Python
+  interpreter in which user code is executed.  Multiple
+  engines are started to make parallel computing possible.
+* The IPython *controller*.  This process manages a set of 
+  engines, maintaining a queue for each and presenting
+  an asynchronous interface to the set of engines.
+* The IPython *client*.  This process is typically an
+  interactive Python process that is used to coordinate the
+  engines to get a parallel computation done.
+
+Collectively, these three processes are called the IPython *kernel*.
+
+These three processes communicate over TCP/IP connections with a well defined
+topology. The IPython controller is the only process that listens on TCP/IP
+sockets. Upon starting, an engine connects to a controller and registers
+itself with the controller. These engine/controller TCP/IP connections persist
+for the lifetime of each engine.
+
+The IPython client also connects to the controller using one or more TCP/IP
+connections. These connections persist for the lifetime of the client only.
+
+A given IPython controller and set of engines typically has a relatively short
+lifetime. Typically this lifetime corresponds to the duration of a single
+parallel simulation performed by a single user. Finally, the controller,
+engines and client processes typically execute with the permissions of that
+same user. More specifically, the controller and engines are *not* executed as
+root or with any other superuser permissions.
+
+Application logic
+=================
+
+When running the IPython kernel to perform a parallel computation, a user
+utilizes the IPython client to send Python commands and data through the
+IPython controller to the IPython engines, where those commands are executed
+and the data processed. The design of IPython ensures that the client is the
+only access point for the capabilities of the engines.  That is, the only way of addressing the engines is through a client.
+
+A user can utilize the client to instruct the IPython engines to execute
+arbitrary Python commands. These Python commands can include calls to the
+system shell, access the filesystem, etc., as required by the user's
+application code. From this perspective, when a user runs an IPython engine on
+a host, that engine has the same capabilities and permissions as the user
+themselves (as if they were logged onto the engine's host with a terminal).
+
+Secure network connections
+==========================
+
+Overview
+--------
+
+All TCP/IP connections between the client and controller as well as the
+engines and controller are fully encrypted and authenticated. This section
+describes the details of the encryption and authentication approached used
+within IPython.
+
+IPython uses the Foolscap network protocol [Foolscap]_ for all communications
+between processes. Thus, the details of IPython's security model are directly
+related to those of Foolscap. Thus, much of the following discussion is
+actually just a discussion of the security that is built in to Foolscap.
+
+Encryption
+----------
+
+For encryption purposes, IPython and Foolscap use the well known Secure Socket
+Layer (SSL) protocol [RFC5246]_. We use the implementation of this protocol
+provided by the OpenSSL project through the pyOpenSSL [pyOpenSSL]_ Python
+bindings to OpenSSL.
+
+Authentication
+--------------
+
+IPython clients and engines must also authenticate themselves with the
+controller. This is handled in a capabilities based security model
+[Capability]_. In this model, the controller creates a strong cryptographic
+key or token that represents each set of capability that the controller
+offers. Any party who has this key and presents it to the controller has full
+access to the corresponding capabilities of the controller. This model is
+analogous to using a physical key to gain access to physical items
+(capabilities) behind a locked door.
+
+For a capabilities based authentication system to prevent unauthorized access,
+two things must be ensured:
+
+* The keys must be cryptographically strong.  Otherwise attackers could gain 
+  access by a simple brute force key guessing attack.
+* The actual keys must be distributed only to authorized parties.
+
+The keys in Foolscap are called Foolscap URL's or FURLs. The following section
+gives details about how these FURLs are created in Foolscap. The IPython
+controller creates a number of FURLs for different purposes:
+
+* One FURL that grants IPython engines access to the controller.  Also
+  implicit in this access is permission to execute code sent by an
+  authenticated IPython client.
+* Two or more FURLs that grant IPython clients access to the controller.
+  Implicit in this access is permission to give the controller's engine code 
+  to execute.
+
+Upon starting, the controller creates these different FURLS and writes them
+files in the user-read-only directory :file:`$HOME/.ipython/security`. Thus, only the
+user who starts the controller has access to the FURLs.
+
+For an IPython client or engine to authenticate with a controller, it must
+present the appropriate FURL to the controller upon connecting. If the
+FURL matches what the controller expects for a given capability, access is
+granted. If not, access is denied. The exchange of FURLs is done after
+encrypted communications channels have been established to prevent attackers
+from capturing them.
+
+.. note::
+
+    The FURL is similar to an unsigned private key in SSH.
+
+Details of the Foolscap handshake
+---------------------------------
+
+In this section we detail the precise security handshake that takes place at
+the beginning of any network connection in IPython. For the purposes of this
+discussion, the SERVER is the IPython controller process and the CLIENT is the
+IPython engine or client process.
+
+Upon starting, all IPython processes do the following:
+
+1. Create a public key x509 certificate (ISO/IEC 9594).
+2. Create a hash of the contents of the certificate using the SHA-1 algorithm.
+   The base-32 encoded version of this hash is saved by the process as its 
+   process id (actually in Foolscap, this is the Tub id, but here refer to
+   it as the process id).
+
+Upon starting, the IPython controller also does the following:
+
+1. Save the x509 certificate to disk in a secure location. The CLIENT
+   certificate is never saved to disk.
+2. Create a FURL for each capability that the controller has.  There are
+   separate capabilities the controller offers for clients and engines.  The
+   FURL is created using:  a) the process id of the SERVER, b) the IP 
+   address and port the SERVER is listening on and c) a 160 bit,    
+   cryptographically secure string that represents the capability (the
+   "capability id").
+3. The FURLs are saved to disk in a secure location on the SERVER's host.
+
+For a CLIENT to be able to connect to the SERVER and access a capability of
+that SERVER, the CLIENT must have knowledge of the FURL for that SERVER's
+capability. This typically requires that the file containing the FURL be
+moved from the SERVER's host to the CLIENT's host. This is done by the end
+user who started the SERVER and wishes to have a CLIENT connect to the SERVER.
+
+When a CLIENT connects to the SERVER, the following handshake protocol takes
+place:
+
+1. The CLIENT tells the SERVER what process (or Tub) id it expects the SERVER
+   to have.
+2. If the SERVER has that process id, it notifies the CLIENT that it will now 
+   enter encrypted mode.  If the SERVER has a different id, the SERVER aborts.
+3. Both CLIENT and SERVER initiate the SSL handshake protocol.
+4. Both CLIENT and SERVER request the certificate of their peer and verify
+   that certificate.  If this succeeds, all further communications are 
+   encrypted.
+5. Both CLIENT and SERVER send a hello block containing connection parameters
+   and their process id.
+6. The CLIENT and SERVER check that their peer's stated process id matches the
+   hash of the x509 certificate the peer presented.  If not, the connection is
+   aborted.
+7. The CLIENT verifies that the SERVER's stated id matches the id of the
+   SERVER the CLIENT is intending to connect to.  If not, the connection is 
+   aborted.
+8. The CLIENT and SERVER elect a master who decides on the final connection
+   parameters.
+
+The public/private key pair associated with each process's x509 certificate
+are completely hidden from this handshake protocol. There are however, used
+internally by OpenSSL as part of the SSL handshake protocol. Each process
+keeps their own private key hidden and sends its peer only the public key
+(embedded in the certificate).
+
+Finally, when the CLIENT requests access to a particular SERVER capability,
+the following happens:
+
+1. The CLIENT asks the SERVER for access to a capability by presenting that 
+   capabilities id.
+2. If the SERVER has a capability with that id, access is granted. If not,
+   access is not granted.
+3. Once access has been gained, the CLIENT can use the capability.
+
+Specific security vulnerabilities
+=================================
+
+There are a number of potential security vulnerabilities present in IPython's
+architecture. In this section we discuss those vulnerabilities and detail how
+the security architecture described above prevents them from being exploited.
+
+Unauthorized clients
+--------------------
+
+The IPython client can instruct the IPython engines to execute arbitrary
+Python code with the permissions of the user who started the engines. If an
+attacker were able to connect their own hostile IPython client to the IPython
+controller, they could instruct the engines to execute code.
+
+This attack is prevented by the capabilities based client authentication
+performed after the encrypted channel has been established. The relevant
+authentication information is encoded into the FURL that clients must
+present to gain access to the IPython controller. By limiting the distribution
+of those FURLs, a user can grant access to only authorized persons.
+
+It is highly unlikely that a client FURL could be guessed by an attacker
+in a brute force guessing attack. A given instance of the IPython controller
+only runs for a relatively short amount of time (on the order of hours). Thus
+an attacker would have only a limited amount of time to test a search space of
+size 2**320. Furthermore, even if a controller were to run for a longer amount
+of time, this search space is quite large (larger for instance than that of
+typical username/password pair).
+
+Unauthorized engines
+--------------------
+
+If an attacker were able to connect a hostile engine to a user's controller,
+the user might unknowingly send sensitive code or data to the hostile engine.
+This attacker's engine would then have full access to that code and data.
+
+This type of attack is prevented in the same way as the unauthorized client
+attack, through the usage of the capabilities based authentication scheme.
+
+Unauthorized controllers
+------------------------
+
+It is also possible that an attacker could try to convince a user's IPython
+client or engine to connect to a hostile IPython controller. That controller
+would then have full access to the code and data sent between the IPython
+client and the IPython engines.
+
+Again, this attack is prevented through the FURLs, which ensure that a
+client or engine connects to the correct controller. It is also important to
+note that the FURLs also encode the IP address and port that the
+controller is listening on, so there is little chance of mistakenly connecting
+to a controller running on a different IP address and port.
+
+When starting an engine or client, a user must specify which FURL to use
+for that connection. Thus, in order to introduce a hostile controller, the
+attacker must convince the user to use the FURLs associated with the
+hostile controller. As long as a user is diligent in only using FURLs from
+trusted sources, this attack is not possible.
+
+Other security measures
+=======================
+
+A number of other measures are taken to further limit the security risks
+involved in running the IPython kernel.
+
+First, by default, the IPython controller listens on random port numbers.
+While this can be overridden by the user, in the default configuration, an
+attacker would have to do a port scan to even find a controller to attack.
+When coupled with the relatively short running time of a typical controller
+(on the order of hours), an attacker would have to work extremely hard and
+extremely *fast* to even find a running controller to attack.
+
+Second, much of the time, especially when run on supercomputers or clusters,
+the controller is running behind a firewall. Thus, for engines or client to
+connect to the controller:
+
+* The different processes have to all be behind the firewall.
+
+or:
+
+* The user has to use SSH port forwarding to tunnel the
+  connections through the firewall.
+   
+In either case, an attacker is presented with addition barriers that prevent
+attacking or even probing the system.
+
+Summary
+=======
+
+IPython's architecture has been carefully designed with security in mind. The
+capabilities based authentication model, in conjunction with the encrypted
+TCP/IP channels, address the core potential vulnerabilities in the system,
+while still enabling user's to use the system in open networks.
+
+Other questions
+===============
+
+About keys
+----------
+
+Can you clarify the roles of the certificate and its keys versus the FURL,
+which is also called a key?
+
+The certificate created by IPython processes is a standard public key x509
+certificate, that is used by the SSL handshake protocol to setup encrypted
+channel between the controller and the IPython engine or client. This public
+and private key associated with this certificate are used only by the SSL
+handshake protocol in setting up this encrypted channel.
+
+The FURL serves a completely different and independent purpose from the
+key pair associated with the certificate. When we refer to a FURL as a
+key, we are using the word "key" in the capabilities based security model
+sense. This has nothing to do with "key" in the public/private key sense used
+in the SSL protocol.
+
+With that said the FURL is used as an cryptographic key, to grant
+IPython engines and clients access to particular capabilities that the
+controller offers.
+
+Self signed certificates
+------------------------
+
+Is the controller creating a self-signed certificate? Is this created for per
+instance/session, one-time-setup or each-time the controller is started?
+
+The Foolscap network protocol, which handles the SSL protocol details, creates
+a self-signed x509 certificate using OpenSSL for each IPython process. The
+lifetime of the certificate is handled differently for the IPython controller
+and the engines/client.
+
+For the IPython engines and client, the certificate is only held in memory for
+the lifetime of its process. It is never written to disk.
+
+For the controller, the certificate can be created anew each time the
+controller starts or it can be created once and reused each time the
+controller starts. If at any point, the certificate is deleted, a new one is
+created the next time the controller starts.
+
+SSL private key
+---------------
+
+How the private key (associated with the certificate) is distributed?
+
+In the usual implementation of the SSL protocol, the private key is never
+distributed. We follow this standard always.
+
+SSL versus Foolscap authentication
+----------------------------------
+
+Many SSL connections only perform one sided authentication (the server to the
+client). How is the client authentication in IPython's system related to SSL
+authentication?
+
+We perform a two way SSL handshake in which both parties request and verify
+the certificate of their peer. This mutual authentication is handled by the
+SSL handshake and is separate and independent from the additional
+authentication steps that the CLIENT and SERVER perform after an encrypted
+channel is established.
+
+.. [RFC5246] <http://tools.ietf.org/html/rfc5246>

IPython/ColorANSI.py

@@ -26,6 +26,7 b' def make_color_table(in_class):'
     Helper function for building the *TermColors classes."""
     
     color_templates = (
+        # Dark colors
         ("Black"       , "0;30"),
         ("Red"         , "0;31"),
         ("Green"       , "0;32"),
@@ -34,6 +35,7 b' def make_color_table(in_class):'
         ("Purple"      , "0;35"),
         ("Cyan"        , "0;36"),
         ("LightGray"   , "0;37"),
+        # Light colors
         ("DarkGray"    , "1;30"),
         ("LightRed"    , "1;31"),
         ("LightGreen"  , "1;32"),
@@ -41,7 +43,17 b' def make_color_table(in_class):'
         ("LightBlue"   , "1;34"),
         ("LightPurple" , "1;35"),
         ("LightCyan"   , "1;36"),
-        ("White"       , "1;37"),  )
+        ("White"       , "1;37"),  
+        # Blinking colors.  Probably should not be used in anything serious.
+        ("BlinkBlack"  , "5;30"),
+        ("BlinkRed"    , "5;31"),
+        ("BlinkGreen"  , "5;32"),
+        ("BlinkYellow" , "5;33"),
+        ("BlinkBlue"   , "5;34"),
+        ("BlinkPurple" , "5;35"),
+        ("BlinkCyan"   , "5;36"),
+        ("BlinkLightGray", "5;37"),
+        )
 
     for name,value in color_templates:
         setattr(in_class,name,in_class._base % value)

IPython/Extensions/ipy_completers.py

@@ -335,6 +335,12 b' def cd_completer(self, event):'
     if not found:
         if os.path.isdir(relpath):
             return [relpath]
+        # if no completions so far, try bookmarks
+        bks = self.db.get('bookmarks',{}).keys()
+        bkmatches = [s for s in bks if s.startswith(event.symbol)]
+        if bkmatches:
+            return bkmatches
+        
         raise IPython.ipapi.TryNext
 
 

IPython/Extensions/ipy_editors.py

@@ -28,7 +28,8 b' def install_editor(run_template, wait = False):'
             line = 0
         cmd = itplns(run_template, locals())
         print ">",cmd
-        os.system(cmd)
+        if os.system(cmd) != 0:
+            raise IPython.ipapi.TryNext()
         if wait:
             raw_input("Press Enter when done editing:")
 
@@ -65,6 +66,9 b' def idle(exe = None):'
         exe = p + '/idle.py'
     install_editor(exe + ' "$file"')
         
+def mate(exe = 'mate'):
+    """ TextMate, the missing editor"""
+    install_editor(exe + ' -w -l $line "$file"')
 
 # these are untested, report any problems
 

IPython/Extensions/ipy_profile_sh.py

@@ -8,7 +8,7 b' compatibility)'
 """
 
 from IPython import ipapi
-import os,textwrap
+import os,re,textwrap
 
 # The import below effectively obsoletes your old-style ipythonrc[.ini],
 # so consider yourself warned!
@@ -50,9 +50,15 b' def main():'
     ip.ex('import os')
     ip.ex("def up(): os.chdir('..')")
     ip.user_ns['LA'] = LastArgFinder() 
-    # Nice prompt
     
-    o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
+    # You can assign to _prompt_title variable 
+    # to provide some extra information for prompt
+    # (e.g. the current mode, host/username...)
+
+    ip.user_ns['_prompt_title'] = ''
+    
+    # Nice prompt
+    o.prompt_in1= r'\C_Green${_prompt_title}\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green> '
     o.prompt_out= '<\#> '
     
@@ -98,9 +104,15 b' def main():'
     for cmd in syscmds:
         # print "sys",cmd #dbg
         noext, ext = os.path.splitext(cmd)
-        key = mapper(noext)
+        if ext.lower() == '.exe':
+            cmd = noext
+        
+        key = mapper(cmd)
         if key not in ip.IP.alias_table:
-            ip.defalias(key, cmd)
+            # Dots will be removed from alias names, since ipython
+            # assumes names with dots to be python code
+            
+            ip.defalias(key.replace('.',''), cmd)
 
     # mglob combines 'find', recursion, exclusion... '%mglob?' to learn more
     ip.load("IPython.external.mglob")    
@@ -117,7 +129,7 b' def main():'
             # and the next best thing to real 'ls -F'
             ip.defalias('d','dir /w /og /on')
     
-    ip.set_hook('input_prefilter', dotslash_prefilter_f)
+    ip.set_hook('input_prefilter', slash_prefilter_f)
     extend_shell_behavior(ip)
 
 class LastArgFinder:
@@ -139,13 +151,13 b' class LastArgFinder:'
             return parts[-1]
         return ""
 
-def dotslash_prefilter_f(self,line):
-    """ ./foo now runs foo as system command 
+def slash_prefilter_f(self,line):
+    """ ./foo, ~/foo and /bin/foo now run foo as system command
     
-    Removes the need for doing !./foo
+    Removes the need for doing !./foo, !~/foo or !/bin/foo
     """
     import IPython.genutils
-    if line.startswith("./"):
+    if re.match('(?:[.~]|/[a-zA-Z_0-9]+)/', line):
         return "_ip.system(" + IPython.genutils.make_quoted_expr(line)+")"
     raise ipapi.TryNext
 

IPython/Magic.py

@@ -422,7 +422,11 b' python-profiler package from non-free.""")'
                 else:
                     fndoc = 'No documentation'
             else:
+                if fn.__doc__:
                 fndoc = fn.__doc__.rstrip()
+                else:
+                    fndoc = 'No documentation'
+                
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
@@ -2328,7 +2332,17 b' Currently the magic system has the following functions:\\n"""'
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
+        try:
         self.shell.hooks.editor(filename,lineno)
+        except IPython.ipapi.TryNext:
+            warn('Could not open editor')
+            return
+        
+        # XXX TODO: should this be generalized for all string vars?
+        # For now, this is special-cased to blocks created by cpaste
+        if args.strip() == 'pasted_block':
+            self.shell.user_ns['pasted_block'] = file_read(filename)
+        
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
@@ -2338,6 +2352,8 b' Currently the magic system has the following functions:\\n"""'
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
+        
+                                                     
         if use_temp:
             try:
                 return open(filename).read()
@@ -2648,7 +2664,9 b' Defaulting color scheme to \'NoColor\'"""'
                             # each entry in the alias table must be (N,name),
                             # where N is the number of positional arguments of the
                             # alias.
-                            alias_table[ff] = (0,ff)
+                            # Dots will be removed from alias names, since ipython
+                            # assumes names with dots to be python code
+                            alias_table[ff.replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             else:
                 for pdir in path:
@@ -2658,7 +2676,7 b' Defaulting color scheme to \'NoColor\'"""'
                         if isexec(ff) and base.lower() not in self.shell.no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
-                            alias_table[base.lower()] = (0,ff)
+                            alias_table[base.lower().replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             # Make sure the alias table doesn't contain keywords or builtins
             self.shell.alias_table_validate()
@@ -3210,14 +3228,24 b' Defaulting color scheme to \'NoColor\'"""'
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
         
+        '%cpaste -r' re-executes the block previously entered by cpaste.
+        
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
         """
-        opts,args = self.parse_options(parameter_s,'s:',mode='string')
+        opts,args = self.parse_options(parameter_s,'rs:',mode='string')
         par = args.strip()
+        if opts.has_key('r'):
+            b = self.user_ns.get('pasted_block', None)
+            if b is None:
+                raise UsageError('No previous pasted block available')
+            print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
+            exec b in self.user_ns
+            return
+        
         sentinel = opts.get('s','--')
 
         # Regular expressions that declare text we strip from the input:
@@ -3245,8 +3273,8 b' Defaulting color scheme to \'NoColor\'"""'
         #print "block:\n",block
         if not par:
             b = textwrap.dedent(block)
-            exec b in self.user_ns            
             self.user_ns['pasted_block'] = b
+            exec b in self.user_ns
         else:
             self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par

IPython/Release.py

@@ -1,7 +1,5 b''
 # -*- coding: utf-8 -*-
-"""Release data for the IPython project.
-
-$Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
+"""Release data for the IPython project."""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
@@ -23,9 +21,9 b" name = 'ipython'"
 # bdist_deb does not accept underscores (a Debian convention).
 
 development = False    # change this to False to do a release
-version_base = '0.9.beta'
+version_base = '0.9.1'
 branch = 'ipython'
-revision = '1099'
+revision = '1143'
 
 if development:
     if branch == 'ipython':
@@ -36,14 +34,19 b' else:'
     version = version_base
 
 
-description = "Tools for interactive development in Python."
+description = "An interactive computing environment for Python"
 
 long_description = \
 """
-IPython provides a replacement for the interactive Python interpreter with
-extra functionality.
+The goal of IPython is to create a comprehensive environment for
+interactive and exploratory computing.  To support this goal, IPython
+has two main components:
+
+* An enhanced interactive Python shell.
+
+* An architecture for interactive parallel computing.
 
-Main features:
+The enhanced interactive Python shell has the following main features:
 
  * Comprehensive object introspection.
 
@@ -66,14 +69,33 b' Main features:'
 
  * Access to the system shell with user-extensible alias system.
 
- * Easily embeddable in other Python programs.
+* Easily embeddable in other Python programs and wxPython GUIs.
 
  * Integrated access to the pdb debugger and the Python profiler. 
 
- The latest development version is always available at the IPython subversion
- repository_.
+The parallel computing architecture has the following main features:
+
+* Quickly parallelize Python code from an interactive Python/IPython session.
+
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
+
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.
+
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Share live parallel jobs with other users securely.  
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling in parallel code.
 
-.. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
+The latest development version is always available from IPython's `Launchpad 
+site <http://launchpad.net/ipython>`_.
  """
 
 license = 'BSD'

IPython/Shell.py

@@ -40,11 +40,12 b' except ImportError:'
 # IPython imports
 import IPython
 from IPython import ultraTB, ipapi
+from IPython.Magic import Magic
 from IPython.genutils import Term,warn,error,flag_calls, ask_yes_no
 from IPython.iplib import InteractiveShell
 from IPython.ipmaker import make_IPython
-from IPython.Magic import Magic
 from IPython.ipstruct import Struct
+from IPython.testing import decorators as testdec
 
 # Globals
 # global flag to pass around information about Ctrl-C without exceptions
@@ -414,7 +415,7 b' class MTInteractiveShell(InteractiveShell):'
         if (self.worker_ident is None
             or self.worker_ident == thread.get_ident() ):
             InteractiveShell.runcode(self,code)
-            return
+            return False
 
         # Case 3
         # Store code in queue, so the execution thread can handle it.
@@ -608,6 +609,7 b' class MatplotlibShellBase:'
         if self.mpl_use._called:
             self.matplotlib.rcParams['backend'] = self.mpl_backend
         
+    @testdec.skip_doctest
     def magic_run(self,parameter_s=''):
         Magic.magic_run(self,parameter_s,runner=self.mplot_exec)
 
@@ -774,6 +776,17 b' class IPShellGTK(IPThread):'
                  debug=1,shell_class=MTInteractiveShell):
 
         import gtk
+        # Check for set_interactive, coming up in new pygtk.
+        # Disable it so that this code works, but notify 
+        # the user that he has a better option as well.
+        # XXX TODO better support when set_interactive is released
+        try:
+            gtk.set_interactive(False)
+            print "Your PyGtk has set_interactive(), so you can use the"
+            print "more stable single-threaded Gtk mode."
+            print "See https://bugs.launchpad.net/ipython/+bug/270856"
+        except AttributeError:
+            pass
         
         self.gtk = gtk
         self.gtk_mainloop = hijack_gtk()

IPython/UserConfig/ipythonrc-physics

@@ -38,6 +38,8 b' ececute rad = pi/180.'
 execute print '*** q is an alias for PhysicalQuantityInteractive'
 execute print '*** g = 9.8 m/s^2 has been defined'
 execute print '*** rad = pi/180  has been defined'
+execute import ipy_constants as C
+execute print '*** C is the physical constants module'
 
 # Files to execute
 execfile 

IPython/completer.py

@@ -314,7 +314,10 b' class IPCompleter(Completer):'
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
+        if sys.platform == 'win32':
         protectables = ' '
+        else:
+            protectables = ' ()'
 
         if text.startswith('!'):
             text = text[1:]

IPython/config/api.py

@@ -16,13 +16,11 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import os
-from IPython.config.cutils import get_home_dir, get_ipython_dir
+from os.path import join as pjoin
+
+from IPython.genutils import get_home_dir, get_ipython_dir
 from IPython.external.configobj import ConfigObj
 
-# Traitlets config imports
-from IPython.config import traitlets
-from IPython.config.config import *
-from traitlets import *
 
 class ConfigObjManager(object):
     
@@ -53,7 +51,7 b' class ConfigObjManager(object):'
 
     def write_default_config_file(self):
         ipdir = get_ipython_dir()
-        fname = ipdir + '/' + self.filename
+        fname = pjoin(ipdir, self.filename)
         if not os.path.isfile(fname):
             print "Writing the configuration file to: " + fname
             self.write_config_obj_to_file(fname)
@@ -87,11 +85,11 b' class ConfigObjManager(object):'
 
         # In ipythondir if it is set
         if ipythondir is not None:
-            trythis = ipythondir + '/' + filename
+            trythis = pjoin(ipythondir, filename)
             if os.path.isfile(trythis):
                 return trythis        
 
-        trythis = get_ipython_dir() + '/' + filename
+        trythis = pjoin(get_ipython_dir(), filename)
         if os.path.isfile(trythis):
             return trythis
 

IPython/config/cutils.py

@@ -22,71 +22,6 b' import sys'
 # Normal code begins
 #---------------------------------------------------------------------------
 
-class HomeDirError(Exception):
-    pass
-
-def get_home_dir():
-    """Return the closest possible equivalent to a 'home' directory.
-
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
-    Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
-
-    isdir = os.path.isdir
-    env = os.environ
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
-        else:
-            raise HomeDirError,'support for your operating system not implemented.'
-
-def get_ipython_dir():
-    ipdir_def = '.ipython'
-    home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir,ipdir_def)))
-    return ipdir
-
 def import_item(key):
     """
     Import and return bar given the string foo.bar.

IPython/demo.py

@@ -180,7 +180,7 b' def re_mark(mark):'
 
 class Demo(object):
 
-    re_stop     = re_mark('-?\s?stop\s?-?')
+    re_stop     = re_mark('-*\s?stop\s?-*')
     re_silent   = re_mark('silent')
     re_auto     = re_mark('auto')
     re_auto_all = re_mark('auto_all')

IPython/frontend/_process/killableprocess.py

@@ -50,7 +50,6 b' import subprocess'
 from subprocess import PIPE
 import sys
 import os
-import time
 import types
 
 try:
@@ -69,7 +68,15 b' except ImportError:'
 
 mswindows = (sys.platform == "win32")
 
+skip = False
+
 if mswindows:
+    import platform
+    if platform.uname()[3] == '' or platform.uname()[3] > '6.0.6000':
+        # Killable process does not work under vista when starting for
+        # something else than cmd.
+        skip = True
+    else:
     import winprocess
 else:
     import signal
@@ -78,6 +85,10 b' if not mswindows:'
     def DoNothing(*args):
         pass
 
+
+if skip:
+ Popen = subprocess.Popen
+else:
 class Popen(subprocess.Popen):
     if not mswindows:
         # Override __init__ to set a preexec_fn

IPython/frontend/_process/pipedprocess.py

@@ -50,7 +50,7 b' class PipedProcess(Thread):'
         """
         env = os.environ
         env['TERM'] = 'xterm'
-        process = Popen((self.command_string + ' 2>&1', ), shell=True,
+        process = Popen(self.command_string + ' 2>&1', shell=True,
                                 env=env,
                                 universal_newlines=True,
                                 stdout=PIPE, stdin=PIPE, )

IPython/frontend/asyncfrontendbase.py

@@ -14,30 +14,14 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-import uuid
+from IPython.external import guid
 
-try:
-    from zope.interface import Interface, Attribute, implements, classProvides
-except ImportError, e:
-    e.message = """%s
-________________________________________________________________________________
-zope.interface is required to run asynchronous frontends.""" % e.message
-    e.args = (e.message, ) + e.args[1:]
-
-from frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
-
-from IPython.kernel.engineservice import IEngineCore
-from IPython.kernel.core.history import FrontEndHistory
 
-try:
+from zope.interface import Interface, Attribute, implements, classProvides
     from twisted.python.failure import Failure
-except ImportError, e:
-    e.message = """%s
-________________________________________________________________________________
-twisted is required to run asynchronous frontends.""" % e.message
-    e.args = (e.message, ) + e.args[1:]
-
-
+from IPython.frontend.frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
+from IPython.kernel.core.history import FrontEndHistory
+from IPython.kernel.engineservice import IEngineCore
 
 
 class AsyncFrontEndBase(FrontEndBase):
@@ -75,7 +59,7 b' class AsyncFrontEndBase(FrontEndBase):'
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate() 
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)

IPython/frontend/cocoa/cocoa_frontend.py

@@ -26,7 +26,7 b' __docformat__ = "restructuredtext en"'
 
 import sys
 import objc
-import uuid
+from IPython.external import guid
 
 from Foundation import NSObject, NSMutableArray, NSMutableDictionary,\
                         NSLog, NSNotificationCenter, NSMakeRange,\
@@ -361,7 +361,7 b' class IPythonCocoaController(NSObject, AsyncFrontEndBase):'
     
     def next_block_ID(self):
         
-        return uuid.uuid4()
+        return guid.generate() 
     
     def new_cell_block(self):
         """A new CellBlock at the end of self.textView.textStorage()"""

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -14,14 +14,18 b' __docformat__ = "restructuredtext en"'
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
+
+try:
 from IPython.kernel.core.interpreter import Interpreter                     
 import IPython.kernel.engineservice as es
 from IPython.testing.util import DeferredTestCase
 from twisted.internet.defer import succeed
 from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
-                                                    
 from Foundation import NSMakeRect
 from AppKit import NSTextView, NSScrollView                                                
+except ImportError:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted, Foolscap and PyObjC")
 
 class TestIPythonCocoaControler(DeferredTestCase):
     """Tests for IPythonCocoaController"""
@@ -31,7 +35,6 b' class TestIPythonCocoaControler(DeferredTestCase):'
         self.engine = es.EngineService()
         self.engine.startService()
     
-    
     def tearDown(self):
         self.controller = None
         self.engine.stopService()

IPython/frontend/frontendbase.py

@@ -21,14 +21,16 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 import string
-import uuid
-import _ast
+import codeop
+from IPython.external import guid
 
-from zopeinterface import Interface, Attribute, implements, classProvides
 
+from IPython.frontend.zopeinterface import (
+    Interface, 
+    Attribute, 
+)
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.core.util import Bunch
-from IPython.kernel.engineservice import IEngineCore
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
@@ -131,10 +133,6 b' class IFrontEnd(Interface):'
         
         pass
     
-    def compile_ast(block):
-        """Compiles block to an _ast.AST"""
-        
-        pass
     
     def get_history_previous(current_block):
         """Returns the block previous in  the history. Saves currentBlock if
@@ -217,28 +215,14 b' class FrontEndBase(object):'
         """
         
         try:
-            ast = self.compile_ast(block)
+            is_complete = codeop.compile_command(block.rstrip() + '\n\n',
+                            "<string>", "exec") 
         except:
             return False
         
         lines = block.split('\n')
-        return (len(lines)==1 or str(lines[-1])=='')
-    
-    
-    def compile_ast(self, block):
-        """Compile block to an AST
-        
-        Parameters:
-            block : str
-        
-        Result:
-            AST
-        
-        Throws:
-            Exception if block cannot be compiled
-        """
-        
-        return compile(block, "<string>", "exec", _ast.PyCF_ONLY_AST)
+        return ((is_complete is not None) 
+                    and (len(lines)==1 or str(lines[-1])==''))
     
     
     def execute(self, block, blockID=None):
@@ -258,7 +242,7 b' class FrontEndBase(object):'
             raise Exception("Block is not compilable")
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate()
         
         try:
             result = self.shell.execute(block)

IPython/frontend/linefrontendbase.py

@@ -20,6 +20,8 b' import re'
 
 import IPython
 import sys
+import codeop
+import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
@@ -76,6 +78,11 b' class LineFrontEndBase(FrontEndBase):'
 
         if banner is not None:
             self.banner = banner
+
+    def start(self):
+        """ Put the frontend in a state where it is ready for user
+            interaction.
+        """
         if self.banner is not None:
             self.write(self.banner, refresh=False)
 
@@ -141,9 +148,18 b' class LineFrontEndBase(FrontEndBase):'
                         and not re.findall(r"\n[\t ]*\n[\t ]*$", string)):
             return False
         else:
+            self.capture_output()
+            try:
             # Add line returns here, to make sure that the statement is
             # complete.
-            return FrontEndBase.is_complete(self, string.rstrip() + '\n\n')
+                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                            "<string>", "exec")
+                self.release_output()
+            except Exception, e:
+                # XXX: Hack: return True so that the
+                # code gets executed and the error captured.
+                is_complete = True
+            return is_complete
 
 
     def write(self, string, refresh=True):
@@ -166,6 +182,18 b' class LineFrontEndBase(FrontEndBase):'
             raw_string = python_string
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
+
+        ## try:
+        ##     self.history.input_cache[-1] = raw_string.rstrip()
+        ##     result = self.shell.execute(python_string)
+        ##     self.last_result = result
+        ##     self.render_result(result)
+        ## except:
+        ##     self.show_traceback()
+        ## finally:
+        ##     self.after_execute()
+
+        try:
         try:
             self.history.input_cache[-1] = raw_string.rstrip()
             result = self.shell.execute(python_string)
@@ -176,12 +204,13 b' class LineFrontEndBase(FrontEndBase):'
         finally:
             self.after_execute()
 
+
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, string):
-        """ Priflter the input to turn it in valid python.
+        """ Prefilter the input to turn it in valid python.
         """
         string = string.replace('\r\n', '\n')
         string = string.replace('\t', 4*' ')
@@ -210,9 +239,12 b' class LineFrontEndBase(FrontEndBase):'
         line = self.input_buffer
         new_line, completions = self.complete(line)
         if len(completions)>1:
-            self.write_completion(completions)
+            self.write_completion(completions, new_line=new_line)
+        elif not line == new_line:
         self.input_buffer = new_line
         if self.debug:
+            print >>sys.__stdout__, 'line', line
+            print >>sys.__stdout__, 'new_line', new_line
             print >>sys.__stdout__, completions 
 
 
@@ -222,10 +254,15 b' class LineFrontEndBase(FrontEndBase):'
         return 80
 
 
-    def write_completion(self, possibilities):
+    def write_completion(self, possibilities, new_line=None):
         """ Write the list of possible completions.
+
+            new_line is the completed input line that should be displayed
+            after the completion are writen. If None, the input_buffer
+            before the completion is used.
         """
-        current_buffer = self.input_buffer
+        if new_line is None:
+            new_line = self.input_buffer
         
         self.write('\n')
         max_len = len(max(possibilities, key=len)) + 1
@@ -246,7 +283,7 b' class LineFrontEndBase(FrontEndBase):'
         self.write(''.join(buf))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
-        self.input_buffer = current_buffer
+        self.input_buffer = new_line
 
 
     def new_prompt(self, prompt):
@@ -275,6 +312,8 b' class LineFrontEndBase(FrontEndBase):'
         else:
             self.input_buffer += self._get_indent_string(
                                                 current_buffer[:-1])
+            if len(current_buffer.split('\n')) == 2:
+                self.input_buffer += '\t\t'
             if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
                 self.input_buffer += '\t'
 

IPython/frontend/prefilterfrontend.py

@@ -24,6 +24,7 b' __docformat__ = "restructuredtext en"'
 import sys
 
 from linefrontendbase import LineFrontEndBase, common_prefix
+from frontendbase import FrontEndBase
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
@@ -34,6 +35,7 b' from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap'
 from IPython.genutils import Term
 import pydoc
 import os
+import sys
 
 
 def mk_system_call(system_call_function, command):
@@ -58,6 +60,8 b' class PrefilterFrontEnd(LineFrontEndBase):'
     completion...
     """
     
+    debug = False
+    
     def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters:
             -----------
@@ -65,12 +69,24 b' class PrefilterFrontEnd(LineFrontEndBase):'
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
         """
+        LineFrontEndBase.__init__(self, *args, **kwargs)
+        self.shell.output_trap = RedirectorOutputTrap(
+                            out_callback=self.write,
+                            err_callback=self.write,
+                                            )
+        self.shell.traceback_trap = SyncTracebackTrap(
+                        formatters=self.shell.traceback_trap.formatters,
+                            )
+
+        # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[])
+            ipython0 = make_IPython(argv=[], 
+                                    user_ns=self.shell.user_ns,
+                                    user_global_ns=self.shell.user_global_ns)
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
@@ -86,24 +102,13 b' class PrefilterFrontEnd(LineFrontEndBase):'
                                                             'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
+
+
         if not 'banner' in kwargs and self.banner is None:
-            kwargs['banner'] = self.ipython0.BANNER + """
+            self.banner = self.ipython0.BANNER + """
 This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
 
-        LineFrontEndBase.__init__(self, *args, **kwargs)
-        # XXX: Hack: mix the two namespaces
-        self.shell.user_ns.update(self.ipython0.user_ns)
-        self.ipython0.user_ns = self.shell.user_ns 
-        self.shell.user_global_ns.update(self.ipython0.user_global_ns)
-        self.ipython0.user_global_ns = self.shell.user_global_ns
-
-        self.shell.output_trap = RedirectorOutputTrap(
-                            out_callback=self.write,
-                            err_callback=self.write,
-                                            )
-        self.shell.traceback_trap = SyncTracebackTrap(
-                        formatters=self.shell.traceback_trap.formatters,
-                            )
+        self.start()
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
@@ -113,7 +118,7 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         """ Use ipython0 to capture the last traceback and display it.
         """
         self.capture_output()
-        self.ipython0.showtraceback()
+        self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
 
@@ -164,6 +169,8 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
 
 
     def complete(self, line):
+        # FIXME: This should be factored out in the linefrontendbase
+        # method.
         word = line.split('\n')[-1].split(' ')[-1]
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
@@ -189,6 +196,20 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         # capture it.
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
+        
+        ## try:
+        ##     for line in input_string.split('\n'):
+        ##         filtered_lines.append(
+        ##                 self.ipython0.prefilter(line, False).rstrip())
+        ## except:
+        ##     # XXX: probably not the right thing to do.
+        ##     self.ipython0.showsyntaxerror()
+        ##     self.after_execute()
+        ## finally:
+        ##     self.release_output()
+
+
+        try:
         try:
             for line in input_string.split('\n'):
                 filtered_lines.append(
@@ -200,6 +221,8 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         finally:
             self.release_output()
 
+
+
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string

IPython/frontend/tests/test_frontendbase.py

@@ -1,152 +1,32 b''
 # encoding: utf-8
-
-"""This file contains unittests for the frontendbase module."""
+"""
+Test the basic functionality of frontendbase. 
+"""
 
 __docformat__ = "restructuredtext en"
 
-#---------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                         
 #                                                                           
-#  Distributed under the terms of the BSD License.  The full license is in  
-#  the file COPYING, distributed as part of this software.                  
-#---------------------------------------------------------------------------
-                                                                            
-#---------------------------------------------------------------------------
-# Imports                                                                   
-#---------------------------------------------------------------------------
-
-import unittest
-from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-from IPython.frontend import frontendbase 
-from IPython.kernel.engineservice import EngineService
-
-class FrontEndCallbackChecker(AsyncFrontEndBase):
-    """FrontEndBase subclass for checking callbacks"""
-    def __init__(self, engine=None, history=None):
-        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
-                                                    history=history)
-        self.updateCalled = False
-        self.renderResultCalled = False
-        self.renderErrorCalled = False
-    
-    def update_cell_prompt(self, result, blockID=None):
-        self.updateCalled = True
-        return result
-    
-    def render_result(self, result):
-        self.renderResultCalled = True
-        return result
-    
-    
-    def render_error(self, failure):
-        self.renderErrorCalled = True
-        return failure
-    
-
-
-    
-class TestAsyncFrontendBase(unittest.TestCase):
-    def setUp(self):
-        """Setup the EngineService and FrontEndBase"""
-        
-        self.fb = FrontEndCallbackChecker(engine=EngineService())
-    
-    
-    def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(
-                                    AsyncFrontEndBase))
-    
-    
-    def test_is_complete_returns_False_for_incomplete_block(self):
-        """"""
-        
-        block = """def test(a):"""
-        
-        assert(self.fb.is_complete(block) == False)
-    
-    def test_is_complete_returns_True_for_complete_block(self):
-        """"""
-        
-        block = """def test(a): pass"""
-            
-        assert(self.fb.is_complete(block))
-        
-        block = """a=3"""
-        
-        assert(self.fb.is_complete(block))
-    
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
     
-    def test_blockID_added_to_result(self):
-        block = """3+3"""
+from IPython.frontend.frontendbase import FrontEndBase
         
-        d = self.fb.execute(block, blockID='TEST_ID')
-        
-        d.addCallback(self.checkBlockID, expected='TEST_ID')
-    
-    def test_blockID_added_to_failure(self):
-        block = "raise Exception()"
-        
-        d = self.fb.execute(block,blockID='TEST_ID')
-        d.addErrback(self.checkFailureID, expected='TEST_ID')
-    
-    def checkBlockID(self, result, expected=""):
-        assert(result['blockID'] == expected)
-    
-    
-    def checkFailureID(self, failure, expected=""):
-        assert(failure.blockID == expected)
-    
-    
-    def test_callbacks_added_to_execute(self):
-        """test that
-        update_cell_prompt
-        render_result
-        
-        are added to execute request
+def test_iscomplete():
+    """ Check that is_complete works. 
         """
-        
-        d = self.fb.execute("10+10")
-        d.addCallback(self.checkCallbacks)
-    
-    
-    def checkCallbacks(self, result):
-        assert(self.fb.updateCalled)
-        assert(self.fb.renderResultCalled)
-    
-    
-    def test_error_callback_added_to_execute(self):
-        """test that render_error called on execution error"""
-        
-        d = self.fb.execute("raise Exception()")
-        d.addCallback(self.checkRenderError)
-    
-    def checkRenderError(self, result):
-        assert(self.fb.renderErrorCalled)
-    
-    def test_history_returns_expected_block(self):
-        """Make sure history browsing doesn't fail"""
-        
-        blocks = ["a=1","a=2","a=3"]
-        for b in blocks:
-            d = self.fb.execute(b)
-        
-        # d is now the deferred for the last executed block
-        d.addCallback(self.historyTests, blocks)
-        
-    
-    def historyTests(self, result, blocks):
-        """historyTests"""
-        
-        assert(len(blocks) >= 3)
-        assert(self.fb.get_history_previous("") == blocks[-2])
-        assert(self.fb.get_history_previous("") == blocks[-3])
-        assert(self.fb.get_history_next() == blocks[-2])
-    
-    
-    def test_history_returns_none_at_startup(self):
-        """test_history_returns_none_at_startup"""
-        
-        assert(self.fb.get_history_previous("")==None)
-        assert(self.fb.get_history_next()==None)
-    
+    f = FrontEndBase()
+    assert f.is_complete('(a + a)')
+    assert not f.is_complete('(a + a')
+    assert f.is_complete('1')
+    assert not f.is_complete('1 + ')
+    assert not f.is_complete('1 + \n\n')
+    assert f.is_complete('if True:\n  print 1\n')
+    assert not f.is_complete('if True:\n  print 1')
+    assert f.is_complete('def f():\n  print 1\n')
+
+if __name__ == '__main__':
+    test_iscomplete()
     

IPython/frontend/tests/test_process.py

@@ -17,6 +17,8 b' from time import sleep'
 import sys
 
 from IPython.frontend._process import PipedProcess
+from IPython.testing import decorators as testdec
+
 
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
@@ -25,7 +27,8 b' def test_capture_out():'
     p = PipedProcess('echo 1', out_callback=s.write, )
     p.start()
     p.join()
-    assert s.getvalue() == '1\n'
+    result = s.getvalue().rstrip()
+    assert result == '1'
 
 
 def test_io():
@@ -40,7 +43,8 b' def test_io():'
         sleep(0.1)
     p.process.stdin.write(test_string)
     p.join()
-    assert s.getvalue() == test_string
+    result = s.getvalue()
+    assert result == test_string
 
 
 def test_kill():

IPython/frontend/wx/console_widget.py

@@ -23,6 +23,7 b' import wx'
 import wx.stc  as  stc
 
 from wx.py import editwindow
+import time
 import sys
 LINESEP = '\n'
 if sys.platform == 'win32':
@@ -35,7 +36,7 b' import re'
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
-    _DEFAULT_STYLE = 12
+    _DEFAULT_SIZE = 12
 
 _DEFAULT_STYLE = {
     'stdout'      : 'fore:#0000FF',
@@ -115,12 +116,15 b' class ConsoleWidget(editwindow.EditWindow):'
     # The color of the carret (call _apply_style() after setting)
     carret_color = 'BLACK'
     
+    # Store the last time a refresh was done
+    _last_refresh_time = 0
+
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
     
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
-                        size=wx.DefaultSize, style=0, ):
+                        size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
         self._configure_scintilla()
 
@@ -168,9 +172,14 b' class ConsoleWidget(editwindow.EditWindow):'
                 
         self.GotoPos(self.GetLength())
         if refresh:
-            # Maybe this is faster than wx.Yield()
-            self.ProcessEvent(wx.PaintEvent())
-            #wx.Yield()
+            current_time = time.time()
+            if current_time - self._last_refresh_time > 0.03:
+                if sys.platform == 'win32':
+                    wx.SafeYield()
+                else:
+                    wx.Yield()
+                #    self.ProcessEvent(wx.PaintEvent())
+                self._last_refresh_time = current_time 
 
    
     def new_prompt(self, prompt):
@@ -183,7 +192,6 b' class ConsoleWidget(editwindow.EditWindow):'
         # now we update our cursor giving end of prompt
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
-        wx.Yield()
         self.EnsureCaretVisible()
 
 

IPython/frontend/wx/wx_frontend.py

@@ -128,6 +128,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # while it is being swapped
     _out_buffer_lock = Lock()
 
+    # The different line markers used to higlight the prompts.
     _markers = dict()
 
     #--------------------------------------------------------------------------
@@ -135,13 +136,17 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     #--------------------------------------------------------------------------
  
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
-                 size=wx.DefaultSize, style=wx.CLIP_CHILDREN,
+                 size=wx.DefaultSize,
+                 style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
                  *args, **kwds):
         """ Create Shell instance.
         """
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
 
+        # Stick in our own raw_input:
+        self.ipython0.raw_input = self.raw_input
+
         # Marker for complete buffer.
         self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
                                 background=_COMPLETE_BUFFER_BG)
@@ -164,9 +169,11 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         # Inject self in namespace, for debug
         if self.debug:
             self.shell.user_ns['self'] = self
+        # Inject our own raw_input in namespace
+        self.shell.user_ns['raw_input'] = self.raw_input
 
 
-    def raw_input(self, prompt):
+    def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
         self.new_prompt(prompt)
@@ -174,15 +181,13 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if hasattr(self, '_cursor'):
             del self._cursor 
         self.SetCursor(wx.StockCursor(wx.CURSOR_CROSS))
-        self.waiting = True
         self.__old_on_enter = self._on_enter
+        event_loop = wx.EventLoop()
         def my_on_enter():
-            self.waiting = False
+            event_loop.Exit()
         self._on_enter = my_on_enter
-        # XXX: Busy waiting, ugly.
-        while self.waiting:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         self._on_enter = self.__old_on_enter
         self._input_state = 'buffering'
         self._cursor = wx.BusyCursor()
@@ -191,16 +196,18 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
 
     def system_call(self, command_string):
         self._input_state = 'subprocess'
+        event_loop = wx.EventLoop()
+        def _end_system_call():
+            self._input_state = 'buffering'
+            self._running_process = False
+            event_loop.Exit()
+
         self._running_process = PipedProcess(command_string, 
                     out_callback=self.buffered_write,
-                    end_callback = self._end_system_call)
+                    end_callback = _end_system_call)
         self._running_process.start()
-        # XXX: another one of these polling loops to have a blocking
-        # call
-        wx.Yield()
-        while self._running_process:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         # Be sure to flush the buffer.
         self._buffer_flush(event=None)
 
@@ -226,8 +233,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             for name in symbol_string.split('.')[1:] + ['__doc__']:
                 symbol = getattr(symbol, name)
             self.AutoCompCancel()
-            wx.Yield()
-            self.CallTipShow(self.GetCurrentPos(), symbol)
+            # Check that the symbol can indeed be converted to a string:
+            symbol += ''
+            wx.CallAfter(self.CallTipShow, self.GetCurrentPos(), symbol)
         except:
             # The retrieve symbol couldn't be converted to a string
             pass
@@ -238,9 +246,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             true, open the menu.
         """
         if self.debug:
-            print >>sys.__stdout__, "_popup_completion", 
+            print >>sys.__stdout__, "_popup_completion" 
         line = self.input_buffer
-        if (self.AutoCompActive() and not line[-1] == '.') \
+        if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
             offset=0
@@ -284,19 +292,21 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _COMPLETE_BUFFER_MARKER)
-        # Update the display:
-        wx.Yield()
+        # Use a callafter to update the display robustly under windows
+        def callback():
         self.GotoPos(self.GetLength())
-        PrefilterFrontEnd.execute(self, python_string, raw_string=raw_string)
+            PrefilterFrontEnd.execute(self, python_string, 
+                                            raw_string=raw_string)
+        wx.CallAfter(callback)
 
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
 
     def capture_output(self):
-        __builtin__.raw_input = self.raw_input
         self.SetLexer(stc.STC_LEX_NULL)
         PrefilterFrontEnd.capture_output(self)
+        __builtin__.raw_input = self.raw_input
         
     
     def release_output(self):
@@ -316,7 +326,19 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     def show_traceback(self):
         start_line = self.GetCurrentLine()
         PrefilterFrontEnd.show_traceback(self)
-        wx.Yield()
+        self.ProcessEvent(wx.PaintEvent())
+        #wx.Yield()
+        for i in range(start_line, self.GetCurrentLine()):
+            self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
+
+    
+    #--------------------------------------------------------------------------
+    # FrontEndBase interface 
+    #--------------------------------------------------------------------------
+    
+    def render_error(self, e):
+        start_line = self.GetCurrentLine()
+        self.write('\n' + e + '\n')
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
@@ -351,6 +373,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if self._input_state == 'subprocess':
                 if self.debug:
                     print >>sys.__stderr__, 'Killing running process'
+                if hasattr(self._running_process, 'process'):
                 self._running_process.process.kill()
             elif self._input_state == 'buffering':
                 if self.debug:
@@ -376,7 +399,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                 char = '\04'
             self._running_process.process.stdin.write(char)
             self._running_process.process.stdin.flush()
-        elif event.KeyCode in (ord('('), 57):
+        elif event.KeyCode in (ord('('), 57, 53):
             # Calltips
             event.Skip()
             self.do_calltip()
@@ -410,8 +433,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                last_line = self.input_buffer.split('\n')[-1]
-                if not re.match(r'^\s*$', last_line):
+                current_line, current_line_number = self.CurLine
+                if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
@@ -427,7 +450,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if event.KeyCode in (59, ord('.')):
             # Intercepting '.'
             event.Skip()
-            self._popup_completion(create=True)
+            wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
 
@@ -456,13 +479,6 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # Private API
     #--------------------------------------------------------------------------
  
-    def _end_system_call(self):
-        """ Called at the end of a system call.
-        """
-        self._input_state = 'buffering'
-        self._running_process = False
-
-
     def _buffer_flush(self, event):
         """ Called by the timer to flush the write buffer.
             

IPython/frontend/zopeinterface.py

@@ -16,13 +16,6 b' __docformat__ = "restructuredtext en"'
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-import string
-import uuid
-import _ast
-
 try:
     from zope.interface import Interface, Attribute, implements, classProvides
 except ImportError:

IPython/genutils.py

@@ -979,6 +979,38 b' def get_home_dir():'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds either .ipython or _ipython to the end of the path.
+    """
+    if os.name == 'posix':
+         ipdir_def = '.ipython'
+    else:
+         ipdir_def = '_ipython'
+    home_dir = get_home_dir()
+    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
+                                           os.path.join(home_dir,ipdir_def)))
+    return ipdir
+
+def get_security_dir():
+    """Get the IPython security directory.
+    
+    This directory is the default location for all security related files,
+    including SSL/TLS certificates and FURL files.
+    
+    If the directory does not exist, it is created with 0700 permissions.
+    If it exists, permissions are set to 0700.
+    """
+    security_dir = os.path.join(get_ipython_dir(), 'security')
+    if not os.path.isdir(security_dir):
+        os.mkdir(security_dir, 0700)
+    else:
+        os.chmod(security_dir, 0700)
+    return security_dir
+        
 #****************************************************************************
 # strings and text
 

IPython/history.py

@@ -8,6 +8,7 b' import os'
 
 # IPython imports
 from IPython.genutils import Term, ask_yes_no
+import IPython.ipapi
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
@@ -222,6 +223,7 b' class ShadowHist:'
         # cmd => idx mapping
         self.curidx = 0
         self.db = db
+        self.disabled = False
     
     def inc_idx(self):
         idx = self.db.get('shadowhist_idx', 1)
@@ -229,12 +231,19 b' class ShadowHist:'
         return idx
         
     def add(self, ent):
+        if self.disabled:
+            return
+        try:
         old = self.db.hget('shadowhist', ent, _sentinel)
         if old is not _sentinel:
             return
         newidx = self.inc_idx()
         #print "new",newidx # dbg
         self.db.hset('shadowhist',ent, newidx)
+        except:
+            IPython.ipapi.get().IP.showtraceback()
+            print "WARNING: disabling shadow history"
+            self.disabled = True
     
     def all(self):
         d = self.db.hdict('shadowhist')

IPython/hooks.py

@@ -25,7 +25,8 b' ip = IPython.ipapi.get()'
 def calljed(self,filename, linenum):
     "My editor hook calls the jed editor directly."
     print "Calling my own editor, jed ..."
-    os.system('jed +%d %s' % (linenum,filename))
+    if os.system('jed +%d %s' % (linenum,filename)) != 0:
+        raise ipapi.TryNext()
 
 ip.set_hook('editor', calljed)
 
@@ -84,7 +85,8 b' def editor(self,filename, linenum=None):'
         editor = '"%s"' % editor
         
     # Call the actual editor
-    os.system('%s %s %s' % (editor,linemark,filename))
+    if os.system('%s %s %s' % (editor,linemark,filename)) != 0:
+        raise ipapi.TryNext()
 
 import tempfile
 def fix_error_editor(self,filename,linenum,column,msg):
@@ -105,7 +107,8 b' def fix_error_editor(self,filename,linenum,column,msg):'
         return
     t = vim_quickfix_file()
     try:
-        os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name)
+        if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name):
+            raise ipapi.TryNext()
     finally:
         t.close()
 

IPython/iplib.py

@@ -1419,8 +1419,12 b' want to merge them back into the new files.""" % locals()'
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
+        try:
         self.hooks.fix_error_editor(e.filename,
             int0(e.lineno),int0(e.offset),e.msg)
+        except IPython.ipapi.TryNext:
+            warn('Could not open editor')
+            return False
         return True
         
     def edit_syntax_error(self):
@@ -1572,6 +1576,11 b' want to merge them back into the new files.""" % locals()'
             else:                
                 banner = self.BANNER+self.banner2
 
+        # if you run stuff with -c <cmd>, raw hist is not updated
+        # ensure that it's in sync
+        if len(self.input_hist) != len (self.input_hist_raw):
+            self.input_hist_raw = InputList(self.input_hist)
+
         while 1:
             try:
                 self.interact(banner)

IPython/kernel/config/__init__.py

@@ -15,17 +15,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+from os.path import join as pjoin
+
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
-from IPython.config.cutils import get_ipython_dir
+from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
-try:
-    ipython_dir = get_ipython_dir() + '/'
-except:
-    # This will defaults to the cwd
-    ipython_dir = ''
+security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
@@ -33,7 +31,7 b' except:'
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
-    furl_file = ipython_dir + 'ipcontroller-engine.furl'
+    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
@@ -63,16 +61,17 b' controller_config = dict('
 
     logfile = '',    # Empty means log to stdout
     import_statement = '',
+    reuse_furls = False, # If False, old furl files are deleted
 
     engine_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-engine.pem',
+        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = ipython_dir + 'ipcontroller-engine.furl',
+    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
@@ -83,12 +82,12 b' controller_config = dict('
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
@@ -97,7 +96,7 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-client.pem'
+        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
@@ -108,10 +107,10 b' controller_config = dict('
 client_config = dict(
     client_interfaces = dict(
         task = dict(
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )

IPython/kernel/contexts.py

@@ -8,8 +8,6 b' which can also be useful as templates for writing new, application-specific'
 managers.
 """
 
-from __future__ import with_statement
-
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------

IPython/kernel/controllerservice.py

@@ -50,7 +50,7 b' from IPython.kernel.engineservice import \\'
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
@@ -170,7 +170,7 b' class ControllerService(object, service.Service):'
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
-        ipdir = cutils.get_ipython_dir()
+        ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):

IPython/kernel/core/interpreter.py

@@ -680,6 +680,13 b' class Interpreter(object):'
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
 
+        # The compiler module does not like unicode. We need to convert
+        # it encode it:
+        if isinstance(python, unicode):
+            # Use the utf-8-sig BOM so the compiler detects this a UTF-8
+            # encode string.
+            python = '\xef\xbb\xbf' + python.encode('utf-8')
+
         # The compiler module will parse the code into an abstract syntax tree.
         ast = compiler.parse(python)
 

IPython/kernel/core/tests/test_redirectors.py

@@ -13,10 +13,17 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 
+# Stdlib imports
 import os
 from cStringIO import StringIO
 
+# Our own imports
+from IPython.testing import decorators as dec
 
+#-----------------------------------------------------------------------------
+# Test functions
+
[e-mail]_win32
 def test_redirector():
     """ Checks that the redirector can be used to do synchronous capture.
     """
@@ -33,9 +40,12 b' def test_redirector():'
         r.stop()
         raise
     r.stop()
-    assert out.getvalue() == "".join("%ic\n%i\n" %(i, i) for i in range(10))
+    result1 = out.getvalue()
+    result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
+    assert result1 == result2
 
 
[e-mail]_win32
 def test_redirector_output_trap():
     """ This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
@@ -54,8 +64,7 b' def test_redirector_output_trap():'
         trap.unset()
         raise
     trap.unset()
-    assert out.getvalue() == "".join("%ic\n%ip\n%i\n" %(i, i, i) 
-                                                    for i in range(10))
-
-    
+    result1 = out.getvalue()
+    result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
+    assert result1 == result2
 

IPython/kernel/engineconnector.py

@@ -18,7 +18,8 b' __docformat__ = "restructuredtext en"'
 import os
 import cPickle as pickle
 
-from twisted.python import log
+from twisted.python import log, failure
+from twisted.internet import defer
 
 from IPython.kernel.fcutil import find_furl
 from IPython.kernel.enginefc import IFCEngine
@@ -62,13 +63,17 b' class EngineConnector(object):'
             self.tub.startService()
         self.engine_service = engine_service
         self.engine_reference = IFCEngine(self.engine_service)
+        try:
         self.furl = find_furl(furl_or_file)
+        except ValueError:
+            return defer.fail(failure.Failure())
+            # return defer.fail(failure.Failure(ValueError('not a valid furl or furl file: %r' % furl_or_file)))
         d = self.tub.getReference(self.furl)
         d.addCallbacks(self._register, self._log_failure)
         return d
     
     def _log_failure(self, reason):
-        log.err('engine registration failed:')
+        log.err('EngineConnector: engine registration failed:')
         log.err(reason)
         return reason
 

IPython/kernel/scripts/ipcluster.py

@@ -1,324 +1,486 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-"""Start an IPython cluster conveniently, either locally or remotely.
+"""Start an IPython cluster = (controller + engines)."""
 
-Basic usage
------------
-
-For local operation, the simplest mode of usage is:
-
-  %prog -n N
-
-where N is the number of engines you want started.
-
-For remote operation, you must call it with a cluster description file:
-
-  %prog -f clusterfile.py
-
-The cluster file is a normal Python script which gets run via execfile().  You
-can have arbitrary logic in it, but all that matters is that at the end of the
-execution, it declares the variables 'controller', 'engines', and optionally
-'sshx'.  See the accompanying examples for details on what these variables must
-contain.
-
-
-Notes
------
-
-WARNING: this code is still UNFINISHED and EXPERIMENTAL!  It is incomplete,
-some listed options are not really implemented, and all of its interfaces are
-subject to change.
-
-When operating over SSH for a remote cluster, this program relies on the
-existence of a particular script called 'sshx'.  This script must live in the
-target systems where you'll be running your controller and engines, and is
-needed to configure your PATH and PYTHONPATH variables for further execution of
-python code at the other end of an SSH connection.  The script can be as simple
-as:
-
-#!/bin/sh
-. $HOME/.bashrc
-"$@"
-
-which is the default one provided by IPython.  You can modify this or provide
-your own.  Since it's quite likely that for different clusters you may need
-this script to configure things differently or that it may live in different
-locations, its full path can be set in the same file where you define the
-cluster setup.  IPython's order of evaluation for this variable is the
-following:
-
-  a) Internal default: 'sshx'.  This only works if it is in the default system
-  path which SSH sets up in non-interactive mode.
-
-  b) Environment variable: if $IPYTHON_SSHX is defined, this overrides the
-  internal default.
-
-  c) Variable 'sshx' in the cluster configuration file: finally, this will
-  override the previous two values.
- 
-This code is Unix-only, with precious little hope of any of this ever working
-under Windows, since we need SSH from the ground up, we background processes,
-etc.  Ports of this functionality to Windows are welcome.
-
-
-Call summary
-------------
-
-    %prog [options]
-"""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-# Stdlib imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
 import os
-import signal
+import re
 import sys
-import time
-
-from optparse import OptionParser
-from subprocess import Popen,call
+import signal
+pjoin = os.path.join
 
-#---------------------------------------------------------------------------
-# IPython imports
-#---------------------------------------------------------------------------
-from IPython.tools import utils
-from IPython.config import cutils
+from twisted.internet import reactor, defer
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python import failure, log
+from twisted.internet.error import ProcessDone, ProcessTerminated
+from twisted.internet.utils import getProcessOutput
 
-#---------------------------------------------------------------------------
-# Normal code begins
-#---------------------------------------------------------------------------
+from IPython.external import argparse
+from IPython.external import Itpl
+from IPython.kernel.twistedutil import gatherBoth
+from IPython.kernel.util import printer
+from IPython.genutils import get_ipython_dir, num_cpus
 
-def parse_args():
-    """Parse command line and return opts,args."""
+#-----------------------------------------------------------------------------
+# General process handling code
+#-----------------------------------------------------------------------------
 
-    parser = OptionParser(usage=__doc__)
-    newopt = parser.add_option  # shorthand
+def find_exe(cmd):
+    try:
+        import win32api
+    except ImportError:
+        raise ImportError('you need to have pywin32 installed for this to work')
+    else:
+        (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd)
+        return path
 
-    newopt("--controller-port", type="int", dest="controllerport",
-           help="the TCP port the controller is listening on")
+class ProcessStateError(Exception):
+    pass
 
-    newopt("--controller-ip", type="string", dest="controllerip",
-           help="the TCP ip address of the controller")
+class UnknownStatus(Exception):
+    pass
 
-    newopt("-n", "--num", type="int", dest="n",default=2,
-           help="the number of engines to start")
+class LauncherProcessProtocol(ProcessProtocol):
+    """
+    A ProcessProtocol to go with the ProcessLauncher.
+    """
+    def __init__(self, process_launcher):
+        self.process_launcher = process_launcher
+    
+    def connectionMade(self):
+        self.process_launcher.fire_start_deferred(self.transport.pid)
+    
+    def processEnded(self, status):
+        value = status.value
+        if isinstance(value, ProcessDone):
+            self.process_launcher.fire_stop_deferred(0)
+        elif isinstance(value, ProcessTerminated):
+            self.process_launcher.fire_stop_deferred(
+                {'exit_code':value.exitCode,
+                 'signal':value.signal,
+                 'status':value.status
+                }
+            )
+        else:
+            raise UnknownStatus("unknown exit status, this is probably a bug in Twisted")
 
-    newopt("--engine-port", type="int", dest="engineport",
-           help="the TCP port the controller will listen on for engine "
-           "connections")
+    def outReceived(self, data):
+        log.msg(data)
     
-    newopt("--engine-ip", type="string", dest="engineip",
-           help="the TCP ip address the controller will listen on "
-           "for engine connections")
+    def errReceived(self, data):
+        log.err(data)
 
-    newopt("--mpi", type="string", dest="mpi",
-           help="use mpi with package:  for instance --mpi=mpi4py")
+class ProcessLauncher(object):
+    """
+    Start and stop an external process in an asynchronous manner.
     
-    newopt("-l", "--logfile", type="string", dest="logfile",
-           help="log file name")
+    Currently this uses deferreds to notify other parties of process state
+    changes.  This is an awkward design and should be moved to using
+    a formal NotificationCenter.
+    """
+    def __init__(self, cmd_and_args):
+        self.cmd = cmd_and_args[0]
+        self.args = cmd_and_args
+        self._reset()
+    
+    def _reset(self):
+        self.process_protocol = None
+        self.pid = None
+        self.start_deferred = None
+        self.stop_deferreds = []
+        self.state = 'before' # before, running, or after
+    
+    @property
+    def running(self):
+        if self.state == 'running':
+            return True
+        else:
+            return False
+    
+    def fire_start_deferred(self, pid):
+        self.pid = pid
+        self.state = 'running'
+        log.msg('Process %r has started with pid=%i' % (self.args, pid))
+        self.start_deferred.callback(pid)
+    
+    def start(self):
+        if self.state == 'before':
+            self.process_protocol = LauncherProcessProtocol(self)
+            self.start_deferred = defer.Deferred()
+            self.process_transport = reactor.spawnProcess(
+                self.process_protocol,
+                self.cmd,
+                self.args,
+                env=os.environ
+            )
+            return self.start_deferred
+        else:
+            s = 'the process has already been started and has state: %r' % \
+                self.state
+            return defer.fail(ProcessStateError(s))
+    
+    def get_stop_deferred(self):
+        if self.state == 'running' or self.state == 'before':
+            d = defer.Deferred()
+            self.stop_deferreds.append(d)
+            return d
+        else:
+            s = 'this process is already complete'
+            return defer.fail(ProcessStateError(s))
 
-    newopt('-f','--cluster-file',dest='clusterfile',
-           help='file describing a remote cluster')
+    def fire_stop_deferred(self, exit_code):
+        log.msg('Process %r has stopped with %r' % (self.args, exit_code))
+        self.state = 'after'
+        for d in self.stop_deferreds:
+            d.callback(exit_code)
 
-    return parser.parse_args()
+    def signal(self, sig):
+        """
+        Send a signal to the process.
 
-def numAlive(controller,engines):
-    """Return the number of processes still alive."""
-    retcodes = [controller.poll()] + \
-               [e.poll() for e in engines]
-    return retcodes.count(None)
+        The argument sig can be ('KILL','INT', etc.) or any signal number.
+        """
+        if self.state == 'running':
+            self.process_transport.signalProcess(sig)
 
-stop = lambda pid: os.kill(pid,signal.SIGINT)
-kill = lambda pid: os.kill(pid,signal.SIGTERM)
+    # def __del__(self):
+    #     self.signal('KILL')
 
-def cleanup(clean,controller,engines):
-    """Stop the controller and engines with the given cleanup method."""
+    def interrupt_then_kill(self, delay=1.0):
+        self.signal('INT')
+        reactor.callLater(delay, self.signal, 'KILL')
     
-    for e in engines:
-        if e.poll() is None:
-            print 'Stopping engine, pid',e.pid
-            clean(e.pid)
-    if controller.poll() is None:
-        print 'Stopping controller, pid',controller.pid
-        clean(controller.pid)
 
+#-----------------------------------------------------------------------------
+# Code for launching controller and engines
+#-----------------------------------------------------------------------------
 
-def ensureDir(path):
-    """Ensure a directory exists or raise an exception."""
-    if not os.path.isdir(path):
-        os.makedirs(path)
 
+class ControllerLauncher(ProcessLauncher):
 
-def startMsg(control_host,control_port=10105):
-    """Print a startup message"""
-    print
-    print 'Your cluster is up and running.'
-    print
-    print 'For interactive use, you can make a MultiEngineClient with:'
-    print
-    print 'from IPython.kernel import client'
-    print "mec = client.MultiEngineClient()"
-    print
-    print 'You can then cleanly stop the cluster from IPython using:'
-    print
-    print 'mec.kill(controller=True)'
-    print
+    def __init__(self, extra_args=None):
+        if sys.platform == 'win32':
+            args = [find_exe('ipcontroller.bat')]
+        else:
+            args = ['ipcontroller']
+        self.extra_args = extra_args
+        if extra_args is not None:
+            args.extend(extra_args)
 
+        ProcessLauncher.__init__(self, args)
     
-def clusterLocal(opt,arg):
-    """Start a cluster on the local machine."""
     
-    # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
-    pjoin = os.path.join
+class EngineLauncher(ProcessLauncher):
 
-    logfile = opt.logfile
-    if logfile is None:
-        logdir_base = pjoin(ipdir,'log')
-        ensureDir(logdir_base)
-        logfile = pjoin(logdir_base,'ipcluster-')
-
-    print 'Starting controller:',
-    controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
-    print 'Controller PID:',controller.pid
-
-    print 'Starting engines:   ',
-    time.sleep(5)
-
-    englogfile = '%s%s-' % (logfile,controller.pid)
-    mpi = opt.mpi
-    if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
-        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', 
-            mpi, '--logfile',englogfile])]
-        # engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
-    else: # do what we would normally do
-        engines = [ Popen(['ipengine','--logfile',englogfile])
-                    for i in range(opt.n) ]
-    eids = [e.pid for e in engines]
-    print 'Engines PIDs:  ',eids
-    print 'Log files: %s*' % englogfile
-    
-    proc_ids = eids + [controller.pid]
-    procs = engines + [controller]
-
-    grpid = os.getpgrp()
-    try:
-        startMsg('127.0.0.1')
-        print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
-        print
-        print 'kill -INT',grpid
-        print
-        try:
-            while True:
-                time.sleep(5)
-        except:
-            pass
-    finally:
-        print 'Stopping cluster.  Cleaning up...'
-        cleanup(stop,controller,engines)
-        for i in range(4):
-            time.sleep(i+2)
-            nZombies = numAlive(controller,engines)
-            if  nZombies== 0:
-                print 'OK: All processes cleaned up.'
-                break
-            print 'Trying again, %d processes did not stop...' % nZombies
-            cleanup(kill,controller,engines)
-            if numAlive(controller,engines) == 0:
-                print 'OK: All processes cleaned up.'
-                break
+    def __init__(self, extra_args=None):
+        if sys.platform == 'win32':
+            args = [find_exe('ipengine.bat')]
         else:
-            print '*'*75
-            print 'ERROR: could not kill some processes, try to do it',
-            print 'manually.'
-            zombies = []
-            if controller.returncode is None:
-                print 'Controller is alive: pid =',controller.pid
-                zombies.append(controller.pid)
-            liveEngines = [ e for e in engines if e.returncode is None ]
-            for e in liveEngines:
-                print 'Engine is alive:     pid =',e.pid
-                zombies.append(e.pid)
-            print
-            print 'Zombie summary:',' '.join(map(str,zombies))
-
-def clusterRemote(opt,arg):
-    """Start a remote cluster over SSH"""
-
-    # Load the remote cluster configuration
-    clConfig = {}
-    execfile(opt.clusterfile,clConfig)
-    contConfig = clConfig['controller']
-    engConfig = clConfig['engines']
-    # Determine where to find sshx:
-    sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
-    
-    # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
-    pjoin = os.path.join
-
-    logfile = opt.logfile
-    if logfile is None:
-        logdir_base = pjoin(ipdir,'log')
-        ensureDir(logdir_base)
-        logfile = pjoin(logdir_base,'ipcluster')
-
-    # Append this script's PID to the logfile name always
-    logfile = '%s-%s' % (logfile,os.getpid())
-    
-    print 'Starting controller:'
-    # Controller data:
-    xsys = os.system
-
-    contHost = contConfig['host']
-    contLog = '%s-con-%s-' % (logfile,contHost)
-    cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
-          (contHost,sshx,contLog)
-    #print 'cmd:<%s>' % cmd  # dbg
-    xsys(cmd)
-    time.sleep(2)
-
-    print 'Starting engines:   '
-    for engineHost,engineData in engConfig.iteritems():
-        if isinstance(engineData,int):
-            numEngines = engineData
+            args = ['ipengine']
+        self.extra_args = extra_args
+        if extra_args is not None:
+            args.extend(extra_args)
+        
+        ProcessLauncher.__init__(self, args)
+
+
+class LocalEngineSet(object):
+    
+    def __init__(self, extra_args=None):
+        self.extra_args = extra_args
+        self.launchers = []
+    
+    def start(self, n):
+        dlist = []
+        for i in range(n):
+            el = EngineLauncher(extra_args=self.extra_args)
+            d = el.start()
+            self.launchers.append(el)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_start)
+        return dfinal
+    
+    def _handle_start(self, r):
+        log.msg('Engines started with pids: %r' % r)
+        return r
+    
+    def _handle_stop(self, r):
+        log.msg('Engines received signal: %r' % r)
+        return r
+    
+    def signal(self, sig):
+        dlist = []
+        for el in self.launchers:
+            d = el.get_stop_deferred()
+            dlist.append(d)
+            el.signal(sig)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_stop)
+        return dfinal
+    
+    def interrupt_then_kill(self, delay=1.0):
+        dlist = []
+        for el in self.launchers:
+            d = el.get_stop_deferred()
+            dlist.append(d)
+            el.interrupt_then_kill(delay)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_stop)
+        return dfinal
+
+
+class BatchEngineSet(object):
+    
+    # Subclasses must fill these in.  See PBSEngineSet
+    submit_command = ''
+    delete_command = ''
+    job_id_regexp = ''
+    
+    def __init__(self, template_file, **kwargs):
+        self.template_file = template_file
+        self.context = {}
+        self.context.update(kwargs)
+        self.batch_file = self.template_file+'-run'
+
+    def parse_job_id(self, output):
+        m = re.match(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
         else:
-            raise NotImplementedError('port configuration not finished for engines')
-
-        print 'Sarting %d engines on %s' % (numEngines,engineHost)
-        engLog = '%s-eng-%s-' % (logfile,engineHost)
-        for i in range(numEngines):
-            cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
-                      (engineHost,sshx,contHost,engLog)
-            #print 'cmd:<%s>' % cmd  # dbg
-            xsys(cmd)
-        # Wait after each host a little bit
-        time.sleep(1)
-            
-    startMsg(contConfig['host'])
+            raise Exception("job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+    
+    def write_batch_script(self, n):
+        self.context['n'] = n
+        template = open(self.template_file, 'r').read()
+        log.msg('Using template for batch script: %s' % self.template_file)
+        script_as_string = Itpl.itplns(template, self.context)
+        log.msg('Writing instantiated batch script: %s' % self.batch_file)
+        f = open(self.batch_file,'w')
+        f.write(script_as_string)
+        f.close()
+
+    def handle_error(self, f):
+        f.printTraceback()
+        f.raiseException()
+    
+    def start(self, n):
+        self.write_batch_script(n)
+        d = getProcessOutput(self.submit_command,
+            [self.batch_file],env=os.environ)
+        d.addCallback(self.parse_job_id)
+        d.addErrback(self.handle_error)
+        return d
+        
+    def kill(self):
+        d = getProcessOutput(self.delete_command,
+            [self.job_id],env=os.environ)
+        return d
+
+class PBSEngineSet(BatchEngineSet):
+    
+    submit_command = 'qsub'
+    delete_command = 'qdel'
+    job_id_regexp = '\d+'
+    
+    def __init__(self, template_file, **kwargs):
+        BatchEngineSet.__init__(self, template_file, **kwargs)
+
+
+#-----------------------------------------------------------------------------
+# Main functions for the different types of clusters
+#-----------------------------------------------------------------------------
+
+# TODO:
+# The logic in these codes should be moved into classes like LocalCluster
+# MpirunCluster, PBSCluster, etc.  This would remove alot of the duplications.
+# The main functions should then just parse the command line arguments, create
+# the appropriate class and call a 'start' method.
+
+def main_local(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+        cont_args.append('-x')
+    if args.y:
+        cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(cont_pid):
+        engine_args = []
+        engine_args.append('--logfile=%s' % \
+            pjoin(args.logdir,'ipengine%s-' % cont_pid))
+        eset = LocalEngineSet(extra_args=engine_args)
+        def shutdown(signum, frame):
+            log.msg('Stopping local cluster')
+            # We are still playing with the times here, but these seem
+            # to be reliable in allowing everything to exit cleanly.
+            eset.interrupt_then_kill(0.5)
+            cl.interrupt_then_kill(0.5)
+            reactor.callLater(1.0, reactor.stop)
+        signal.signal(signal.SIGINT,shutdown)
+        d = eset.start(args.n)
+        return d
+    def delay_start(cont_pid):
+        # This is needed because the controller doesn't start listening
+        # right when it starts and the controller needs to write
+        # furl files for the engine to pick up
+        reactor.callLater(1.0, start_engines, cont_pid)
+    dstart.addCallback(delay_start)
+    dstart.addErrback(lambda f: f.raiseException())
+
+def main_mpirun(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+        cont_args.append('-x')
+    if args.y:
+        cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(cont_pid):
+        raw_args = ['mpirun']
+        raw_args.extend(['-n',str(args.n)])
+        raw_args.append('ipengine')
+        raw_args.append('-l')
+        raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid))
+        if args.mpi:
+            raw_args.append('--mpi=%s' % args.mpi)
+        eset = ProcessLauncher(raw_args)
+        def shutdown(signum, frame):
+            log.msg('Stopping local cluster')
+            # We are still playing with the times here, but these seem
+            # to be reliable in allowing everything to exit cleanly.
+            eset.interrupt_then_kill(1.0)
+            cl.interrupt_then_kill(1.0)
+            reactor.callLater(2.0, reactor.stop)
+        signal.signal(signal.SIGINT,shutdown)
+        d = eset.start()
+        return d
+    def delay_start(cont_pid):
+        # This is needed because the controller doesn't start listening
+        # right when it starts and the controller needs to write
+        # furl files for the engine to pick up
+        reactor.callLater(1.0, start_engines, cont_pid)
+    dstart.addCallback(delay_start)
+    dstart.addErrback(lambda f: f.raiseException())
+
+def main_pbs(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+	cont_args.append('-x')
+    if args.y:
+	cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(r):
+        pbs_set =  PBSEngineSet(args.pbsscript)
+        def shutdown(signum, frame):
+            log.msg('Stopping pbs cluster')
+            d = pbs_set.kill()
+            d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
+            d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
+        signal.signal(signal.SIGINT,shutdown)
+        d = pbs_set.start(args.n)
+        return d
+    dstart.addCallback(start_engines)
+    dstart.addErrback(lambda f: f.raiseException())
+
+
+def get_args():
+    base_parser = argparse.ArgumentParser(add_help=False)
+    base_parser.add_argument(
+        '-x',
+        action='store_true',
+        dest='x',
+        help='turn off client security'
+    )
+    base_parser.add_argument(
+        '-y',
+        action='store_true',
+        dest='y',
+        help='turn off engine security'
+    )
+    base_parser.add_argument(
+        "--logdir", 
+        type=str,
+        dest="logdir",
+        help="directory to put log files (default=$IPYTHONDIR/log)",
+        default=pjoin(get_ipython_dir(),'log')
+    )
+    base_parser.add_argument(
+        "-n",
+        "--num", 
+        type=int,
+        dest="n",
+        default=2,
+        help="the number of engines to start"
+    )
+    
+    parser = argparse.ArgumentParser(
+        description='IPython cluster startup.  This starts a controller and\
+        engines using various approaches.  THIS IS A TECHNOLOGY PREVIEW AND\
+        THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.'
+    )
+    subparsers = parser.add_subparsers(
+        help='available cluster types.  For help, do "ipcluster TYPE --help"')
+    
+    parser_local = subparsers.add_parser(
+        'local',
+        help='run a local cluster',
+        parents=[base_parser]
+    )
+    parser_local.set_defaults(func=main_local)
+    
+    parser_mpirun = subparsers.add_parser(
+        'mpirun',
+        help='run a cluster using mpirun',
+        parents=[base_parser]
+    )
+    parser_mpirun.add_argument(
+        "--mpi",
+        type=str,
+        dest="mpi", # Don't put a default here to allow no MPI support
+        help="how to call MPI_Init (default=mpi4py)"
+    )
+    parser_mpirun.set_defaults(func=main_mpirun)
+    
+    parser_pbs = subparsers.add_parser(
+        'pbs', 
+        help='run a pbs cluster',
+        parents=[base_parser]
+    )
+    parser_pbs.add_argument(
+        '--pbs-script',
+        type=str, 
+        dest='pbsscript',
+        help='PBS script template',
+        default='pbs.template'
+    )
+    parser_pbs.set_defaults(func=main_pbs)
+    args = parser.parse_args()
+    return args
         
 def main():
-    """Main driver for the two big options: local or remote cluster."""
-    
-    opt,arg = parse_args()
-
-    clusterfile = opt.clusterfile
-    if clusterfile:
-        clusterRemote(opt,arg)
-    else:
-        clusterLocal(opt,arg)
-        
+    args = get_args()
+    reactor.callWhenRunning(args.func, args)
+    log.startLogging(sys.stdout)
+    reactor.run()
             
 if __name__=='__main__':
     main()

IPython/kernel/scripts/ipcontroller.py

@@ -64,7 +64,10 b' def make_tub(ip, port, secure, cert_file):'
         if have_crypto:
             tub = Tub(certFile=cert_file)
         else:
-            raise SecurityError("OpenSSL is not available, so we can't run in secure mode, aborting")
+            raise SecurityError("""
+OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
+Try running without security using 'ipcontroller -xy'.
+""")
     else:
         tub = UnauthenticatedTub()
     
@@ -202,6 +205,17 b' def start_controller():'
         except:
             log.msg("Error running import_statement: %s" % cis)
     
+    # Delete old furl files unless the reuse_furls is set
+    reuse = config['controller']['reuse_furls']
+    if not reuse:
+        paths = (config['controller']['engine_furl_file'],
+            config['controller']['controller_interfaces']['task']['furl_file'],
+            config['controller']['controller_interfaces']['multiengine']['furl_file']
+        )
+        for p in paths:
+            if os.path.isfile(p):
+                os.remove(p)
+    
     # Create the service hierarchy
     main_service = service.MultiService()
     # The controller service
@@ -316,6 +330,12 b' def init_config():'
         dest="ipythondir",
         help="look for config files and profiles in this directory"
     )
+    parser.add_option(
+        "-r",
+        action="store_true",
+        dest="reuse_furls",
+        help="try to reuse all furl files"
+    )
     
     (options, args) = parser.parse_args()
     
@@ -349,6 +369,8 b' def init_config():'
         config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
     if options.engine_furl_file is not None:
         config['controller']['engine_furl_file'] = options.engine_furl_file
+    if options.reuse_furls is not None:
+            config['controller']['reuse_furls'] = options.reuse_furls
 
     if options.logfile is not None:
         config['controller']['logfile'] = options.logfile

IPython/kernel/scripts/ipengine.py

@@ -91,7 +91,7 b' def start_engine():'
         try:
             engine_service.execute(shell_import_statement)
         except:
-            log.msg("Error running import_statement: %s" % sis)
+            log.msg("Error running import_statement: %s" % shell_import_statement)
     
     # Create the service hierarchy
     main_service = service.MultiService()
@@ -105,8 +105,13 b' def start_engine():'
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
+    log.msg("Using furl file: %s" % furl_file)
     d = engine_connector.connect_to_controller(engine_service, furl_file)
-    d.addErrback(lambda _: reactor.stop())
+    def handle_error(f):
+        log.err(f)
+        if reactor.running:
+            reactor.stop()
+    d.addErrback(handle_error)
     
     reactor.run()
 

IPython/kernel/tests/engineservicetest.py

@@ -245,7 +245,7 b' class IEngineSerializedTestCase(object):'
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireCore."""
+        """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
@@ -288,7 +288,7 b' class IEngineQueuedTestCase(object):'
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireQueued."""
+        """Does self.engine have the methods and attributes in IEngineQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
@@ -326,7 +326,7 b' class IEnginePropertiesTestCase(object):'
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireProperties."""
+        """Does self.engine have the methods and attributes in IEngineProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     

IPython/kernel/tests/multienginetest.py

@@ -20,7 +20,6 b' from twisted.internet import defer'
 from IPython.kernel import engineservice as es
 from IPython.kernel import multiengine as me
 from IPython.kernel import newserialized
-from IPython.kernel.error import NotDefined
 from IPython.testing import util
 from IPython.testing.parametric import parametric, Parametric
 from IPython.kernel import newserialized

IPython/kernel/tests/test_contexts.py

@@ -1,4 +1,6 b''
-from __future__ import with_statement
+#from __future__ import with_statement
+
+# XXX This file is currently disabled to preserve 2.4 compatibility.
 
 #def test_simple():
 if 0:
@@ -25,17 +27,17 b' if 0:'
 
     mec.pushAll()
 
-    with parallel as pr:
-        # A comment
-        remote()  # this means the code below only runs remotely
-        print 'Hello remote world'
-        x = range(10)
-        # Comments are OK
-        # Even misindented.
-        y = x+1
+    ## with parallel as pr:
+    ##     # A comment
+    ##     remote()  # this means the code below only runs remotely
+    ##     print 'Hello remote world'
+    ##     x = range(10)
+    ##     # Comments are OK
+    ##     # Even misindented.
+    ##     y = x+1
 
 
-    with pfor('i',sequence) as pr:
-        print x[i]
+    ## with pfor('i',sequence) as pr:
+    ##     print x[i]
 
     print pr.x + pr.y

IPython/kernel/tests/test_controllerservice.py

@@ -30,8 +30,9 b' try:'
     from controllertest import IControllerCoreTestCase
     from IPython.testing.util import DeferredTestCase
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
     class BasicControllerServiceTest(DeferredTestCase,
         IControllerCoreTestCase):
     

IPython/kernel/tests/test_enginefc.py

@@ -37,9 +37,10 b' try:'
         IEngineSerializedTestCase, \
         IEngineQueuedTestCase
 except ImportError:
-    print "we got an error!!!"
-    raise
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+
     class EngineFCTest(DeferredTestCase, 
                        IEngineCoreTestCase, 
                        IEngineSerializedTestCase,

IPython/kernel/tests/test_engineservice.py

@@ -35,8 +35,10 b' try:'
         IEngineQueuedTestCase, \
         IEnginePropertiesTestCase
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")    
+
+
     class BasicEngineServiceTest(DeferredTestCase,
                                  IEngineCoreTestCase, 
                                  IEngineSerializedTestCase,

IPython/kernel/tests/test_multiengine.py

@@ -23,8 +23,10 b' try:'
     from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
         ISynchronousMultiEngineTestCase)
 except ImportError:
-    pass
-else:    
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+ 
     class BasicMultiEngineTestCase(DeferredTestCase, IMultiEngineTestCase):
     
         def setUp(self):

IPython/kernel/tests/test_multienginefc.py

@@ -30,8 +30,8 b' try:'
     from IPython.kernel.error import CompositeError
     from IPython.kernel.util import printer
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
     
     def _raise_it(f):
         try:

IPython/kernel/tests/test_newserialized.py

@@ -28,8 +28,9 b' try:'
         SerializeIt, \
         UnSerializeIt
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
     #-------------------------------------------------------------------------------
     # Tests
     #-------------------------------------------------------------------------------
@@ -99,4 +100,3 b' else:'
                 self.assert_(a.shape == final.shape)
             
             
-    
  No newline at end of file

IPython/kernel/tests/test_pendingdeferred.py

@@ -25,8 +25,8 b' try:'
     from IPython.kernel import error
     from IPython.kernel.util import printer
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
     
     class Foo(object):
     

IPython/kernel/tests/test_task.py

@@ -26,8 +26,9 b' try:'
     from IPython.testing.util import DeferredTestCase
     from IPython.kernel.tests.tasktest import ITaskControllerTestCase
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
     #-------------------------------------------------------------------------------
     # Tests
     #-------------------------------------------------------------------------------

IPython/kernel/tests/test_taskfc.py

@@ -33,8 +33,9 b' try:'
     from IPython.kernel.error import CompositeError
     from IPython.kernel.parallelfunction import ParallelFunction
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
 
     #-------------------------------------------------------------------------------
     # Tests

IPython/testing/decorators.py

@@ -8,6 +8,10 b' the decorator, in order to preserve metadata such as function name,'
 setup and teardown functions and so on - see nose.tools for more
 information.
 
+This module provides a set of useful decorators meant to be ready to use in
+your own tests.  See the bottom of the file for the ready-made ones, and if you
+find yourself writing a new one that may be of generic use, add it here.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
@@ -15,6 +19,7 b' code will be added at the bottom if we end up extending this.'
 
 # Stdlib imports
 import inspect
+import sys
 
 # Third-party imports
 
@@ -120,16 +125,36 b" skip_doctest = make_label_dec('skip_doctest',"
     omit from testing, while preserving the docstring for introspection, help,
     etc.""")                              
 
+def skip(msg=''):
+    """Decorator - mark a test function for skipping from test suite.
+
+    This function *is* already a decorator, it is not a factory like
+    make_label_dec or some of those in decorators_numpy.
+
+    :Parameters:
 
-def skip(func):
-    """Decorator - mark a test function for skipping from test suite."""
+      func : function
+        Test function to be skipped
+
+      msg : string
+        Optional message to be added.
+      """
 
     import nose
     
+    def inner(func):
+
     def wrapper(*a,**k):
-        raise nose.SkipTest("Skipping test for function: %s" %
-                            func.__name__)
+            if msg: out = '\n'+msg
+            else: out = ''
+            raise nose.SkipTest("Skipping test for function: %s%s" %
+                                (func.__name__,out))
     
     return apply_wrapper(wrapper,func)
 
+    return inner
 
+# Decorators to skip certain tests on specific platforms.
+skip_win32 = skipif(sys.platform=='win32',"This test does not run under Windows")
+skip_linux = skipif(sys.platform=='linux2',"This test does not run under Linux")
+skip_osx = skipif(sys.platform=='darwin',"This test does not run under OSX")

IPython/testing/plugin/Makefile

@@ -1,6 +1,5 b''
 # Set this prefix to where you want to install the plugin
-PREFIX=~/usr/local
-PREFIX=~/tmp/local
+PREFIX=/usr/local
 
 NOSE0=nosetests -vs --with-doctest --doctest-tests --detailed-errors
 NOSE=nosetests -vvs --with-ipdoctest --doctest-tests --doctest-extension=txt \

IPython/testing/plugin/ipdoctest.py

@@ -658,6 +658,24 b' class ExtensionDoctest(doctests.Doctest):'
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
+        parser.add_option('--doctest-tests', action='store_true',
+                          dest='doctest_tests',
+                          default=env.get('NOSE_DOCTEST_TESTS',True),
+                          help="Also look for doctests in test modules. "
+                          "Note that classes, methods and functions should "
+                          "have either doctests or non-doctest tests, "
+                          "not both. [NOSE_DOCTEST_TESTS]")
+        parser.add_option('--doctest-extension', action="append",
+                          dest="doctestExtension",
+                          help="Also look for doctests in files with "
+                          "this extension [NOSE_DOCTEST_EXTENSION]")
+        # Set the default as a list, if given in env; otherwise
+        # an additional value set on the command line will cause
+        # an error.
+        env_setting = env.get('NOSE_DOCTEST_EXTENSION')
+        if env_setting is not None:
+            parser.set_defaults(doctestExtension=tolist(env_setting))
+
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
@@ -743,16 +761,19 b' class ExtensionDoctest(doctests.Doctest):'
         Modified version that accepts extension modules as valid containers for
         doctests.
         """
-        #print 'Filename:',filename  # dbg
+        print 'Filename:',filename  # dbg
 
         # XXX - temporarily hardcoded list, will move to driver later
         exclude = ['IPython/external/',
-                   'IPython/Extensions/ipy_',
                    'IPython/platutils_win32',
                    'IPython/frontend/cocoa',
                    'IPython_doctest_plugin',
                    'IPython/Gnuplot',
-                   'IPython/Extensions/PhysicalQIn']
+                   'IPython/Extensions/ipy_',
+                   'IPython/Extensions/PhysicalQIn',
+                   'IPython/Extensions/scitedirector',
+                   'IPython/testing/plugin',
+                   ]
 
         for fex in exclude:
             if fex in filename:  # substring
@@ -782,3 +803,4 b' class IPythonDoctest(ExtensionDoctest):'
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
+

IPython/testing/plugin/test_refs.py

@@ -1,147 +1,19 b''
+"""Some simple tests for the plugin while running scripts.
+"""
 # Module imports
 # Std lib
 import inspect
 
-# Third party
-
 # Our own
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
-# Utilities
-
-# Note: copied from OInspect, kept here so the testing stuff doesn't create
-# circular dependencies and is easier to reuse.
-def getargspec(obj):
-    """Get the names and default values of a function's arguments.
-
-    A tuple of four things is returned: (args, varargs, varkw, defaults).
-    'args' is a list of the argument names (it may contain nested lists).
-    'varargs' and 'varkw' are the names of the * and ** arguments or None.
-    'defaults' is an n-tuple of the default values of the last n arguments.
-
-    Modified version of inspect.getargspec from the Python Standard
-    Library."""
-
-    if inspect.isfunction(obj):
-        func_obj = obj
-    elif inspect.ismethod(obj):
-        func_obj = obj.im_func
-    else:
-        raise TypeError, 'arg is not a Python function'
-    args, varargs, varkw = inspect.getargs(func_obj.func_code)
-    return args, varargs, varkw, func_obj.func_defaults
-
-#-----------------------------------------------------------------------------
 # Testing functions
 
 def test_trivial():
     """A trivial passing test."""
     pass
 
-
[e-mail]
-def test_deliberately_broken():
-    """A deliberately broken test - we want to skip this one."""
-    1/0
-
-
-# Verify that we can correctly skip the doctest for a function at will, but
-# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
-def doctest_bad(x,y=1,**k):
-    """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    print 'x:',x
-    print 'y:',y
-    print 'k:',k
-
-
-def call_doctest_bad():
-    """Check that we can still call the decorated functions.
-    
-    >>> doctest_bad(3,y=4)
-    x: 3
-    y: 4
-    k: {}
-    """
-    pass
-
-
-# Doctest skipping should work for class methods too
-class foo(object):
-    """Foo
-
-    Example:
-
-    >>> 1+1
-    2
-    """
-
-    @dec.skip_doctest
-    def __init__(self,x):
-        """Make a foo.
-
-        Example:
-
-        >>> f = foo(3)
-        junk
-        """
-        print 'Making a foo.'
-        self.x = x
-        
-    @dec.skip_doctest
-    def bar(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        >>> f.bar(0)
-        boom!
-        >>> 1/0
-        bam!
-        """
-        return 1/y
-
-    def baz(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        Making a foo.
-        >>> f.baz(3)
-        True
-        """
-        return self.x==y
-
-
-def test_skip_dt_decorator():
-    """Doctest-skipping decorator should preserve the docstring.
-    """
-    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
-    check = """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    # Fetch the docstring from doctest_bad after decoration.
-    val = doctest_bad.__doc__
-    
-    assert check==val,"doctest_bad docstrings don't match"
-
-
-def test_skip_dt_decorator2():
-    """Doctest-skipping decorator should preserve function signature.
-    """
-    # Hardcoded correct answer
-    dtargs = (['x', 'y'], None, 'k', (1,))
-    # Introspect out the value
-    dtargsr = getargspec(doctest_bad)
-    assert dtargsr==dtargs, \
-           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
-
-
 def doctest_run():
     """Test running a trivial script.
 
@@ -149,7 +21,6 b' def doctest_run():'
     x is: 1
     """
     
-#@dec.skip_doctest
 def doctest_runvars():
     """Test that variables defined in scripts get loaded correcly via %run.
 

IPython/testing/tutils.py

@@ -9,6 +9,7 b' Utilities for testing code.'
 # testing machinery from snakeoil that were good have already been merged into
 # the nose plugin, so this can be taken away soon.  Leave a warning for now,
 # we'll remove it in a later release (around 0.10 or so).
+
 from warnings import warn
 warn('This will be removed soon.  Use IPython.testing.util instead',
      DeprecationWarning)

IPython/ultraTB.py

@@ -446,6 +446,7 b' class ListTB(TBTools):'
         Also lifted nearly verbatim from traceback.py
         """
         
+        have_filedata = False
         Colors = self.Colors
         list = []
         try:
@@ -459,8 +460,9 b' class ListTB(TBTools):'
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
-                    pass
+                    have_filedata = False
                 else:
+                    have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
@@ -492,6 +494,7 b' class ListTB(TBTools):'
                 list.append('%s\n' % str(stype))
 
         # vds:>>
+        if have_filedata:
         __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds:<<
 
@@ -809,10 +812,10 b' class VerboseTB(TBTools):'
 
         # vds: >>
         if records:
-             frame, file, lnum, func, lines, index = records[-1]
-             #print "file:", str(file), "linenb", str(lnum)         
-             file = abspath(file)
-             __IPYTHON__.hooks.synchronize_with_editor(file, lnum, 0)
+             filepath, lnum = records[-1][1:3]
+             #print "file:", str(file), "linenb", str(lnum) # dbg
+             filepath = os.path.abspath(filepath)
+             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string

MANIFEST.in

@@ -1,7 +1,6 b''
-include README_Windows.txt
-include win32_manual_post_install.py
 include ipython.py
 include setupbase.py
+include setupegg.py
 
 graft scripts
 
@@ -30,3 +29,4 b' global-exclude *.pyc'
 global-exclude .dircopy.log
 global-exclude .svn
 global-exclude .bzr
+global-exclude .hgignore

README.txt

@@ -1,11 +1,11 b''
-===============
-IPython1 README
-===============
- 
-.. contents::
+==============
+IPython README
+==============
 
 Overview
 ========
 
-Welcome to IPython.  New users should consult our documentation, which can be found
-in the docs/source subdirectory.
+Welcome to IPython. Our documentation can be found in the docs/source
+subdirectory. We also have ``.html`` and ``.pdf`` versions of this
+documentation available on the IPython `website <http://ipython.scipy.org>`_.
+

docs/Makefile

@@ -28,17 +28,16 b' help:'
 	@echo "dist         all, and then puts the results in dist/"
 
 clean:
-	-rm -rf build/*
+	-rm -rf build/* dist/*
 
 pdf: latex
 	cd build/latex && make all-pdf
 
 all: html pdf
 
-dist: all
+dist: clean all
 	mkdir -p dist
-	-rm -rf dist/*
-	ln build/latex/IPython.pdf dist/
+	ln build/latex/ipython.pdf dist/
 	cp -al build/html dist/
 	@echo "Build finished.  Final docs are in dist/"
 

docs/examples/kernel/mcpricer.py

@@ -40,3 +40,4 b' def main():'
 
 if __name__ == '__main__':
     main()
+

docs/examples/kernel/nwmerge.py

@@ -45,7 +45,10 b' def mergesort(list_of_lists, key=None):'
     for i, itr in enumerate(iter(pl) for pl in list_of_lists):
         try:
             item = itr.next()
-            toadd = (key(item), i, item, itr) if key else (item, i, itr)
+            if key:
+                toadd = (key(item), i, item, itr)
+            else:
+                toadd = (item, i, itr)
             heap.append(toadd)
         except StopIteration:
             pass

docs/examples/kernel/rmt.ipy

@@ -53,5 +53,5 b" if __name__ == '__main__':"
     %timeit -n1 -r1 parallelDiffs(rc, nmats, matsize)
 
     # Uncomment these to plot the histogram
-    import pylab
-    pylab.hist(parallelDiffs(rc,matsize,matsize))
+    # import pylab
+    # pylab.hist(parallelDiffs(rc,matsize,matsize))

docs/source/changes.txt

@@ -5,6 +5,89 b" What's new"
 ==========
 
 .. contents::
+..
+    1  Release 0.9.1
+    2  Release 0.9
+      2.1  New features
+      2.2  Bug fixes
+      2.3  Backwards incompatible changes
+      2.4  Changes merged in from IPython1
+        2.4.1  New features
+        2.4.2  Bug fixes
+        2.4.3  Backwards incompatible changes
+    3  Release 0.8.4
+    4  Release 0.8.3
+    5  Release 0.8.2
+    6  Older releases
+..
+
+Release dev
+===========
+
+New features
+------------
+
+* The wonderful TextMate editor can now be used with %edit on OS X.  Thanks
+  to Matt Foster for this patch.
+
+* Fully refactored :command:`ipcluster` command line program for starting
+  IPython clusters.  This new version is a complete rewrite and 1) is fully
+  cross platform (we now use Twisted's process management), 2) has much
+  improved performance, 3) uses subcommands for different types of clusters, 
+  4) uses argparse for parsing command line options, 5) has better support
+  for starting clusters using :command:`mpirun`, 6) has experimental support
+  for starting engines using PBS.  However, this new version of ipcluster
+  should be considered a technology preview.  We plan on changing the API
+  in significant ways before it is final.
+
+* The :mod:`argparse` module has been added to :mod:`IPython.external`.
+
+* Fully description of the security model added to the docs.
+
+* cd completer: show bookmarks if no other completions are available.
+
+* sh profile: easy way to give 'title' to prompt: assign to variable
+  '_prompt_title'. It looks like this::
+      
+        [~]|1> _prompt_title = 'sudo!'
+        sudo![~]|2>
+
+* %edit: If you do '%edit pasted_block', pasted_block
+  variable gets updated with new data (so repeated 
+  editing makes sense)
+
+Bug fixes
+---------
+
+* The ipengine and ipcontroller scripts now handle missing furl files
+  more gracefully by giving better error messages.
+
+* %rehashx: Aliases no longer contain dots. python3.0 binary
+  will create alias python30. Fixes:
+  #259716 "commands with dots in them don't work"
+
+* %cpaste: %cpaste -r repeats the last pasted block.
+  The block is assigned to pasted_block even if code
+  raises exception.
+
+Backwards incompatible changes
+------------------------------
+
+* The controller now has a ``-r`` flag that needs to be used if you want to
+  reuse existing furl files.  Otherwise they are deleted (the default).
+
+* Remove ipy_leo.py. "easy_install ipython-extension" to get it.
+  (done to decouple it from ipython release cycle)
+
+
+
+Release 0.9.1
+=============
+
+This release was quickly made to restore compatibility with Python 2.4, which
+version 0.9 accidentally broke.  No new features were introduced, other than
+some additional testing support for internal use.
+
 
 Release 0.9
 ===========
@@ -12,89 +95,170 b' Release 0.9'
 New features
 ------------
 
+* All furl files and security certificates are now put in a read-only
+  directory named ~./ipython/security.
+
+* A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that
+  determines the user's IPython directory in a robust manner.
+
+* Laurent's WX application has been given a top-level script called
+  ipython-wx, and it has received numerous fixes. We expect this code to be
+  architecturally better integrated with Gael's WX 'ipython widget' over the
+  next few releases.
+
+* The Editor synchronization work by Vivian De Smedt has been merged in.  This
+  code adds a number of new editor hooks to synchronize with editors under
+  Windows.
+
+* A new, still experimental but highly functional, WX shell by Gael Varoquaux.
+  This work was sponsored by Enthought, and while it's still very new, it is
+  based on a more cleanly organized arhictecture of the various IPython
+  components. We will continue to develop this over the next few releases as a
+  model for GUI components that use IPython.
+
+* Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework),
+  authored by Barry Wark.  Currently the WX and the Cocoa ones have slightly
+  different internal organizations, but the whole team is working on finding
+  what the right abstraction points are for a unified codebase.
+
+* As part of the frontend work, Barry Wark also implemented an experimental
+  event notification system that various ipython components can use.  In the
+  next release the implications and use patterns of this system regarding the
+  various GUI options will be worked out.
+
+* IPython finally has a full test system, that can test docstrings with
+  IPython-specific functionality.  There are still a few pieces missing for it
+  to be widely accessible to all users (so they can run the test suite at any
+  time and report problems), but it now works for the developers.  We are
+  working hard on continuing to improve it, as this was probably IPython's
+  major Achilles heel (the lack of proper test coverage made it effectively
+  impossible to do large-scale refactoring).  The full test suite can now
+  be run using the :command:`iptest` command line program.
+
 	* The notion of a task has been completely reworked.  An `ITask` interface has
-	  been created.  This interface defines the methods that tasks need to implement.
-	  These methods are now responsible for things like submitting tasks and processing
-	  results.  There are two basic task types: :class:`IPython.kernel.task.StringTask`
-	  (this is the old `Task` object, but renamed) and the new 
-	  :class:`IPython.kernel.task.MapTask`, which is based on a function.
+  been created.  This interface defines the methods that tasks need to
+  implement.  These methods are now responsible for things like submitting
+  tasks and processing results.  There are two basic task types:
+  :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but
+  renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on
+  a function.
+
 	* A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to
-	  standardize the idea of a `map` method.  This interface has a single
-	  `map` method that has the same syntax as the built-in `map`.  We have also defined
+  standardize the idea of a `map` method.  This interface has a single `map`
+  method that has the same syntax as the built-in `map`.  We have also defined
 	  a `mapper` factory interface that creates objects that implement
-	  :class:`IPython.kernel.mapper.IMapper` for different controllers.  Both
-	  the multiengine and task controller now have mapping capabilties.
-	* The parallel function capabilities have been reworks.  The major changes are that
-	  i) there is now an `@parallel` magic that creates parallel functions, ii)
-	  the syntax for mulitple variable follows that of `map`, iii) both the
+  :class:`IPython.kernel.mapper.IMapper` for different controllers.  Both the
+  multiengine and task controller now have mapping capabilties.
+
+* The parallel function capabilities have been reworks.  The major changes are
+  that i) there is now an `@parallel` magic that creates parallel functions,
+  ii) the syntax for mulitple variable follows that of `map`, iii) both the
 	  multiengine and task controller now have a parallel function implementation.
-	* All of the parallel computing capabilities from `ipython1-dev` have been merged into
-	  IPython proper.  This resulted in the following new subpackages:  
+
+* All of the parallel computing capabilities from `ipython1-dev` have been
+  merged into IPython proper.  This resulted in the following new subpackages:
 	  :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`,
 	  :mod:`IPython.tools` and :mod:`IPython.testing`.
-	* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and friends
-	  have been completely refactored.  Now we are checking for dependencies using
-	  the approach that matplotlib uses.
-	* The documentation has been completely reorganized to accept the documentation
-	  from `ipython1-dev`.
+
+* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and
+  friends have been completely refactored.  Now we are checking for
+  dependencies using the approach that matplotlib uses.
+
+* The documentation has been completely reorganized to accept the 
+  documentation from `ipython1-dev`.
+
 	* We have switched to using Foolscap for all of our network protocols in
-	  :mod:`IPython.kernel`.  This gives us secure connections that are both encrypted
-	  and authenticated.
+  :mod:`IPython.kernel`.  This gives us secure connections that are both
+  encrypted and authenticated.
+
 	* We have a brand new `COPYING.txt` files that describes the IPython license
 	  and copyright.  The biggest change is that we are putting "The IPython
-	  Development Team" as the copyright holder.  We give more details about exactly
-	  what this means in this file.  All developer should read this and use the new
-	  banner in all IPython source code files.
+  Development Team" as the copyright holder. We give more details about
+  exactly what this means in this file. All developer should read this and use
+  the new banner in all IPython source code files.
+
 	* sh profile: ./foo runs foo as system command, no need to do !./foo anymore
-	* String lists now support 'sort(field, nums = True)' method (to easily
-	  sort system command output). Try it with 'a = !ls -l ; a.sort(1, nums=1)'
+
+* String lists now support ``sort(field, nums = True)`` method (to easily sort
+  system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``.
+
 	* '%cpaste foo' now assigns the pasted block as string list, instead of string
-	* The ipcluster script now run by default with no security.  This is done because
-	  the main usage of the script is for starting things on localhost.  Eventually
-	  when ipcluster is able to start things on other hosts, we will put security
-	  back.
+
+* The ipcluster script now run by default with no security.  This is done
+  because the main usage of the script is for starting things on localhost.
+  Eventually when ipcluster is able to start things on other hosts, we will put
+  security back.
+
 	* 'cd --foo' searches directory history for string foo, and jumps to that dir.
 	  Last part of dir name is checked first. If no matches for that are found,
 	  look at the whole path.
 
+  
 Bug fixes
 ---------
 
-	* The colors escapes in the multiengine client are now turned off on win32 as they
-	  don't print correctly.
-	* The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing mpi_import_statement
-	  incorrectly, which was leading the engine to crash when mpi was enabled.
-	* A few subpackages has missing `__init__.py` files.
-	* The documentation is only created is Sphinx is found.  Previously, the `setup.py`
-	  script would fail if it was missing.
-	* Greedy 'cd' completion has been disabled again (it was enabled in 0.8.4)
+* The Windows installer has been fixed.  Now all IPython scripts have ``.bat``
+  versions created.  Also, the Start Menu shortcuts have been updated.
+
+* The colors escapes in the multiengine client are now turned off on win32 as
+  they don't print correctly.
+
+* The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing
+  mpi_import_statement incorrectly, which was leading the engine to crash when
+  mpi was enabled.
+
+* A few subpackages had missing ``__init__.py`` files.
+
+* The documentation is only created if Sphinx is found.  Previously, the
+  ``setup.py`` script would fail if it was missing.
+
+* Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as
+  it caused problems on certain platforms.
           
 
 Backwards incompatible changes
 ------------------------------
 
+* The ``clusterfile`` options of the :command:`ipcluster` command has been
+  removed as it was not working and it will be replaced soon by something much
+  more robust.
+
+* The :mod:`IPython.kernel` configuration now properly find the user's
+  IPython directory.
+
+* In ipapi, the :func:`make_user_ns` function has been replaced with
+  :func:`make_user_namespaces`, to support dict subclasses in namespace
+  creation.
+
 	* :class:`IPython.kernel.client.Task` has been renamed
 	  :class:`IPython.kernel.client.StringTask` to make way for new task types.
+
 	* The keyword argument `style` has been renamed `dist` in `scatter`, `gather`
 	  and `map`.
+
 	* Renamed the values that the rename `dist` keyword argument can have from
 	  `'basic'` to `'b'`.
+
 	* IPython has a larger set of dependencies if you want all of its capabilities.
 	  See the `setup.py` script for details.
+
 	* The constructors for :class:`IPython.kernel.client.MultiEngineClient` and 
 	  :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple.
 	  Instead they take the filename of a file that contains the FURL for that
 	  client.  If the FURL file is in your IPYTHONDIR, it will be found automatically
 	  and the constructor can be left empty.
+
 	* The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created 
 	  using the factory functions :func:`get_multiengine_client` and 
 	  :func:`get_task_client`.  These return a `Deferred` to the actual client.
+
 	* The command line options to `ipcontroller` and `ipengine` have changed to
 	  reflect the new Foolscap network protocol and the FURL files.  Please see the
 	  help for these scripts for details.
-	* The configuration files for the kernel have changed because of the Foolscap stuff.
-	  If you were using custom config files before, you should delete them and regenerate
-	  new ones.
+
+* The configuration files for the kernel have changed because of the Foolscap
+  stuff.  If you were using custom config files before, you should delete them
+  and regenerate new ones.
 
 Changes merged in from IPython1
 -------------------------------
@@ -102,41 +266,56 b' Changes merged in from IPython1'
 New features
 ............
 
-	* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted
-	  and zope.interface are now easy installable, we can declare them as dependencies
+* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted and
+  zope.interface are now easy installable, we can declare them as dependencies
 	  in our setupegg.py script.
+
 	* IPython is now compatible with Twisted 2.5.0 and 8.x.
+
 	* Added a new example of how to use :mod:`ipython1.kernel.asynclient`.
+
 	* Initial draft of a process daemon in :mod:`ipython1.daemon`.  This has not
 	  been merged into IPython and is still in `ipython1-dev`.
+  
 	* The ``TaskController`` now has methods for getting the queue status.
+
 	* The ``TaskResult`` objects not have information about how long the task
 	  took to run.
+  
 	* We are attaching additional attributes to exceptions ``(_ipython_*)`` that 
 	  we use to carry additional info around.
+  
 	* New top-level module :mod:`asyncclient` that has asynchronous versions (that
 	  return deferreds) of the client classes.  This is designed to users who want
-	  to run their own Twisted reactor
+  to run their own Twisted reactor.
+  
 	* All the clients in :mod:`client` are now based on Twisted.  This is done by 
 	  running the Twisted reactor in a separate thread and using the
 	  :func:`blockingCallFromThread` function that is in recent versions of Twisted.
+
 	* Functions can now be pushed/pulled to/from engines using
-	  :meth:`MultiEngineClient.push_function` and :meth:`MultiEngineClient.pull_function`.
+  :meth:`MultiEngineClient.push_function` and
+  :meth:`MultiEngineClient.pull_function`.
+
 	* Gather/scatter are now implemented in the client to reduce the work load
 	  of the controller and improve performance.
+
 	* Complete rewrite of the IPython docuementation.  All of the documentation
 	  from the IPython website has been moved into docs/source as restructured
 	  text documents.  PDF and HTML documentation are being generated using 
 	  Sphinx.
+
 	* New developer oriented documentation: development guidelines and roadmap. 
-	* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` file
-	  that is organized by release and is meant to provide something more relevant
-	  for users.
+
+* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt``
+  file that is organized by release and is meant to provide something more
+  relevant for users.
 
 Bug fixes
 .........
 
 	* Created a proper ``MANIFEST.in`` file to create source distributions.
+
 	* Fixed a bug in the ``MultiEngine`` interface.  Previously, multi-engine 
 	  actions were being collected with a :class:`DeferredList` with 
 	  ``fireononeerrback=1``.  This meant that methods were returning 
@@ -154,36 +333,41 b' Backwards incompatible changes'
 	* All names have been renamed to conform to the lowercase_with_underscore
 	  convention.  This will require users to change references to all names like
 	  ``queueStatus`` to ``queue_status``.
+
 	* Previously, methods like :meth:`MultiEngineClient.push` and    	
 	  :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``.  This was 
 	  becoming a problem as we weren't able to introduce new keyword arguments into
-	  the API.  Now these methods simple take a dict or sequence.  This has also allowed
-	  us to get rid of the ``*All`` methods like :meth:`pushAll` and :meth:`pullAll`.
-	  These things are now handled with the ``targets`` keyword argument that defaults
-	  to ``'all'``.
+  the API.  Now these methods simple take a dict or sequence.  This has also
+  allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and
+  :meth:`pullAll`.  These things are now handled with the ``targets`` keyword
+  argument that defaults to ``'all'``.
+
 	* The :attr:`MultiEngineClient.magicTargets` has been renamed to
 	  :attr:`MultiEngineClient.targets`. 
-	* All methods in the MultiEngine interface now accept the optional keyword argument
-	  ``block``.
+
+* All methods in the MultiEngine interface now accept the optional keyword
+  argument ``block``.
+
 	* Renamed :class:`RemoteController` to :class:`MultiEngineClient` and 
 	  :class:`TaskController` to :class:`TaskClient`.
+
 	* Renamed the top-level module from :mod:`api` to :mod:`client`.
-	* Most methods in the multiengine interface now raise a :exc:`CompositeError` exception
-	  that wraps the user's exceptions, rather than just raising the raw user's exception.
+
+* Most methods in the multiengine interface now raise a :exc:`CompositeError`
+  exception that wraps the user's exceptions, rather than just raising the raw
+  user's exception.
+
 	* Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` 
 	  and ``pull``.
 
+  
 Release 0.8.4
 =============
 
-Someone needs to describe what went into 0.8.4.
+This was a quick release to fix an unfortunate bug that slipped into the 0.8.3
+release.  The ``--twisted`` option was disabled, as it turned out to be broken
+across several platforms.
 
-Release 0.8.2
-=============
-
-* %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
-  and jumps to /foo. The current behaviour is closer to the documented 
-  behaviour, and should not trip anyone.
   
 Release 0.8.3
 =============
@@ -192,9 +376,18 b' Release 0.8.3'
   it by passing -pydb command line argument to IPython. Note that setting
   it in config file won't work.
 
+  
+Release 0.8.2
+=============
+
+* %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
+  and jumps to /foo. The current behaviour is closer to the documented 
+  behaviour, and should not trip anyone.
+
+  
 Older releases
 ==============
 
-Changes in earlier releases of IPython are described in the older file ``ChangeLog``.  
-Please refer to this document for details.
+Changes in earlier releases of IPython are described in the older file
+``ChangeLog``.  Please refer to this document for details.
 

docs/source/conf.py

@@ -1,7 +1,11 b''
 # -*- coding: utf-8 -*-
 #
-# IPython documentation build configuration file, created by
-# sphinx-quickstart on Thu May  8 16:45:02 2008.
+# IPython documentation build configuration file.
+
+# NOTE: This file has been edited manually from the auto-generated one from
+# sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
+# needed, generate a scratch one and merge by hand any new fields needed.
+
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
@@ -16,14 +20,26 b' import sys, os'
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-#sys.path.append(os.path.abspath('some/directory'))
+sys.path.append(os.path.abspath('../sphinxext'))
+
+# Import support for ipython console session syntax highlighting (lives
+# in the sphinxext directory defined above)
+import ipython_console_highlighting
+
+# We load the ipython release info into a dict by explicit execution
+iprelease = {}
+execfile('../../IPython/Release.py',iprelease)
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = []
+extensions = ['sphinx.ext.autodoc',
+              'inheritance_diagram', 'only_directives',
+              'ipython_console_highlighting', 
+              # 'plot_directive', # disabled for now, needs matplotlib
+              ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -41,10 +57,11 b" copyright = '2008, The IPython Development Team'"
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
 #
-# The short X.Y version.
-version = '0.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.beta1'
+release = iprelease['version']
+# The short X.Y version.
+version = '.'.join(release.split('.',2)[:2])
+
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -57,7 +74,7 b" today_fmt = '%B %d, %Y'"
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
-#exclude_dirs = []
+exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
@@ -125,7 +142,7 b" html_last_updated_fmt = '%b %d, %Y'"
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'IPythondoc'
+htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
@@ -140,11 +157,8 b" latex_font_size = '11pt'"
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
-latex_documents = [ ('index', 'IPython.tex', 'IPython Documentation',
-                     ur"""Brian Granger, Fernando Pérez and Ville Vainio\\
-                     \ \\
-                     With contributions from:\\
-                     Benjamin Ragan-Kelley.""",
+latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
+                     ur"""The IPython Development Team""",
                      'manual'),
                     ]
 
@@ -164,3 +178,10 b" latex_documents = [ ('index', 'IPython.tex', 'IPython Documentation',"
 
 # If false, no module index is generated.
 #latex_use_modindex = True
+
+
+# Cleanup
+# -------
+# delete release info to avoid pickling errors from sphinx
+
+del iprelease

docs/source/config/customization.txt

@@ -18,6 +18,8 b' time. A hybrid approach of specifying a few options in ipythonrc and'
 doing the more advanced configuration in ipy_user_conf.py is also
 possible.
 
+.. _ipythonrc:
+
 The ipythonrc approach
 ======================
 
@@ -36,11 +38,11 b' fairly primitive). Note that these are not python files, and this is'
 deliberate, because it allows us to do some things which would be quite
 tricky to implement if they were normal python files.
 
-First, an rcfile can contain permanent default values for almost all
-command line options (except things like -help or -Version). Sec
-`command line options`_ contains a description of all command-line
-options. However, values you explicitly specify at the command line
-override the values defined in the rcfile.
+First, an rcfile can contain permanent default values for almost all command
+line options (except things like -help or -Version). :ref:`This section
+<command_line_options>` contains a description of all command-line
+options. However, values you explicitly specify at the command line override
+the values defined in the rcfile.
 
 Besides command line option values, the rcfile can specify values for
 certain extra special options which are not available at the command
@@ -266,13 +268,13 b' which look like this::'
 IPython profiles
 ================
 
-As we already mentioned, IPython supports the -profile command-line
-option (see sec. `command line options`_). A profile is nothing more
-than a particular configuration file like your basic ipythonrc one,
-but with particular customizations for a specific purpose. When you
-start IPython with 'ipython -profile <name>', it assumes that in your
-IPYTHONDIR there is a file called ipythonrc-<name> or
-ipy_profile_<name>.py, and loads it instead of the normal ipythonrc.
+As we already mentioned, IPython supports the -profile command-line option (see
+:ref:`here <command_line_options>`).  A profile is nothing more than a
+particular configuration file like your basic ipythonrc one, but with
+particular customizations for a specific purpose. When you start IPython with
+'ipython -profile <name>', it assumes that in your IPYTHONDIR there is a file
+called ipythonrc-<name> or ipy_profile_<name>.py, and loads it instead of the
+normal ipythonrc.
 
 This system allows you to maintain multiple configurations which load
 modules, set options, define functions, etc. suitable for different

docs/source/config/index.txt

@@ -3,7 +3,7 b' Configuration and customization'
 ===============================
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    initial_config.txt
    customization.txt

docs/source/config/initial_config.txt

@@ -11,28 +11,27 b' in a directory named by default $HOME/.ipython. You can change this by'
 defining the environment variable IPYTHONDIR, or at runtime with the
 command line option -ipythondir.
 
-If all goes well, the first time you run IPython it should
-automatically create a user copy of the config directory for you,
-based on its builtin defaults. You can look at the files it creates to
-learn more about configuring the system. The main file you will modify
-to configure IPython's behavior is called ipythonrc (with a .ini
-extension under Windows), included for reference in `ipythonrc`_
-section. This file is very commented and has many variables you can
-change to suit your taste, you can find more details in
-Sec. customization_. Here we discuss the basic things you will want to
-make sure things are working properly from the beginning.
+If all goes well, the first time you run IPython it should automatically create
+a user copy of the config directory for you, based on its builtin defaults. You
+can look at the files it creates to learn more about configuring the
+system. The main file you will modify to configure IPython's behavior is called
+ipythonrc (with a .ini extension under Windows), included for reference
+:ref:`here <ipythonrc>`. This file is very commented and has many variables you
+can change to suit your taste, you can find more details :ref:`here
+<customization>`. Here we discuss the basic things you will want to make sure
+things are working properly from the beginning.
 
 
-.. _Accessing help:
+.. _accessing_help:
 
 Access to the Python help system
 ================================
 
-This is true for Python in general (not just for IPython): you should
-have an environment variable called PYTHONDOCS pointing to the directory
-where your HTML Python documentation lives. In my system it's
-/usr/share/doc/python-docs-2.3.4/html, check your local details or ask
-your systems administrator.
+This is true for Python in general (not just for IPython): you should have an
+environment variable called PYTHONDOCS pointing to the directory where your
+HTML Python documentation lives. In my system it's
+:file:`/usr/share/doc/python-doc/html`, check your local details or ask your
+systems administrator.
 
 This is the directory which holds the HTML version of the Python
 manuals. Unfortunately it seems that different Linux distributions
@@ -40,8 +39,9 b' package these files differently, so you may have to look around a bit.'
 Below I show the contents of this directory on my system for reference::
 
     [html]> ls
-    about.dat acks.html dist/ ext/ index.html lib/ modindex.html
-    stdabout.dat tut/ about.html api/ doc/ icons/ inst/ mac/ ref/ style.css
+    about.html  dist/  icons/      lib/           python2.5.devhelp.gz  whatsnew/
+    acks.html   doc/   index.html  mac/           ref/
+    api/        ext/   inst/       modindex.html  tut/
 
 You should really make sure this variable is correctly set so that
 Python's pydoc-based help system works. It is a powerful and convenient
@@ -108,6 +108,8 b' The following terminals seem to handle the color sequences fine:'
       support under cygwin, please post to the IPython mailing list so
       this issue can be resolved for all users.
 
+.. _pyreadline: https://code.launchpad.net/pyreadline
+      
 These have shown problems:
 
     * Windows command prompt in WinXP/2k logged into a Linux machine via
@@ -157,13 +159,12 b' $HOME/.ipython/ipythonrc and set the colors option to the desired value.'
 Object details (types, docstrings, source code, etc.)
 =====================================================
 
-IPython has a set of special functions for studying the objects you
-are working with, discussed in detail in Sec. `dynamic object
-information`_. But this system relies on passing information which is
-longer than your screen through a data pager, such as the common Unix
-less and more programs. In order to be able to see this information in
-color, your pager needs to be properly configured. I strongly
-recommend using less instead of more, as it seems that more simply can
+IPython has a set of special functions for studying the objects you are working
+with, discussed in detail :ref:`here <dynamic_object_info>`. But this system
+relies on passing information which is longer than your screen through a data
+pager, such as the common Unix less and more programs. In order to be able to
+see this information in color, your pager needs to be properly configured. I
+strongly recommend using less instead of more, as it seems that more simply can
 not understand colored text correctly.
 
 In order to configure less as your default pager, do the following:

docs/source/credits.txt

@@ -4,24 +4,39 b''
 Credits
 =======
 
-IPython is mainly developed by Fernando Pérez
-<[e-mail]>, but the project was born from mixing in
-Fernando's code with the IPP project by Janko Hauser
-<jhauser-AT-zscout.de> and LazyPython by Nathan Gray
-<n8gray-AT-caltech.edu>. For all IPython-related requests, please
-contact Fernando.
+IPython is led by Fernando Pérez.
 
-As of early 2006, the following developers have joined the core team:
+As of this writing, the following developers have joined the core team:
 
     * [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
       Google Summer of Code project to develop python interactive
       notebooks (XML documents) and graphical interface. This project
       was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
-      Toni Alatalo <antont-AT-an.org>
-    * [Brian Granger] <bgranger-AT-scu.edu>: extending IPython to allow
+  Toni Alatalo <antont-AT-an.org>.
+
+* [Brian Granger] <ellisonbg-AT-gmail.com>: extending IPython to allow
       support for interactive parallel computing.
-    * [Ville Vainio] <vivainio-AT-gmail.com>: Ville is the new
-      maintainer for the main trunk of IPython after version 0.7.1.
+
+* [Benjamin (Min) Ragan-Kelley]: key work on IPython's parallel
+  computing infrastructure.
+
+* [Ville Vainio] <vivainio-AT-gmail.com>: Ville has made many improvements
+  to the core of IPython and was the maintainer of the main IPython
+  trunk from version 0.7.1 to 0.8.4.
+
+* [Gael Varoquaux] <gael.varoquaux-AT-normalesup.org>: work on the merged
+  architecture for the interpreter as of version 0.9, implementing a new WX GUI
+  based on this system.
+
+* [Barry Wark] <barrywark-AT-gmail.com>: implementing a new Cocoa GUI, as well
+  as work on the new interpreter architecture and Twisted support.
+
+* [Laurent Dufrechou] <laurent.dufrechou-AT-gmail.com>: development of the WX
+  GUI support.
+
+* [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu>: maintainer of the
+  PyReadline project, necessary for IPython under windows.
+
 
 The IPython project is also very grateful to:
 
@@ -50,9 +65,12 b" an O'Reilly Python editor. His Oct/11/2001 article about IPP and"
 LazyPython, was what got this project started. You can read it at:
 http://www.onlamp.com/pub/a/python/2001/10/11/pythonnews.html.
 
-And last but not least, all the kind IPython users who have emailed new
-code, bug reports, fixes, comments and ideas. A brief list follows,
-please let me know if I have ommitted your name by accident:
+And last but not least, all the kind IPython users who have emailed new code,
+bug reports, fixes, comments and ideas. A brief list follows, please let us
+know if we have ommitted your name by accident:
+
+* Dan Milstein <danmil-AT-comcast.net>.  A bold refactoring of the 
+  core prefilter stuff in the IPython interpreter.
 
     * [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
       color problem. This bug alone caused many lost hours and
@@ -60,80 +78,130 b' please let me know if I have ommitted your name by accident:'
       fan of Ogg & friends, now I have one more reason to like these folks.
       Jack is also contributing with Debian packaging and many other
       things.
+ 
     * [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
       reports, bug fixes, ideas, lots more. The ipython.el mode for
       (X)Emacs is Alex's code, providing full support for IPython under
       (X)Emacs.
+
     * [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
       information, Fink package management.
+
     * [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
       around the exception handling idiosyncracies of WxPython. Readline
       and color support for Windows.
+
     * [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
       improved readline support, including fixes for Python 2.3.
+
     * [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
+
     * [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
+
     * [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
+
     * [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
+
     * [Philip Hisley] <compsys-AT-starpower.net>
+
     * [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
       more.
+
     * [Robin Siebler] <robinsiebler-AT-starband.net>
+
     * [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
+
     * [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
+
     * [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
+
     * [Syver Enstad] <syver-en-AT-online.no> Windows setup.
+
     * [Richard] <rxe-AT-renre-europe.com> Global embedding.
+
     * [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
       compatibility.
+  
     * [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
       installation.
+  
     * [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
+
     * [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
       documentation fixes.
+  
     * [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
       ideas. Patches for Windows installer.
+  
     * [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
+
     * [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
+
     * [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
       Win32/CygWin.
+  
     * [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
       nice, lightweight string interpolation.
+  
     * [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
+
     * [Gever Tulley] <gever-AT-helium.com> Code contributions.
+
     * [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
+
     * [Oliver Sander] <osander-AT-gmx.de> Bug reports.
+
     * [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
       logging module.
+
     * [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
       Fixes, enhancement suggestions for system shell use.
+
     * [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
       reports on Windows installation issues. Contributed a true Windows
       binary installer.
+
     * [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
       to traceback printing.
+
     * [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
       prompt specials.
+
     * [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
       the multithreaded IPython.
+
     * [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
       author, helped with all the development of support for matplotlib
       in IPyhton, including making necessary changes to matplotlib itself.
+
     * [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
+
     * [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
       with (X)Emacs support, threading patches, ideas...
+
     * [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
       packaging and distribution.
+
     * [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
       tab-completing named arguments of user-defined functions.
+
     * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
       support implementation for searching namespaces.
+
     * [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
       so that when pdb is activated from within IPython, coloring, tab
       completion and other features continue to work seamlessly.
+
     * [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
       editor invocation on syntax errors (see
       http://www.scipy.net/roundup/ipython/issue36).
+
     * [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
       paging system.
+
     * [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
+
+* [Ondrej Certik] <ondrej-AT-certik.cz>: set up the IPython docs to use the new
+  Sphinx system used by Python, Matplotlib and many more projects.
+
+* [Stefan van der Walt] <stefan-AT-sun.ac.za>: support for the new config system.

docs/source/development/development.txt

@@ -1,191 +1,142 b''
 .. _development:
 
-==================================
+==============================
 IPython development guidelines
-==================================
-
-.. contents::
+==============================
 
 
 Overview
 ========
 
-IPython is the next generation of IPython.  It is named such for two reasons:
-
-- Eventually, IPython will become IPython version 1.0.
-- This new code base needs to be able to co-exist with the existing IPython until
-  it is a full replacement for it.  Thus we needed a different name.  We couldn't
-  use ``ipython`` (lowercase) as some files systems are case insensitive.
-
-There are two, no three, main goals of the IPython effort:
-
-1. Clean up the existing codebase and write lots of tests.
-2. Separate the core functionality of IPython from the terminal to enable IPython
-   to be used from within a variety of GUI applications.
-3. Implement a system for interactive parallel computing.
-
-While the third goal may seem a bit unrelated to the main focus of IPython, it turns
-out that the technologies required for this goal are nearly identical with those
-required for goal two. This is the main reason the interactive parallel computing
-capabilities are being put into IPython proper. Currently the third of these goals is
-furthest along.
-
-This document describes IPython from the perspective of developers.  
-
-
-Project organization
-====================
+This document describes IPython from the perspective of developers. Most
+importantly, it gives information for people who want to contribute to the
+development of IPython. So if you want to help out, read on!
 
-Subpackages
------------
+How to contribute to IPython
+============================
 
-IPython is organized into semi self-contained subpackages.  Each of the subpackages will have its own:
+IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_.
+This makes it easy for people to contribute to the development of IPython.
+Here is a sketch of how to get going.
 
-- **Dependencies**.  One of the most important things to keep in mind in
-  partitioning code amongst subpackages, is that they should be used to cleanly
-  encapsulate dependencies.  
-- **Tests**.  Each subpackage shoud have its own ``tests`` subdirectory that
-  contains all of the tests for that package.  For information about writing tests
-  for IPython, see the `Testing System`_ section of this document.
-- **Configuration**.  Each subpackage should have its own ``config`` subdirectory
-  that contains the configuration information for the components of the 
-  subpackage.  For information about how the IPython configuration system
-  works, see the `Configuration System`_ section of this document.
-- **Scripts**.  Each subpackage should have its own ``scripts`` subdirectory that
-  contains all of the command line scripts associated with the subpackage.
+Install Bazaar and create a Launchpad account
+---------------------------------------------
 
-Installation and dependencies
------------------------------
+First make sure you have installed Bazaar (see their `website
+<http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about
+you, try the following::
    
-IPython will not use `setuptools`_ for installation. Instead, we will use standard
-``setup.py`` scripts that use `distutils`_. While there are a number a extremely nice
-features that `setuptools`_ has (like namespace packages), the current implementation
-of `setuptools`_ has performance problems, particularly on shared file systems. In
-particular, when Python packages are installed on NSF file systems, import times
-become much too long (up towards 10 seconds). 
+    $ bzr whoami
+    Joe Coder <[e-mail]>
 
-Because IPython is being used extensively in the context of high performance
-computing, where performance is critical but shared file systems are common, we feel
-these performance hits are not acceptable. Thus, until the performance problems
-associated with `setuptools`_ are addressed, we will stick with plain `distutils`_. We
-are hopeful that these problems will be addressed and that we will eventually begin
-using `setuptools`_. Because of this, we are trying to organize IPython in a way that
-will make the eventual transition to `setuptools`_ as painless as possible.
+This should display your name and email. Next, you will want to create an
+account on the `Launchpad website <http://www.launchpad.net>`_ and setup your
+ssh keys. For more information of setting up your ssh keys, see `this link
+<https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_.
 
-Because we will be using `distutils`_, there will be no method for automatically installing dependencies.  Instead, we are following the approach of `Matplotlib`_ which can be summarized as follows:
+Get the main IPython branch from Launchpad
+------------------------------------------
 
-- Distinguish between required and optional dependencies.  However, the required
-  dependencies for IPython should be only the Python standard library.
-- Upon installation check to see which optional dependencies are present and tell
-  the user which parts of IPython need which optional dependencies.
+Now, you can get a copy of the main IPython development branch (we call this
+the "trunk")::
 
-It is absolutely critical that each subpackage of IPython has a clearly specified set
-of dependencies and that dependencies are not carelessly inherited from other IPython
-subpackages. Furthermore, tests that have certain dependencies should not fail if
-those dependencies are not present. Instead they should be skipped and print a
-message.
-
-.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
-.. _distutils: http://docs.python.org/lib/module-distutils.html
-.. _Matplotlib: http://matplotlib.sourceforge.net/
-
-Specific subpackages
---------------------
-
-``core``
-	This is the core functionality of IPython that is independent of the
-	terminal, network and GUIs.  Most of the code that is in the current
-	IPython trunk will be refactored, cleaned up and moved here.
-
-``kernel``
-	The enables the IPython core to be expose to a the network.  This is
-	also where all of the parallel computing capabilities are to be found.
-	
-``config``
-	The configuration package used by IPython.
+    $ bzr branch lp:ipython
 
-``frontends``
-	The various frontends for IPython.  A frontend is the end-user application
-	that exposes the capabilities of IPython to the user.  The most basic frontend
-	will simply be a terminal based application that looks just like today 's 
-	IPython.  Other frontends will likely be more powerful and based on GUI toolkits.
+Create a working branch
+-----------------------
 
-``notebook``
-	An application that allows users to work with IPython notebooks.
-
-``tools``
-	This is where general utilities go.
+When working on IPython, you won't actually make edits directly to the
+:file:`lp:ipython` branch. Instead, you will create a separate branch for your
+changes. For now, let's assume you want to do your work in a branch named
+"ipython-mybranch". Create this branch by doing::
 
+    $ bzr branch ipython ipython-mybranch
 
-Version control
-===============
-
-In the past, IPython development has been done using `Subversion`__.  Recently, we made the transition to using `Bazaar`__ and `Launchpad`__.  This makes it much easier for people
-to contribute code to IPython.  Here is a sketch of how to use Bazaar for IPython
-development.  First, you should install Bazaar.  After you have done that, make
-sure that it is working by getting the latest main branch of IPython::
+When you actually create a branch, you will want to give it a name that
+reflects the nature of the work that you will be doing in it, like
+"install-docs-update".
 
-	$ bzr branch lp:ipython
+Make edits in your working branch
+---------------------------------
 
-Now you can create a new branch for you to do your work in::
+Now you are ready to actually make edits in your :file:`ipython-mybranch`
+branch. Before doing this, it is helpful to install this branch so you can
+test your changes as you work. This is easiest if you have setuptools
+installed. Then, just do::
 
-	$ bzr branch ipython ipython-mybranch
+    $ cd ipython-mybranch
+    $ python setupegg.py develop
 
-The typical work cycle in this branch will be to make changes in `ipython-mybranch`
-and then commit those changes using the commit command::
+Now, make some changes. After a while, you will want to commit your changes.
+This let's Bazaar know that you like the changes you have made and gives you
+an opportunity to keep a nice record of what you have done. This looks like
+this::
 
 	$ ...do work in ipython-mybranch...
-	$ bzr ci -m "the commit message goes here"
+    $ bzr commit -m "the commit message goes here"
 
-Please note that since we now don't use an old-style linear ChangeLog
-(that tends to cause problems with distributed version control
-systems), you should ensure that your log messages are reasonably
-detailed.  Use a docstring-like approach in the commit messages
-(including the second line being left *blank*)::
+Please note that since we now don't use an old-style linear ChangeLog (that
+tends to cause problems with distributed version control systems), you should
+ensure that your log messages are reasonably detailed.  Use a docstring-like
+approach in the commit messages (including the second line being left
+*blank*)::
 
   Single line summary of  changes being committed.
 
-  - more details when warranted ...
-  - including crediting outside contributors if they sent the
+  * more details when warranted ...
+  * including crediting outside contributors if they sent the
     code/bug/idea!
 
-If we couple this with a policy of making single commits for each
-reasonably atomic change, the bzr log should give an excellent view of
-the project, and the `--short` log option becomes a nice summary.
+As you work, you will repeat this edit/commit cycle many times. If you work on
+your branch for a long time, you will also want to get the latest changes from
+the :file:`lp:ipython` branch. This can be done with the following sequence of
+commands::
 
-While working with this branch, it is a good idea to merge in changes that have been
-made upstream in the parent branch.  This can be done by doing::
+    $ ls
+    ipython
+    ipython-mybranch
 
+    $ cd ipython
 	$ bzr pull
+    $ cd ../ipython-mybranch
+    $ bzr merge ../ipython
+    $ bzr commit -m "Merging changes from trunk"
 	
-If this command shows that the branches have diverged, then you should do a merge
-instead::
+Along the way, you should also run the IPython test suite.  You can do this using the :command:`iptest` command::
 
-	$ bzr merge lp:ipython
+    $ cd
+    $ iptest
 
-If you want others to be able to see your branch, you can create an account with
-launchpad and push the branch to your own workspace::
+The :command:`iptest` command will also pick up and run any tests you have written.
 
-	$ bzr push bzr+ssh://<me>@bazaar.launchpad.net/~<me>/+junk/ipython-mybranch
+Post your branch and request a code review
+------------------------------------------
 
-Finally, once the work in your branch is done, you can merge your changes back into
-the `ipython` branch by using merge::
+Once you are done with your edits, you should post your branch on Launchpad so
+that other IPython developers can review the changes and help you merge your
+changes into the main development branch. To post your branch on Launchpad,
+do::
 
-	$ cd ipython
-	$ merge ../ipython-mybranch
-	[resolve any conflicts]
-	$ bzr ci -m "Fixing that bug"
-	$ bzr push
+    $ cd ipython-mybranch
+    $ bzr push lp:~yourusername/ipython/ipython-mybranch
 
-But this will require you to have write permissions to the `ipython` branch.  It you don't
-you can tell one of the IPython devs about your branch and they can do the merge for you.
+Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you
+should see your branch under the "Code" tab. If you click on your branch, you
+can provide a short description of the branch as well as mark its status. Most
+importantly, you should click the link that reads "Propose for merging into
+another branch". What does this do?
 
-More information about Bazaar workflows can be found `here`__.
+This let's the other IPython developers know that your branch is ready to be
+reviewed and merged into the main development branch. During this review
+process, other developers will give you feedback and help you get your code
+ready to be merged. What types of things will we be looking for:
 
-.. __: http://subversion.tigris.org/
-.. __: http://bazaar-vcs.org/
-.. __: http://www.launchpad.net/ipython
-.. __: http://doc.bazaar-vcs.org/bzr.dev/en/user-guide/index.html
+* All code is documented.
+* All code has tests.
+* The entire IPython test suite passes.
+
+Once your changes have been reviewed and approved, someone will merge them
+into the main development branch.
 
 Documentation
 =============
@@ -193,38 +144,32 b' Documentation'
 Standalone documentation
 ------------------------
 
-All standalone documentation should be written in plain text (``.txt``) files using
-`reStructuredText`_ for markup and formatting. All such documentation should be placed
-in the top level directory ``docs`` of the IPython source tree. Or, when appropriate,
-a suitably named subdirectory should be used. The documentation in this location will
-serve as the main source for IPython documentation and all existing documentation
-should be converted to this format.
+All standalone documentation should be written in plain text (``.txt``) files
+using reStructuredText [reStructuredText]_ for markup and formatting. All such
+documentation should be placed in directory :file:`docs/source` of the IPython
+source tree. The documentation in this location will serve as the main source
+for IPython documentation and all existing documentation should be converted
+to this format.
 
-In the future, the text files in the ``docs`` directory will be used to generate all
-forms of documentation for IPython. This include documentation on the IPython website
-as well as *pdf* documentation.
+To build the final documentation, we use Sphinx [Sphinx]_.  Once you have Sphinx installed, you can build the html docs yourself by doing::
 
-.. _reStructuredText: http://docutils.sourceforge.net/rst.html
+    $ cd ipython-mybranch/docs
+    $ make html
 
 Docstring format
 ----------------
 
-Good docstrings are very important. All new code will use `Epydoc`_ for generating API
-docs, so we will follow the `Epydoc`_ conventions. More specifically, we will use
-`reStructuredText`_ for markup and formatting, since it is understood by a wide
-variety of tools. This means that if in the future we have any reason to change from
-`Epydoc`_ to something else, we'll have fewer transition pains.
-
-Details about using `reStructuredText`_ for docstrings can be found `here
+Good docstrings are very important. All new code should have docstrings that
+are formatted using reStructuredText for markup and formatting, since it is
+understood by a wide variety of tools. Details about using reStructuredText
+for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
 
-.. _Epydoc: http://epydoc.sourceforge.net/
-
 Additional PEPs of interest regarding documentation of code:
 
-- `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
-- `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
-- `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
+* `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
+* `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
+* `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 
 Coding conventions
@@ -233,128 +178,133 b' Coding conventions'
 General
 -------
 
-In general, we'll try to follow the standard Python style conventions as described here:
+In general, we'll try to follow the standard Python style conventions as
+described here:
 
-- `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
+* `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
 
 Other comments:
 
-- In a large file, top level classes and functions should be 
+* In a large file, top level classes and functions should be 
   separated by 2-3 lines to make it easier to separate them visually.
-- Use 4 spaces for indentation.
-- Keep the ordering of methods the same in classes that have the same 
-  methods.  This is particularly true for classes that implement 
-  similar interfaces and for interfaces that are similar.
+* Use 4 spaces for indentation.
+* Keep the ordering of methods the same in classes that have the same 
+  methods.  This is particularly true for classes that implement an interface.
 
 Naming conventions
 ------------------
 
-In terms of naming conventions, we'll follow the guidelines from the `Style Guide for
-Python Code`_.
+In terms of naming conventions, we'll follow the guidelines from the `Style
+Guide for Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll use:
 
-- All ``lowercase`` module names.
+* All ``lowercase`` module names.
 
-- ``CamelCase`` for class names.
+* ``CamelCase`` for class names.
 
-- ``lowercase_with_underscores`` for methods, functions, variables and attributes.
+* ``lowercase_with_underscores`` for methods, functions, variables and
+  attributes.
 
-This may be confusing as most of the existing IPython codebase uses a different convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will move IPython over to the new
-convention, providing shadow names for backward compatibility in public interfaces. 
+There are, however, some important exceptions to these rules. In some cases,
+IPython code will interface with packages (Twisted, Wx, Qt) that use other
+conventions. At some level this makes it impossible to adhere to our own
+standards at all times. In particular, when subclassing classes that use other
+naming conventions, you must follow their naming conventions. To deal with
+cases like this, we propose the following policy:
 
-There are, however, some important exceptions to these rules.  In some cases, IPython
-code will interface with packages (Twisted, Wx, Qt) that use other conventions.  At some level this makes it impossible to adhere to our own standards at all times.  In particular, when subclassing classes that use other naming conventions, you must follow their naming conventions.  To deal with cases like this, we propose the following policy:
-
-- If you are subclassing a class that uses different conventions, use its
+* If you are subclassing a class that uses different conventions, use its
   naming conventions throughout your subclass.  Thus, if you are creating a 
-  Twisted Protocol class, used Twisted's ``namingSchemeForMethodsAndAttributes.``
+  Twisted Protocol class, used Twisted's
+  ``namingSchemeForMethodsAndAttributes.``
 
-- All IPython's official interfaces should use our conventions.  In some cases 
-  this will mean that you need to provide shadow names (first implement ``fooBar``
-  and then ``foo_bar = fooBar``).  We want to avoid this at all costs, but it 
-  will probably be necessary at times.  But, please use this sparingly!
+* All IPython's official interfaces should use our conventions.  In some cases
+  this will mean that you need to provide shadow names (first implement
+  ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
+  costs, but it will probably be necessary at times.  But, please use this
+  sparingly!
 
-Implementation-specific *private* methods will use ``_single_underscore_prefix``.
-Names with a leading double underscore will *only* be used in special cases, as they
-makes subclassing difficult (such names are not easily seen by child classes).
+Implementation-specific *private* methods will use
+``_single_underscore_prefix``. Names with a leading double underscore will
+*only* be used in special cases, as they makes subclassing difficult (such
+names are not easily seen by child classes).
 
-Occasionally some run-in lowercase names are used, but mostly for very short names or
-where we are implementing methods very similar to existing ones in a base class (like
-``runlines()`` where ``runsource()`` and ``runcode()`` had established precedent).
+Occasionally some run-in lowercase names are used, but mostly for very short
+names or where we are implementing methods very similar to existing ones in a
+base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
+established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
-explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned upon, as
-namespaces offer cleaner prefixing. The only case where this approach is justified is
-for classes which are expected to be imported into external namespaces and a very
-generic name (like Shell) is too likely to clash with something else. We'll need to
-revisit this issue as we clean up and refactor the code, but in general we should
-remove as many unnecessary ``IP``/``ip`` prefixes as possible. However, if a prefix
-seems absolutely necessary the more specific ``IPY`` or ``ipy`` are preferred.
+explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
+upon, as namespaces offer cleaner prefixing. The only case where this approach
+is justified is for classes which are expected to be imported into external
+namespaces and a very generic name (like Shell) is too likely to clash with
+something else. We'll need to revisit this issue as we clean up and refactor
+the code, but in general we should remove as many unnecessary ``IP``/``ip``
+prefixes as possible. However, if a prefix seems absolutely necessary the more
+specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
-It is extremely important that all code contributed to IPython has tests. Tests should
-be written as unittests, doctests or as entities that the `Nose`_ testing package will
-find. Regardless of how the tests are written, we will use `Nose`_ for discovering and
-running the tests. `Nose`_ will be required to run the IPython test suite, but will
-not be required to simply use IPython.
-
-.. _Nose: http://code.google.com/p/python-nose/
+It is extremely important that all code contributed to IPython has tests.
+Tests should be written as unittests, doctests or as entities that the Nose
+[Nose]_ testing package will find. Regardless of how the tests are written, we
+will use Nose for discovering and running the tests. Nose will be required to
+run the IPython test suite, but will not be required to simply use IPython.
 
-Tests of `Twisted`__ using code should be written by subclassing the ``TestCase`` class
-that comes with ``twisted.trial.unittest``. When this is done, `Nose`_ will be able to
-run the tests and the twisted reactor will be handled correctly.
+Tests of Twisted using code need to follow two additional guidelines:
 
-.. __: http://www.twistedmatrix.com
+1. Twisted using tests should be written by subclassing the :class:`TestCase`
+   class that comes with :mod:`twisted.trial.unittest`. 
 
-Each subpackage in IPython should have its own ``tests`` directory that contains all
-of the tests for that subpackage. This allows each subpackage to be self-contained. If
-a subpackage has any dependencies beyond the Python standard library, the tests for
-that subpackage should be skipped if the dependencies are not found. This is very
-important so users don't get tests failing simply because they don't have dependencies.
+2. All :class:`Deferred` instances that are created in the test must be
+   properly chained and the final one *must* be the return value of the test
+   method.
 
-We also need to look into use Noses ability to tag tests to allow a more modular
-approach of running tests.
-
-.. _devel_config:
+When these two things are done, Nose will be able to run the tests and the
+twisted reactor will be handled correctly.
 
-Configuration system
-====================
+Each subpackage in IPython should have its own :file:`tests` directory that
+contains all of the tests for that subpackage. This allows each subpackage to
+be self-contained. If a subpackage has any dependencies beyond the Python
+standard library, the tests for that subpackage should be skipped if the
+dependencies are not found. This is very important so users don't get tests
+failing simply because they don't have dependencies.
 
-IPython uses `.ini`_ files for configuration purposes. This represents a huge
-improvement over the configuration system used in IPython. IPython works with these
-files using the `ConfigObj`_ package, which IPython includes as
-``ipython1/external/configobj.py``.
+To run the IPython test suite, use the :command:`iptest` command that is installed with IPython::
 
-Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of IPython
-should contain a ``config`` subdirectory that contains all of the configuration
-information for the subpackage. To see how configuration information is defined (along
-with defaults) see at the examples in ``ipython1/kernel/config`` and
-``ipython1/core/config``. Likewise, to see how the configuration information is used,
-see examples in ``ipython1/kernel/scripts/ipengine.py``.
-
-Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We are
-calling this new layer, ``tconfig``, as it will use a `Traits`_-like validation model.
-We won't actually use `Traits`_, but will implement something similar in pure Python.
-But, even in this new system, we will still use `ConfigObj`_ and `.ini`_ files
-underneath the hood. Talk to Fernando if you are interested in working on this part of
-IPython. The current prototype of ``tconfig`` is located in the IPython sandbox.
-
-.. _.ini: http://docs.python.org/lib/module-ConfigParser.html
-.. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html
-.. _Traits: http://code.enthought.com/traits/
+    $ iptest
 
+This command runs Nose with the proper options and extensions.
 
+.. _devel_config:
 
+Release checklist
+=================
 
+Most of the release process is automated by the :file:`release` script in the
+:file:`tools` directory.  This is just a handy reminder for the release manager.
 
+#. Run the release script, which makes the tar.gz, eggs and Win32 .exe
+   installer.  It posts them to the site and registers the release with PyPI.
 
+#. Updating the website with announcements and links to the updated
+   changes.txt in html form. Remember to put a short note both on the news
+   page of the site and on Launcphad.
 
+#. Drafting a short release announcement with i) highlights and ii) a link to
+   the html changes.txt.
 
+#. Make sure that the released version of the docs is live on the site.
 
+#. Celebrate!
 
+.. [Bazaar] Bazaar. http://bazaar-vcs.org/
+.. [Launchpad] Launchpad. http://www.launchpad.net/ipython
+.. [reStructuredText] reStructuredText.  http://docutils.sourceforge.net/rst.html
+.. [Sphinx] Sphinx. http://sphinx.pocoo.org/
+.. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/

docs/source/development/index.txt

@@ -7,3 +7,5 b' Development'
 
    development.txt
    roadmap.txt
+   notification_blueprint.txt
+   config_blueprint.txt

docs/source/development/notification_blueprint.txt

@@ -1,4 +1,4 b''
-.. Notification:
+.. _notification:
 
 ==========================================
 IPython.kernel.core.notification blueprint
@@ -6,41 +6,77 b' IPython.kernel.core.notification blueprint'
 
 Overview
 ========
-The :mod:`IPython.kernel.core.notification` module will provide a simple implementation of a notification center and support for the observer pattern within the :mod:`IPython.kernel.core`. The main intended use case is to provide notification of Interpreter events to an observing frontend during the execution of a single block of code.
+
+The :mod:`IPython.kernel.core.notification` module will provide a simple
+implementation of a notification center and support for the observer pattern
+within the :mod:`IPython.kernel.core`. The main intended use case is to
+provide notification of Interpreter events to an observing frontend during the
+execution of a single block of code.
 
 Functional Requirements
 =======================
+
 The notification center must:
+
 	* Provide synchronous notification of events to all registered observers.
-	* Provide typed or labeled notification types
-	* Allow observers to register callbacks for individual or all notification types
-	* Allow observers to register callbacks for events from individual or all notifying objects
-	* Notification to the observer consists of the notification type, notifying object and user-supplied extra information [implementation: as keyword parameters to the registered callback]
+
+* Provide typed or labeled notification types.
+
+* Allow observers to register callbacks for individual or all notification
+  types.
+
+* Allow observers to register callbacks for events from individual or all
+  notifying objects.
+
+* Notification to the observer consists of the notification type, notifying
+  object and user-supplied extra information [implementation: as keyword
+  parameters to the registered callback].
+
 	* Perform as O(1) in the case of no registered observers.
+
 	* Permit out-of-process or cross-network extension.
 	
 What's not included
-==============================================================
+===================
+
 As written, the :mod:`IPython.kernel.core.notificaiton` module does not:
- 	* Provide out-of-process or network notifications [these should be handled by a separate, Twisted aware module in :mod:`IPython.kernel`].
-	* Provide zope.interface-style interfaces for the notification system [these should also be provided by the :mod:`IPython.kernel` module]
+
+* Provide out-of-process or network notifications (these should be handled by
+  a separate, Twisted aware module in :mod:`IPython.kernel`).
+
+* Provide zope.interface-style interfaces for the notification system (these
+  should also be provided by the :mod:`IPython.kernel` module).
 	
 Use Cases
 =========
+
 The following use cases describe the main intended uses of the notificaiton module and illustrate the main success scenario for each use case:
 
 	1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
+
 		from IPython.kernel.core.notification import NotificationCenter
 		center = NotificationCenter.sharedNotificationCenter
 		center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
 	
 	and elsewhere in his front end::
+
 		def stdout_notification(self, type, notifying_object, out_string=None):
 			self.writeStdOut(out_string)
 	
-	If everything works, the Interpreter will (according to its published API) fire a notification via the :data:`IPython.kernel.core.notification.sharedCenter` of type :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up to the Intereter implementation to figure out when to do this]. The notificaiton center will then call the registered callbacks for that event type (in this case, Dwight's frontend's stdout_notification method). Again, according to its API, the Interpreter provides an additional keyword argument when firing the notificaiton of out_string, a copy of the string it will write to stdout.
+If everything works, the Interpreter will (according to its published API)
+fire a notification via the
+:data:`IPython.kernel.core.notification.sharedCenter` of type
+:const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up
+to the Intereter implementation to figure out when to do this]. The
+notificaiton center will then call the registered callbacks for that event
+type (in this case, Dwight's frontend's stdout_notification method). Again,
+according to its API, the Interpreter provides an additional keyword argument
+when firing the notificaiton of out_string, a copy of the string it will write
+to stdout.
 
-	Like magic, Dwight's frontend is able to provide output, even during long-running calculations. Now if Jim could just convince Dwight to use Twisted...
+Like magic, Dwight's frontend is able to provide output, even during
+long-running calculations. Now if Jim could just convince Dwight to use
+Twisted...
 
 	2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
 	

docs/source/development/roadmap.txt

@@ -4,93 +4,78 b''
 Development roadmap
 ===================
 
-.. contents::
-
 IPython is an ambitious project that is still under heavy development.  However, we want IPython to become useful to as many people as possible, as quickly as possible.  To help us accomplish this, we are laying out a roadmap of where we are headed and what needs to happen to get there.  Hopefully, this will help the IPython developers figure out the best things to work on for each upcoming release.
 
-Speaking of releases, we are going to begin releasing a new version of IPython every four weeks.  We are hoping that a regular release schedule, along with a clear roadmap of where we are headed will propel the project forward.
-
-Where are we headed
-===================
+Work targeted to particular releases
+====================================
 
-Our goal with IPython is simple: to provide a *powerful*, *robust* and *easy to use* framework for parallel computing.  While there are other secondary goals you will hear us talking about at various times, this is the primary goal of IPython that frames the roadmap.
-
-Steps along the way
-===================
+Release 0.10
+------------
 
-Here we describe the various things that we need to work on to accomplish this goal.
+* Initial refactor of :command:`ipcluster`.
 
-Setting up for regular release schedule
----------------------------------------
+* Better TextMate integration.
 
-We would like to begin to release IPython regularly (probably a 4 week release cycle).  To get ready for this, we need to revisit the development guidelines and put in information about releasing IPython.  
+* Merge in the daemon branch.
 
-Process startup and management
-------------------------------
+Release 0.11
+------------
 
-IPython is implemented using a distributed set of processes that communicate using TCP/IP network channels.  Currently, users have to start each of the various processes separately using command line scripts.  This is both difficult and error prone.  Furthermore, there are a number of things that often need to be managed once the processes have been started, such as the sending of signals and the shutting down and cleaning up of processes.
+* Refactor the configuration system and command line options for
+  :command:`ipengine` and :command:`ipcontroller`. This will include the
+  creation of cluster directories that encapsulate all the configuration
+  files, log files and security related files for a particular cluster.
 
-We need to build a system that makes it trivial for users to start and manage IPython processes.  This system should have the following properties:
+* Refactor :command:`ipcluster` to support the new configuration system.
 
-	* It should possible to do everything through an extremely simple API that users
-	  can call from their own Python script.  No shell commands should be needed.
-	* This simple API should be configured using standard .ini files.
-	* The system should make it possible to start processes using a number of different
-	  approaches:  SSH, PBS/Torque, Xgrid, Windows Server, mpirun, etc.
-	* The controller and engine processes should each have a daemon for monitoring, 
-	  signaling and clean up.
-	* The system should be secure.
-	* The system should work under all the major operating systems, including
-	  Windows.
+* Refactor the daemon stuff to support the new configuration system.
 
-Initial work has begun on the daemon infrastructure, and some of the needed logic is contained in the ipcluster script.
+* Merge back in the core of the notebook.
 
-Ease of use/high-level approaches to parallelism
-------------------------------------------------
+Release 0.12
+------------
 
-While our current API for clients is well designed, we can still do a lot better in designing a user-facing API that is super simple.  The main goal here is that it should take *almost no extra code* for users to get their code running in parallel.  For this to be possible, we need to tie into Python's standard idioms that enable efficient coding.  The biggest ones we are looking at are using context managers (i.e., Python 2.5's ``with`` statement) and decorators.  Initial work on this front has begun, but more work is needed.
+* Fully integrate process startup with the daemons for full process 
+  management.
 
-We also need to think about new models for expressing parallelism.  This is fun work as most of the foundation has already been established.
+* Make the capabilites of :command:`ipcluster` available from simple Python 
+  classes.
 
-Security
---------
+Major areas of work
+===================
 
-Currently, IPython has no built in security or security model.  Because we would like IPython to be usable on public computer systems and over wide area networks, we need to come up with a robust solution for security.  Here are some of the specific things that need to be included:
+Refactoring the main IPython core
+---------------------------------
 
-	* User authentication between all processes (engines, controller and clients).
-	* Optional TSL/SSL based encryption of all communication channels.
-	* A good way of picking network ports so multiple users on the same system can
-	  run their own controller and engines without interfering with those of others.
-	* A clear model for security that enables users to evaluate the security risks
-	  associated with using IPython in various manners.
+Process management for :mod:`IPython.kernel`
+--------------------------------------------
 
-For the implementation of this, we plan on using Twisted's support for SSL and authentication.  One things that we really should look at is the `Foolscap`_ network protocol, which provides many of these things out of the box.
+Configuration system
+--------------------
 
-.. _Foolscap: http://foolscap.lothar.com/trac
+Performance problems
+--------------------
 
-The security work needs to be done in conjunction with other network protocol stuff.
+Currently, we have a number of performance issues that are waiting to bite users:
 
-Latent performance issues
--------------------------
+* The controller stores a large amount of state in Python dictionaries. Under
+  heavy usage, these dicts with get very large, causing memory usage problems.
+  We need to develop more scalable solutions to this problem, such as using a
+  sqlite database to store this state. This will also help the controller to
+  be more fault tolerant.
 
-Currently, we have a number of performance issues that are waiting to bite users:
+* We currently don't have a good way of handling large objects in the
+  controller. The biggest problem is that because we don't have any way of
+  streaming objects, we get lots of temporary copies in the low-level buffers.
+  We need to implement a better serialization approach and true streaming
+  support.
 
-	* The controller store a large amount of state in Python dictionaries.  Under heavy
-	  usage, these dicts with get very large, causing memory usage problems.  We need to 
-	  develop more scalable solutions to this problem, such as using a sqlite database
-	  to store this state.  This will also help the controller to be more fault tolerant.
-	* Currently, the client to controller connections are done through XML-RPC using 
-	  HTTP 1.0.  This is very inefficient as XML-RPC is a very verbose protocol and 
-	  each request must be handled with a new connection.  We need to move these network
-	  connections over to PB or Foolscap.
-	* We currently don't have a good way of handling large objects in the controller.
-	  The biggest problem is that because we don't have any way of streaming objects,
-	  we get lots of temporary copies in the low-level buffers.  We need to implement
-	  a better serialization approach and true streaming support.
 	* The controller currently unpickles and repickles objects.  We need to use the 
 	  [push|pull]_serialized methods instead.
-	* Currently the controller is a bottleneck.  We need the ability to scale the 
-	  controller by aggregating multiple controllers into one effective controller.
+
+* Currently the controller is a bottleneck. The best approach for this is to
+  separate the controller itself into multiple processes, one for the core
+  controller and one each for the controller interfaces.
 
 
 

docs/source/faq.txt

@@ -17,8 +17,11 b' Yes and no. When converting a serial code to run in parallel, there often many'
 difficulty questions that need to be answered, such as:
 
 	* How should data be decomposed onto the set of processors?
+
 	* What are the data movement patterns?
+
 	* Can the algorithm be structured to minimize data movement?
+
 	* Is dynamic load balancing important?
 
 We can't answer such questions for you. This is the hard (but fun) work of parallel
@@ -28,9 +31,7 b' resulting parallel code interactively.'
 
 With that said, if your problem is trivial to parallelize, IPython has a number of
 different interfaces that will enable you to parallelize things is almost no time at
-all.  A good place to start is the ``map`` method of our `multiengine interface`_.
-
-.. _multiengine interface: ./parallel_multiengine
+all.  A good place to start is the ``map`` method of our :class:`MultiEngineClient`.
 
 What is the best way to use MPI from Python?
 --------------------------------------------
@@ -44,21 +45,28 b' Some of the unique characteristic of IPython are:'
 	  parallel computation in such a way that new models of parallel computing
 	  can be explored quickly and easily.  If you don't like the models we
 	  provide, you can simply create your own using the capabilities we provide.
+  
 	* IPython is asynchronous from the ground up (we use `Twisted`_).
+
 	* IPython's architecture is designed to avoid subtle problems 
 	  that emerge because of Python's global interpreter lock (GIL).
-	* While IPython'1 architecture is designed to support a wide range
+  
+* While IPython's architecture is designed to support a wide range
 	  of novel parallel computing models, it is fully interoperable with
 	  traditional MPI applications.
+  
 	* IPython has been used and tested extensively on modern supercomputers.
+
 	* IPython's networking layers are completely modular.  Thus, is 
 	  straightforward to replace our existing network protocols with 
 	  high performance alternatives (ones based upon Myranet/Infiniband).
+  
 	* IPython is designed from the ground up to support collaborative
 	  parallel computing.  This enables multiple users to actively develop
 	  and run the *same* parallel computation.
+  
 	* Interactivity is a central goal for us.  While IPython does not have
-	  to be used interactivly, is can be.
+  to be used interactivly, it can be.
 	  
 .. _Twisted: http://www.twistedmatrix.com
 
@@ -72,11 +80,15 b' is structured in this way, you really should think about alternative ways of'
 handling the data movement. Here are some ideas:
 
 	1. Have the engines write data to files on the locals disks of the engines.
+
 	2. Have the engines write data to files on a file system that is shared by
 	   the engines.
+
 	3. Have the engines write data to a database that is shared by the engines.
+
 	4. Simply keep data in the persistent memory of the engines and move the
 	   computation to the data (rather than the data to the computation).
+
 	5. See if you can pass data directly between engines using MPI.
 
 Isn't Python slow to be used for high-performance parallel computing?

docs/source/history.txt

@@ -7,19 +7,22 b' History'
 Origins
 =======
 
-The current IPython system grew out of the following three projects:
+IPython was starting in 2001 by Fernando Perez.  IPython as we know it
+today grew out of the following three projects:
 
-    * [ipython] by Fernando Pérez. I was working on adding
+* ipython by Fernando Pérez. I was working on adding
       Mathematica-type prompts and a flexible configuration system
       (something better than $PYTHONSTARTUP) to the standard Python
       interactive interpreter.
-    * [IPP] by Janko Hauser. Very well organized, great usability. Had
+* IPP by Janko Hauser. Very well organized, great usability. Had
       an old help system. IPP was used as the 'container' code into
       which I added the functionality from ipython and LazyPython.
-    * [LazyPython] by Nathan Gray. Simple but very powerful. The quick
+* LazyPython by Nathan Gray. Simple but very powerful. The quick
       syntax (auto parens, auto quotes) and verbose/colored tracebacks
       were all taken from here.
 
+Here is how Fernando describes it:
+
 When I found out about IPP and LazyPython I tried to join all three
 into a unified system. I thought this could provide a very nice
 working environment, both for regular programming and scientific
@@ -28,29 +31,8 b' prompt history and great object introspection and help facilities. I'
 think it worked reasonably well, though it was a lot more work than I
 had initially planned.
 
+Today and how we got here
+=========================
 
-Current status
-==============
-
-The above listed features work, and quite well for the most part. But
-until a major internal restructuring is done (see below), only bug
-fixing will be done, no other features will be added (unless very minor
-and well localized in the cleaner parts of the code).
-
-IPython consists of some 18000 lines of pure python code, of which
-roughly two thirds is reasonably clean. The rest is, messy code which
-needs a massive restructuring before any further major work is done.
-Even the messy code is fairly well documented though, and most of the
-problems in the (non-existent) class design are well pointed to by a
-PyChecker run. So the rewriting work isn't that bad, it will just be
-time-consuming.
-
-
-Future
-------
+This needs to be filled in.
 
-See the separate new_design document for details. Ultimately, I would
-like to see IPython become part of the standard Python distribution as a
-'big brother with batteries' to the standard Python interactive
-interpreter. But that will never happen with the current state of the
-code, so all contributions are welcome.
  No newline at end of file

docs/source/index.txt

@@ -2,11 +2,15 b''
 IPython Documentation
 =====================
 
-Contents
-========
+.. htmlonly::
+
+   :Release: |release|
+   :Date: |today|
+
+   Contents:
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    overview.txt
    install/index.txt
@@ -20,9 +24,7 b' Contents'
    license_and_copyright.txt
    credits.txt
 
-Indices and tables
-==================
-
+.. htmlonly::
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`

docs/source/install/index.txt

@@ -7,5 +7,4 b' Installation'
 .. toctree::
    :maxdepth: 2
 
-   basic.txt
-   advanced.txt
+   install.txt

docs/source/interactive/index.txt

@@ -3,7 +3,7 b' Using IPython for interactive work'
 ==================================
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    tutorial.txt
    reference.txt

docs/source/interactive/reference.txt

@@ -1,14 +1,8 b''
-.. IPython documentation master file, created by sphinx-quickstart.py on Mon Mar 24 17:01:34 2008.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root 'toctree' directive.
-
 =================
 IPython reference
 =================
 
-.. contents::
-
-.. _Command line options:
+.. _command_line_options:
 
 Command-line usage
 ==================
@@ -288,12 +282,13 b' All options with a [no] prepended can be specified in negated form'
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
-	Specify the string used for input prompts. Note that if you
-	are using numbered prompts, the number is represented with a
-	'\#' in the string. Don't forget to quote strings with spaces
-	embedded in them. Default: 'In [\#]:'.  Sec. Prompts_
-	discusses in detail all the available escapes to customize
-	your prompts.
+
+        Specify the string used for input prompts. Note that if you are using
+	numbered prompts, the number is represented with a '\#' in the
+	string. Don't forget to quote strings with spaces embedded in
+	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
+	discusses in detail all the available escapes to customize your
+	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
@@ -2077,13 +2072,14 b' customizations.'
 Access to the standard Python help
 ----------------------------------
 
-As of Python 2.1, a help system is available with access to object
-docstrings and the Python manuals. Simply type 'help' (no quotes) to
-access it. You can also type help(object) to obtain information about a
-given object, and help('keyword') for information on a keyword. As noted
-in sec. `accessing help`_, you need to properly configure
-your environment variable PYTHONDOCS for this feature to work correctly.
+As of Python 2.1, a help system is available with access to object docstrings
+and the Python manuals. Simply type 'help' (no quotes) to access it. You can
+also type help(object) to obtain information about a given object, and
+help('keyword') for information on a keyword. As noted :ref:`here
+<accessing_help>`, you need to properly configure your environment variable
+PYTHONDOCS for this feature to work correctly.
 
+.. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
@@ -2126,7 +2122,7 b' are not really defined as separate identifiers. Try for example typing'
 {}.get? or after doing import os, type os.path.abspath??.
 
 
-.. _Readline:
+.. _readline:
 
 Readline-based features
 -----------------------
@@ -2240,10 +2236,9 b' explanation in your ipythonrc file.'
 Session logging and restoring
 -----------------------------
 
-You can log all input from a session either by starting IPython with
-the command line switches -log or -logfile (see sec. `command line
-options`_) or by activating the logging at any moment with the magic
-function %logstart.
+You can log all input from a session either by starting IPython with the
+command line switches -log or -logfile (see :ref:`here <command_line_options>`)
+or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
@@ -2279,6 +2274,8 b' resume logging to a file which had previously been started with'
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
+.. _system_shell_access:
+
 System shell access
 -------------------
 
@@ -2389,14 +2386,16 b" These features are basically a terminal version of Ka-Ping Yee's cgitb"
 module, now part of the standard Python library.
 
 
-.. _Input caching:
+.. _input_caching:
 
 Input caching system
 --------------------
 
-IPython offers numbered prompts (In/Out) with input and output caching.
-All input is saved and can be retrieved as variables (besides the usual
-arrow key recall).
+IPython offers numbered prompts (In/Out) with input and output caching
+(also referred to as 'input history'). All input is saved and can be 
+retrieved as variables (besides the usual arrow key recall), in 
+addition to the %rep magic command that brings a history entry
+up for editing on the next  command line.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 _i: stores previous input. _ii: next previous. _iii: next-next previous.
@@ -2429,7 +2428,43 b' sec. 6.2 <#sec:magic> for more details on the macro system.'
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
-.. _Output caching:
+You can also search ('grep') through your history by typing 
+'%hist -g somestring'. This also searches through the so called *shadow history*,
+which remembers all the commands (apart from multiline code blocks)
+you have ever entered. Handy for searching for svn/bzr URL's, IP adrresses
+etc. You can bring shadow history entries listed by '%hist -g' up for editing 
+(or re-execution by just pressing ENTER) with %rep command. Shadow history 
+entries are not available as _iNUMBER variables, and they are identified by 
+the '0' prefix in %hist -g output. That is, history entry 12 is a normal 
+history entry, but 0231 is a shadow history entry.
+
+Shadow history was added because the readline history is inherently very
+unsafe - if you have multiple IPython sessions open, the last session 
+to close will overwrite the history of previountly closed session. Likewise,
+if a crash occurs, history is never saved, whereas shadow history entries
+are added after entering every command (so a command executed 
+in another IPython session is immediately available in other IPython 
+sessions that are open).
+
+To conserve space, a command can exist in shadow history only once - it doesn't 
+make sense to store a common line like "cd .." a thousand times. The idea is 
+mainly to provide a reliable place where valuable, hard-to-remember commands can 
+always be retrieved, as opposed to providing an exact sequence of commands
+you have entered in actual order.
+
+Because shadow history has all the commands you have ever executed,
+time taken by %hist -g will increase oven time. If it ever starts to take
+too long (or it ends up containing sensitive information like passwords), 
+clear the shadow history by `%clear shadow_nuke`.
+
+Time taken to add entries to shadow history should be negligible, but 
+in any case, if you start noticing performance degradation after using 
+IPython for a long time (or running a script that floods the shadow history!), 
+you can 'compress' the shadow history by executing 
+`%clear shadow_compress`. In practice, this should never be necessary
+in normal use.
+
+.. _output_caching:
 
 Output caching system
 ---------------------
@@ -2472,7 +2507,7 b' Directory history'
 
 Your history of visited directories is kept in the global list _dh, and
 the magic %cd command can be used to go to any entry in that list. The
-%dhist command allows you to view this history. do ``cd -<TAB`` to
+%dhist command allows you to view this history. Do ``cd -<TAB`` to
 conventiently view the directory history.
 
 
@@ -3034,7 +3069,7 b' which is being shared by the interactive IPython loop and your GUI'
 thread, you should really handle it with thread locking and
 syncrhonization properties. The Python documentation discusses these.
 
-.. _Interactive demos:
+.. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
@@ -3143,21 +3178,17 b' toolkits, including Tk, GTK and WXPython. It also provides a number of'
 commands useful for scientific computing, all with a syntax compatible
 with that of the popular Matlab program.
 
-IPython accepts the special option -pylab (Sec. `Command line
-options`_). This configures it to support matplotlib, honoring the
-settings in the .matplotlibrc file. IPython will detect the user's
-choice of matplotlib GUI backend, and automatically select the proper
-threading model to prevent blocking. It also sets matplotlib in
-interactive mode and modifies %run slightly, so that any
-matplotlib-based script can be executed using %run and the final
-show() command does not block the interactive shell.
-
-The -pylab option must be given first in order for IPython to
-configure its threading mode. However, you can still issue other
-options afterwards. This allows you to have a matplotlib-based
-environment customized with additional modules using the standard
-IPython profile mechanism (Sec. Profiles_): ''ipython -pylab -p
-myprofile'' will load the profile defined in ipythonrc-myprofile after
-configuring matplotlib.
-
-
+IPython accepts the special option -pylab (see :ref:`here
+<command_line_options>`). This configures it to support matplotlib, honoring
+the settings in the .matplotlibrc file. IPython will detect the user's choice
+of matplotlib GUI backend, and automatically select the proper threading model
+to prevent blocking. It also sets matplotlib in interactive mode and modifies
+%run slightly, so that any matplotlib-based script can be executed using %run
+and the final show() command does not block the interactive shell.
+
+The -pylab option must be given first in order for IPython to configure its
+threading mode. However, you can still issue other options afterwards. This
+allows you to have a matplotlib-based environment customized with additional
+modules using the standard IPython profile mechanism (see :ref:`here
+<profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
+ipythonrc-myprofile after configuring matplotlib.

docs/source/interactive/tutorial.txt

@@ -4,8 +4,6 b''
 Quick IPython tutorial
 ======================
 
-.. contents::
-
 IPython can be used as an improved replacement for the Python prompt,
 and for that you don't really need to read any more of this manual. But
 in this section we'll try to summarize a few tips on how to make the
@@ -24,11 +22,11 b' Tab completion'
 --------------
 
 TAB-completion, especially for attributes, is a convenient way to explore the
-structure of any object you're dealing with. Simply type object_name.<TAB>
-and a list of the object's attributes will be printed (see readline_ for
-more). Tab completion also works on file and directory names, which combined
-with IPython's alias system allows you to do from within IPython many of the
-things you normally would need the system shell for.
+structure of any object you're dealing with. Simply type object_name.<TAB> and
+a list of the object's attributes will be printed (see :ref:`the readline
+section <readline>` for more). Tab completion also works on file and directory
+names, which combined with IPython's alias system allows you to do from within
+IPython many of the things you normally would need the system shell for.
 
 Explore your objects
 --------------------
@@ -39,18 +37,18 b' constructor details for classes. The magic commands %pdoc, %pdef, %psource'
 and %pfile will respectively print the docstring, function definition line,
 full source code and the complete file for any object (when they can be
 found). If automagic is on (it is by default), you don't need to type the '%'
-explicitly. See sec. `dynamic object information`_ for more.
+explicitly. See :ref:`this section <dynamic_object_info>` for more.
 
 The `%run` magic command
 ------------------------
 
-The %run magic command allows you to run any python script and load all of
-its data directly into the interactive namespace. Since the file is re-read
-from disk each time, changes you make to it are reflected immediately (in
-contrast to the behavior of import). I rarely use import for code I am
-testing, relying on %run instead. See magic_ section for more on this and
-other magic commands, or type the name of any magic command and ? to get
-details on it. See also sec. dreload_ for a recursive reload command. %run
+The %run magic command allows you to run any python script and load all of its
+data directly into the interactive namespace. Since the file is re-read from
+disk each time, changes you make to it are reflected immediately (in contrast
+to the behavior of import). I rarely use import for code I am testing, relying
+on %run instead. See :ref:`this section <magic>` for more on this and other
+magic commands, or type the name of any magic command and ? to get details on
+it. See also :ref:`this section <dreload>` for a recursive reload command. %run
 also has special flags for timing the execution of your scripts (-t) and for
 executing them under the control of either Python's pdb debugger (-d) or
 profiler (-p). With all of these, %run can be used as the main tool for
@@ -60,21 +58,21 b' choice.'
 Debug a Python script
 ---------------------
 
-Use the Python debugger, pdb. The %pdb command allows you to toggle on and
-off the automatic invocation of an IPython-enhanced pdb debugger (with
-coloring, tab completion and more) at any uncaught exception. The advantage
-of this is that pdb starts inside the function where the exception occurred,
-with all data still available. You can print variables, see code, execute
-statements and even walk up and down the call stack to track down the true
-source of the problem (which often is many layers in the stack above where
-the exception gets triggered). Running programs with %run and pdb active can
-be an efficient to develop and debug code, in many cases eliminating the need
-for print statements or external debugging tools. I often simply put a 1/0 in
-a place where I want to take a look so that pdb gets called, quickly view
-whatever variables I need to or test various pieces of code and then remove
-the 1/0. Note also that '%run -d' activates pdb and automatically sets
-initial breakpoints for you to step through your code, watch variables, etc.
-See Sec. `Output caching`_ for details.
+Use the Python debugger, pdb. The %pdb command allows you to toggle on and off
+the automatic invocation of an IPython-enhanced pdb debugger (with coloring,
+tab completion and more) at any uncaught exception. The advantage of this is
+that pdb starts inside the function where the exception occurred, with all data
+still available. You can print variables, see code, execute statements and even
+walk up and down the call stack to track down the true source of the problem
+(which often is many layers in the stack above where the exception gets
+triggered). Running programs with %run and pdb active can be an efficient to
+develop and debug code, in many cases eliminating the need for print statements
+or external debugging tools. I often simply put a 1/0 in a place where I want
+to take a look so that pdb gets called, quickly view whatever variables I need
+to or test various pieces of code and then remove the 1/0. Note also that '%run
+-d' activates pdb and automatically sets initial breakpoints for you to step
+through your code, watch variables, etc.  The :ref:`output caching section
+<output_caching>` has more details.
 
 Use the output cache
 --------------------
@@ -84,7 +82,8 b' and variables named _1, _2, etc. alias them. For example, the result of input'
 line 4 is available either as Out[4] or as _4. Additionally, three variables
 named _, __ and ___ are always kept updated with the for the last three
 results. This allows you to recall any previous result and further use it for
-new calculations. See Sec. `Output caching`_ for more.
+new calculations. See :ref:`the output caching section <output_caching>` for
+more.
 
 Suppress output
 ---------------
@@ -102,7 +101,7 b' A similar system exists for caching input. All input is stored in a global'
 list called In , so you can re-execute lines 22 through 28 plus line 34 by
 typing 'exec In[22:29]+In[34]' (using Python slicing notation). If you need
 to execute the same set of lines often, you can assign them to a macro with
-the %macro function. See sec. `Input caching`_ for more.
+the %macro function. See :ref:`here <input_caching>` for more.
 
 Use your input history
 ----------------------
@@ -134,17 +133,18 b' into Python variables.'
 Use Python variables when calling the shell
 -------------------------------------------
 
-Expand python variables when calling the shell (either via '!' and '!!' or
-via aliases) by prepending a $ in front of them. You can also expand complete
-python expressions. See `System shell access`_ for more.
+Expand python variables when calling the shell (either via '!' and '!!' or via
+aliases) by prepending a $ in front of them. You can also expand complete
+python expressions. See :ref:`our shell section <system_shell_access>` for
+more details.
 
 Use profiles
 ------------
 
 Use profiles to maintain different configurations (modules to load, function
 definitions, option settings) for particular tasks. You can then have
-customized versions of IPython for specific purposes. See sec. profiles_ for
-more.
+customized versions of IPython for specific purposes. :ref:`This section
+<profiles>` has more details.
 
 
 Embed IPython in your programs
@@ -152,7 +152,7 b' Embed IPython in your programs'
 
 A few lines of code are enough to load a complete IPython inside your own
 programs, giving you the ability to work with your data interactively after
-automatic processing has been completed. See sec. embedding_ for more.
+automatic processing has been completed. See :ref:`here <embedding>` for more.
 
 Use the Python profiler
 -----------------------
@@ -166,8 +166,8 b' Use IPython to present interactive demos'
 ----------------------------------------
 
 Use the IPython.demo.Demo class to load any Python script as an interactive
-demo. With a minimal amount of simple markup, you can control the execution
-of the script, stopping as needed. See sec. `interactive demos`_ for more.
+demo. With a minimal amount of simple markup, you can control the execution of
+the script, stopping as needed. See :ref:`here <interactive_demos>` for more.
 
 Run doctests
 ------------

docs/source/license_and_copyright.txt

@@ -1,61 +1,91 b''
 .. _license:
 
-=============================
+=====================
 License and Copyright 
-=============================
+=====================
 
-This files needs to be updated to reflect what the new COPYING.txt files says about our license and copyright!
+License
+=======
 
-IPython is released under the terms of the BSD license, whose general
-form can be found at: http://www.opensource.org/licenses/bsd-license.php. The full text of the
-IPython license is reproduced below::
+IPython is licensed under the terms of the new or revised BSD license, as follows::
 
-    IPython is released under a BSD-type license.
-
-    Copyright (c) 2001, 2002, 2003, 2004 Fernando Perez
-    <[e-mail]>.
-
-    Copyright (c) 2001 Janko Hauser <[e-mail]> and  
-    Nathaniel Gray <[e-mail]>.
+    Copyright (c) 2008, IPython Development Team
 
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    a. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-
-    b. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
-    c. Neither the name of the copyright holders nor the names of any
-    contributors to this software may be used to endorse or promote
-    products derived from this software without specific prior written
-    permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE. 
-
-Individual authors are the holders of the copyright for their code and
-are listed in each file.
+    modification, are permitted provided that the following conditions are
+    met:
+
+    Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+    Neither the name of the IPython Development Team nor the names of its
+    contributors may be used to endorse or promote products derived from this
+    software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+    THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+    PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+About the IPython Development Team
+==================================
+
+Fernando Perez began IPython in 2001 based on code from Janko Hauser
+<[e-mail]> and Nathaniel Gray <[e-mail]>. Fernando is still
+the project lead.
+
+The IPython Development Team is the set of all contributors to the IPython
+project. This includes all of the IPython subprojects. Here is a list of the
+currently active contributors:
+
+	* Matthieu Brucher
+	* Ondrej Certik
+	* Laurent Dufrechou
+	* Robert Kern
+	* Brian E. Granger
+	* Fernando Perez (project leader)
+	* Benjamin Ragan-Kelley
+	* Ville M. Vainio
+	* Gael Varoququx
+	* Stefan van der Walt
+	* Tech-X Corporation
+	* Barry Wark
+
+If your name is missing, please add it.  
+
+Our Copyright Policy
+====================
+
+IPython uses a shared copyright model. Each contributor maintains copyright
+over their contributions to IPython. But, it is important to note that these
+contributions are typically only changes to the repositories. Thus, the
+IPython source code, in its entirety is not the copyright of any single person
+or institution. Instead, it is the collective copyright of the entire IPython
+Development Team. If individual contributors want to maintain a record of what
+changes/contributions they have specific copyright on, they should indicate
+their copyright in the commit message of the change, when they commit the
+change to one of the IPython repositories.
+
+Miscellaneous
+=============
 
 Some files (DPyGetOpt.py, for example) may be licensed under different
-conditions. Ultimately each file indicates clearly the conditions under
-which its author/authors have decided to publish the code.
+conditions. Ultimately each file indicates clearly the conditions under which
+its author/authors have decided to publish the code.
 
-Versions of IPython up to and including 0.6.3 were released under the
-GNU Lesser General Public License (LGPL), available at
+Versions of IPython up to and including 0.6.3 were released under the GNU
+Lesser General Public License (LGPL), available at
 http://www.gnu.org/copyleft/lesser.html.
  No newline at end of file

docs/source/overview.txt

@@ -14,7 +14,7 b' However, the interpreter supplied with the standard Python distribution'
 is somewhat limited for extended interactive use.
 
 The goal of IPython is to create a comprehensive environment for
-interactive and exploratory computing.  To support, this goal, IPython
+interactive and exploratory computing.  To support this goal, IPython
 has two main components:
 
 	* An enhanced interactive Python shell.
@@ -25,7 +25,8 b' All of IPython is open source (released under the revised BSD license).'
 Enhanced interactive Python shell
 =================================
 
-IPython's interactive shell (`ipython`), has the following goals:
+IPython's interactive shell (:command:`ipython`), has the following goals,
+amongst others:
 
    1. Provide an interactive shell superior to Python's default. IPython
       has many features for object introspection, system shell access,
@@ -33,17 +34,21 b" IPython's interactive shell (`ipython`), has the following goals:"
       working interactively. It tries to be a very efficient environment
       both for Python code development and for exploration of problems
       using Python objects (in situations like data analysis).
+  
    2. Serve as an embeddable, ready to use interpreter for your own
       programs. IPython can be started with a single call from inside
       another program, providing access to the current namespace. This
       can be very useful both for debugging purposes and for situations
       where a blend of batch-processing and interactive exploration are
-      needed.
+   needed.  New in the 0.9 version of IPython is a reusable wxPython
+   based IPython widget.
+  
    3. Offer a flexible framework which can be used as the base
       environment for other systems with Python as the underlying
       language. Specifically scientific environments like Mathematica,
       IDL and Matlab inspired its design, but similar ideas can be
       useful in many fields.
+  
    4. Allow interactive testing of threaded graphical toolkits. IPython
       has support for interactive, non-blocking control of GTK, Qt and
       WX applications via special threading flags. The normal Python
@@ -56,74 +61,95 b' Main features of the interactive shell'
 	  definition prototypes, source code, source files and other details
 	  of any object accessible to the interpreter with a single
 	  keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
+  
 	* Searching through modules and namespaces with :samp:`*` wildcards, both
 	  when using the :samp:`?` system and via the :samp:`%psearch` command.
+
 	* Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
 	  This works for keywords, modules, methods, variables and files in the
 	  current directory. This is supported via the readline library, and
 	  full access to configuring readline's behavior is provided. 
 	  Custom completers can be implemented easily for different purposes
 	  (system commands, magic arguments etc.)
+
 	* Numbered input/output prompts with command history (persistent
 	  across sessions and tied to each profile), full searching in this
 	  history and caching of all input and output.
+
 	* User-extensible 'magic' commands. A set of commands prefixed with
 	  :samp:`%` is available for controlling IPython itself and provides
 	  directory control, namespace information and many aliases to
 	  common system shell commands.
+
 	* Alias facility for defining your own system aliases.
+
 	* Complete system shell access. Lines starting with :samp:`!` are passed
 	  directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
 	  captures shell output into python variables for further use.
+
 	* Background execution of Python commands in a separate thread.
 	  IPython has an internal job manager called jobs, and a
-	  conveninence backgrounding magic function called :samp:`%bg`.
+  convenience backgrounding magic function called :samp:`%bg`.
+
 	* The ability to expand python variables when calling the system
 	  shell. In a shell command, any python variable prefixed with :samp:`$` is
 	  expanded. A double :samp:`$$` allows passing a literal :samp:`$` to the shell (for
 	  access to shell and environment variables like :envvar:`PATH`).
+
 	* Filesystem navigation, via a magic :samp:`%cd` command, along with a
 	  persistent bookmark system (using :samp:`%bookmark`) for fast access to
 	  frequently visited directories.
+
 	* A lightweight persistence framework via the :samp:`%store` command, which
 	  allows you to save arbitrary Python variables. These get restored
 	  automatically when your session restarts.
+
 	* Automatic indentation (optional) of code as you type (through the
 	  readline library).
+
 	* Macro system for quickly re-executing multiple lines of previous
 	  input with a single name. Macros can be stored persistently via
 	  :samp:`%store` and edited via :samp:`%edit`.
+
 	* Session logging (you can then later use these logs as code in your
 	  programs). Logs can optionally timestamp all input, and also store
 	  session output (marked as comments, so the log remains valid
 	  Python source code).
+
 	* Session restoring: logs can be replayed to restore a previous
 	  session to the state where you left it.
+
 	* Verbose and colored exception traceback printouts. Easier to parse
 	  visually, and in verbose mode they produce a lot of useful
 	  debugging information (basically a terminal version of the cgitb
 	  module).
+
 	* Auto-parentheses: callable objects can be executed without
 	  parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
+
 	* Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
 	  auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
 	  automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
 	  becomes :samp:`my_function("a b")`.
+
 	* Extensible input syntax. You can define filters that pre-process
 	  user input to simplify input in special situations. This allows
 	  for example pasting multi-line code fragments which start with
 	  :samp:`>>>` or :samp:`...` such as those from other python sessions or the
 	  standard Python documentation.
+
 	* Flexible configuration system. It uses a configuration file which
 	  allows permanent setting of all command-line options, module
 	  loading, code and file execution. The system allows recursive file
 	  inclusion, so you can have a base file with defaults and layers
 	  which load other customizations for particular projects.
+
 	* Embeddable. You can call IPython as a python shell inside your own
 	  python programs. This can be used both for debugging code or for
 	  providing interactive abilities to your programs with knowledge
 	  about the local namespaces (very useful in debugging and data
 	  analysis situations).
+
 	* Easy debugger access. You can set IPython to call up an enhanced
 	  version of the Python debugger (pdb) every time there is an
 	  uncaught exception. This drops you inside the code which triggered
@@ -135,11 +161,13 b' Main features of the interactive shell'
 	  tab-completion and traceback coloring support. For even easier
 	  debugger access, try :samp:`%debug` after seeing an exception. winpdb is 
 	  also supported, see ipy_winpdb extension.
+
 	* Profiler support. You can run single statements (similar to
 	  :samp:`profile.run()`) or complete programs under the profiler's control.
 	  While this is possible with standard cProfile or profile modules,
 	  IPython wraps this functionality with magic commands (see :samp:`%prun`
 	  and :samp:`%run -p`) convenient for rapid interactive work.
+
 	* Doctest support. The special :samp:`%doctest_mode` command toggles a mode
 	  that allows you to paste existing doctests (with leading :samp:`>>>`
 	  prompts and whitespace) and uses doctest-compatible prompts and
@@ -153,6 +181,37 b' architecture within IPython that allows such hardware to be used quickly and eas'
 from Python. Moreover, this architecture is designed to support interactive and
 collaborative parallel computing.
 
+The main features of this system are:
+
+* Quickly parallelize Python code from an interactive Python/IPython session.
+
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
+
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.  And all of these styles can be handled
+  interactively.
+
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Write parallel code that will run unchanged on everything from multicore
+  workstations to supercomputers.
+
+* Full integration with Message Passing libraries (MPI).
+
+* Capabilities based security model with full encryption of network connections.
+
+* Share live parallel jobs with other users securely.  We call this collaborative
+  parallel computing.
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling.  Python exceptions raised in parallel execution are
+  gathered and presented to the top-level code.
+
 For more information, see our :ref:`overview <parallel_index>` of using IPython for
 parallel computing.
 

docs/source/parallel/index.txt

@@ -1,17 +1,16 b''
 .. _parallel_index:
 
 ====================================
-Using IPython for Parallel computing
+Using IPython for parallel computing
 ====================================
 
-User Documentation
-==================
-
 .. toctree::
    :maxdepth: 2
 
    parallel_intro.txt
+   parallel_process.txt
    parallel_multiengine.txt
    parallel_task.txt
    parallel_mpi.txt
+   parallel_security.txt
 

docs/source/parallel/parallel_intro.txt

@@ -1,23 +1,20 b''
 .. _ip1par:
 
-======================================
-Using IPython for parallel computing
-======================================
-
-.. contents::
+============================
+Overview and getting started
+============================
 
 Introduction
 ============
 
-This file gives an overview of IPython. IPython has a sophisticated and
-powerful architecture for parallel and distributed computing. This
-architecture abstracts out parallelism in a very general way, which
-enables IPython to support many different styles of parallelism
-including:
+This section gives an overview of IPython's sophisticated and powerful
+architecture for parallel and distributed computing. This architecture
+abstracts out parallelism in a very general way, which enables IPython to
+support many different styles of parallelism including:
 
 	* Single program, multiple data (SPMD) parallelism.
 	* Multiple program, multiple data (MPMD) parallelism.
-	* Message passing using ``MPI``.
+* Message passing using MPI.
 	* Task farming.
 	* Data parallel.
 	* Combinations of these approaches.
@@ -30,18 +27,24 b' the ``I`` in IPython. The following are some example usage cases for IPython:'
 	* Quickly parallelize algorithms that are embarrassingly parallel
 	  using a number of simple approaches.  Many simple things can be
 	  parallelized interactively in one or two lines of code.
+
 	* Steer traditional MPI applications on a supercomputer from an
 	  IPython session on your laptop.
+
 	* Analyze and visualize large datasets (that could be remote and/or
 	  distributed) interactively using IPython and tools like
 	  matplotlib/TVTK.
+
 	* Develop, test and debug new parallel algorithms 
 	  (that may use MPI) interactively.
+
 	* Tie together multiple MPI jobs running on different systems into
 	  one giant distributed and parallel system.
+
 	* Start a parallel job on your cluster and then have a remote 	
 	  collaborator connect to it and pull back data into their 
 	  local IPython session for plotting and analysis.
+
 	* Run a set of tasks on a set of CPUs using dynamic load balancing.
 
 Architecture overview
@@ -51,7 +54,12 b' The IPython architecture consists of three components:'
 
 	* The IPython engine.
 	* The IPython controller.
-	* Various controller Clients.
+* Various controller clients.
+
+These components live in the :mod:`IPython.kernel` package and are
+installed with IPython.  They do, however, have additional dependencies
+that must be installed.  For more information, see our
+:ref:`installation documentation <install_index>`.
 
 IPython engine
 ---------------
@@ -75,16 +83,21 b' IPython engines can connect. For each connected engine, the controller'
 manages a queue. All actions that can be performed on the engine go
 through this queue. While the engines themselves block when user code is
 run, the controller hides that from the user to provide a fully
-asynchronous interface to a set of engines. Because the controller
-listens on a network port for engines to connect to it, it must be
-started before any engines are started.  
+asynchronous interface to a set of engines. 
+
+.. note::
+
+    Because the controller listens on a network port for engines to 
+    connect to it, it must be started *before* any engines are started.  
 
 The controller also provides a single point of contact for users who wish
 to utilize the engines connected to the controller. There are different
 ways of working with a controller. In IPython these ways correspond to different interfaces that the controller is adapted to.  Currently we have two default interfaces to the controller: 
 
-	* The MultiEngine interface.
-	* The Task interface.
+* The MultiEngine interface, which provides the simplest possible way of
+  working with engines interactively.
+* The Task interface, which provides presents the engines as a load balanced
+  task farming system.
 
 Advanced users can easily add new custom interfaces to enable other
 styles of parallelism. 
@@ -100,126 +113,58 b' Controller clients'
 
 For each controller interface, there is a corresponding client. These
 clients allow users to interact with a set of engines through the
-interface.
+interface.  Here are the two default clients:
+
+* The :class:`MultiEngineClient` class.
+* The :class:`TaskClient` class.
 
 Security
 --------
 
-By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a `capabilities`__ based security model.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in, if not you can't.  
-
-.. __: http://en.wikipedia.org/wiki/Capability-based_security
-
-In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named something.furl.  The default location of these files is your ~./ipython directory.
-
-To connect and authenticate to the controller an engine or client simply needs to present an appropriate furl (that was originally created by the controller) to the controller.  Thus, the .furl files need to be copied to a location where the clients and engines can find them.  Typically, this is the ~./ipython directory on the host where the client/engine is running (which could be a different host than the controller).  Once the .furl files are copied over, everything should work fine.
-
-Getting Started
-===============
-
-To use IPython for parallel computing, you need to start one instance of
-the controller and one or more instances of the engine. The controller
-and each engine can run on different machines or on the same machine.
-Because of this, there are many different possibilities for setting up
-the IP addresses and ports used by the various processes.
-
-Starting the controller and engine on your local machine
---------------------------------------------------------
-
-This is the simplest configuration that can be used and is useful for
-testing the system and on machines that have multiple cores and/or
-multple CPUs. The easiest way of doing this is using the ``ipcluster``
-command::
-
-	$ ipcluster -n 4
-	
-This will start an IPython controller and then 4 engines that connect to
-the controller. Lastly, the script will print out the Python commands
-that you can use to connect to the controller. It is that easy.
+By default (as long as `pyOpenSSL` is installed) all network connections between the controller and engines and the controller and clients are secure.  What does this mean?  First of all, all of the connections will be encrypted using SSL.  Second, the connections are authenticated.  We handle authentication in a capability based security model [Capability]_.  In this model,  a "capability (known in some systems as a key) is a communicable, unforgeable token of authority".  Put simply, a capability is like a key to your house.  If you have the key to your house, you can get in.  If not, you can't.  
 
-Underneath the hood, the ``ipcluster`` script uses two other top-level
-scripts that you can also use yourself. These scripts are
-``ipcontroller``, which starts the controller and ``ipengine`` which
-starts one engine. To use these scripts to start things on your local
-machine, do the following.
+In our architecture, the controller is the only process that listens on network ports, and is thus responsible to creating these keys.  In IPython, these keys are known as Foolscap URLs, or FURLs, because of the underlying network protocol we are using.  As a user, you don't need to know anything about the details of these FURLs, other than that when the controller starts, it saves a set of FURLs to files named :file:`something.furl`.  The default location of these files is the :file:`~./ipython/security` directory.
 
-First start the controller::
+To connect and authenticate to the controller an engine or client simply needs to present an appropriate FURL (that was originally created by the controller) to the controller.  Thus, the FURL files need to be copied to a location where the clients and engines can find them.  Typically, this is the :file:`~./ipython/security` directory on the host where the client/engine is running (which could be a different host than the controller).  Once the FURL files are copied over, everything should work fine.
 
-	$ ipcontroller &
+Currently, there are three FURL files that the controller creates:
 	
-Next, start however many instances of the engine you want using (repeatedly) the command::
+ipcontroller-engine.furl
+    This FURL file is the key that gives an engine the ability to connect
+    to a controller.
 
-	$ ipengine &
+ipcontroller-tc.furl
+    This FURL file is the key that a :class:`TaskClient` must use to
+    connect to the task interface of a controller.
 
-.. warning:: 
+ipcontroller-mec.furl
+    This FURL file is the key that a :class:`MultiEngineClient` must use
+    to connect to the multiengine interface of a controller.
 	
-	The order of the above operations is very important.  You *must*
- 	start the controller before the engines, since the engines connect 
-	to the controller as they get started.
+More details of how these FURL files are used are given below.
 
-On some platforms you may need to give these commands in the form
-``(ipcontroller &)`` and ``(ipengine &)`` for them to work properly. The
-engines should start and automatically connect to the controller on the
-default ports, which are chosen for this type of setup. You are now ready
-to use the controller and engines from IPython.
+A detailed description of the security model and its implementation in IPython
+can be found :ref:`here <parallelsecurity>`.
 
-Starting the controller and engines on different machines
----------------------------------------------------------
-
-This section needs to be updated to reflect the new Foolscap capabilities based
-model.
-
-Using ``ipcluster`` with ``ssh``
---------------------------------
-
-The ``ipcluster`` command can also start a controller and engines using
-``ssh``.  We need more documentation on this, but for now here is any
-example startup script::
-
-	controller = dict(host='myhost',
-	                  engine_port=None, # default is 10105
-	                  control_port=None,
-	                  )
-
-	# keys are hostnames, values are the number of engine on that host
-	engines = dict(node1=2,
-	               node2=2,
-	               node3=2,
-	               node3=2,
-	               )
-
-Starting engines using ``mpirun``
----------------------------------
-
-The IPython engines can be started using ``mpirun``/``mpiexec``, even if
-the engines don't call MPI_Init() or use the MPI API in any way. This is
-supported on modern MPI implementations like `Open MPI`_.. This provides
-an really nice way of starting a bunch of engine. On a system with MPI
-installed you can do::
-
-	mpirun -n 4 ipengine --controller-port=10000 --controller-ip=host0
-
-.. _Open MPI: http://www.open-mpi.org/
-
-More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
+Getting Started
+===============
 
-Log files
----------
+To use IPython for parallel computing, you need to start one instance of
+the controller and one or more instances of the engine. Initially, it is best to simply start a controller and engines on a single host using the :command:`ipcluster` command.  To start a controller and 4 engines on you localhost, just do::
 
-All of the components of IPython have log files associated with them.
-These log files can be extremely useful in debugging problems with
-IPython and can be found in the directory ``~/.ipython/log``.  Sending
-the log files to us will often help us to debug any problems.
+    $ ipcluster local -n 4
 
-Next Steps
-==========
+More details about starting the IPython controller and engines can be found :ref:`here <parallel_process>`
 
 Once you have started the IPython controller and one or more engines, you
-are ready to use the engines to do somnething useful. To make sure
-everything is working correctly, try the following commands::
+are ready to use the engines to do something useful. To make sure
+everything is working correctly, try the following commands:
+
+.. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
-	In [2]: mec = client.MultiEngineClient()    # This looks for .furl files in ~./ipython
+	In [2]: mec = client.MultiEngineClient()
 	
 	In [4]: mec.get_ids()
 	Out[4]: [0, 1, 2, 3]
@@ -239,4 +184,22 b' everything is working correctly, try the following commands::'
 	[3] In [1]: print "Hello World"
 	[3] Out[1]: Hello World
 
-If this works, you are ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+Remember, a client also needs to present a FURL file to the controller.  How does this happen?  When a multiengine client is created with no arguments, the client tries to find the corresponding FURL file in the local :file:`~./ipython/security` directory.  If it finds it, you are set.  If you have put the FURL file in a different location or it has a different name, create the client like this::
+
+    mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
+
+Same thing hold true of creating a task client::
+
+    tc = client.TaskClient('/path/to/my/ipcontroller-tc.furl')
+
+You are now ready to learn more about the :ref:`MultiEngine <parallelmultiengine>` and :ref:`Task <paralleltask>` interfaces to the controller.
+
+.. note:: 
+
+    Don't forget that the engine, multiengine client and task client all have
+    *different* furl files. You must move *each* of these around to an
+    appropriate location so that the engines and clients can use them to
+    connect to the controller.
+
+.. [Capability] Capability-based security, http://en.wikipedia.org/wiki/Capability-based_security
+

docs/source/parallel/parallel_mpi.txt

@@ -4,19 +4,154 b''
 Using MPI with IPython
 =======================
 
-The simplest way of getting started with MPI is to install an MPI implementation
-(we recommend `Open MPI`_) and `mpi4py`_ and then start the engines using the
-``mpirun`` command::
+Often, a parallel algorithm will require moving data between the engines.  One way of accomplishing this is by doing a pull and then a push using the multiengine client.  However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination.
 
-	mpirun -n 4 ipengine --mpi=mpi4py
+A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_.  IPython's parallel computing architecture has been designed from the ground up to integrate with MPI.  This document describes how to use MPI with IPython.
 	
-This will automatically import `mpi4py`_ and make sure that `MPI_Init` is called
-at the right time. We also have built in support for `PyTrilinos`_, which can be
-used (assuming `PyTrilinos`_ is installed) by starting the engines with::
+Additional installation requirements
+====================================
+
+If you want to use MPI with IPython, you will need to install:
+
+* A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH.
+* The mpi4py [mpi4py]_ package.
+
+.. note::
+
+    The mpi4py package is not a strict requirement. However, you need to
+    have *some* way of calling MPI from Python. You also need some way of
+    making sure that :func:`MPI_Init` is called when the IPython engines start
+    up. There are a number of ways of doing this and a good number of
+    associated subtleties. We highly recommend just using mpi4py as it
+    takes care of most of these problems. If you want to do something
+    different, let us know and we can help you get started.
+
+Starting the engines with MPI enabled
+=====================================
+
+To use code that calls MPI, there are typically two things that MPI requires.
+
+1. The process that wants to call MPI must be started using
+   :command:`mpirun` or a batch system (like PBS) that has MPI support.
+2. Once the process starts, it must call :func:`MPI_Init`.
+
+There are a couple of ways that you can start the IPython engines and get these things to happen.
+
+Automatic starting using :command:`mpirun` and :command:`ipcluster`
+-------------------------------------------------------------------
+
+The easiest approach is to use the `mpirun` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpirun`::
+
+    $ ipcluster mpirun -n 4
+
+This approach is best as interrupting :command:`ipcluster` will automatically
+stop and clean up the controller and engines.
+
+Manual starting using :command:`mpirun`
+---------------------------------------
+
+If you want to start the IPython engines using the :command:`mpirun`, just do::
+
+    $ mpirun -n 4 ipengine --mpi=mpi4py
+
+This requires that you already have a controller running and that the FURL
+files for the engines are in place. We also have built in support for
+PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by
+starting the engines with::
 
 	mpirun -n 4 ipengine --mpi=pytrilinos
 
-.. _MPI: http://www-unix.mcs.anl.gov/mpi/
-.. _mpi4py: http://mpi4py.scipy.org/
-.. _Open MPI: http://www.open-mpi.org/
-.. _PyTrilinos: http://trilinos.sandia.gov/packages/pytrilinos/
  No newline at end of file
+Automatic starting using PBS and :command:`ipcluster`
+-----------------------------------------------------
+
+The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`.
+
+Actually using MPI
+==================
+
+Once the engines are running with MPI enabled, you are ready to go.  You can now call any code that uses MPI in the IPython engines.  And, all of this can be done interactively.  Here we show a simple example that uses mpi4py [mpi4py]_.
+
+First, lets define a simply function that uses MPI to calculate the sum of a distributed array.  Save the following text in a file called :file:`psum.py`:
+
+.. sourcecode:: python
+
+    from mpi4py import MPI
+    import numpy as np
+    
+    def psum(a):
+        s = np.sum(a)
+        return MPI.COMM_WORLD.Allreduce(s,MPI.SUM)
+
+Now, start an IPython cluster in the same directory as :file:`psum.py`::
+
+    $ ipcluster mpirun -n 4
+
+Finally, connect to the cluster and use this function interactively.  In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function:
+
+.. sourcecode:: ipython
+
+    In [1]: from IPython.kernel import client
+    
+    In [2]: mec = client.MultiEngineClient()
+    
+    In [3]: mec.activate()
+    
+    In [4]: px import numpy as np
+    Parallel execution on engines: all
+    Out[4]: 
+    <Results List>
+    [0] In [13]: import numpy as np
+    [1] In [13]: import numpy as np
+    [2] In [13]: import numpy as np
+    [3] In [13]: import numpy as np
+    
+    In [6]: px a = np.random.rand(100)
+    Parallel execution on engines: all
+    Out[6]: 
+    <Results List>
+    [0] In [15]: a = np.random.rand(100)
+    [1] In [15]: a = np.random.rand(100)
+    [2] In [15]: a = np.random.rand(100)
+    [3] In [15]: a = np.random.rand(100)
+    
+    In [7]: px from psum import psum
+    Parallel execution on engines: all
+    Out[7]: 
+    <Results List>
+    [0] In [16]: from psum import psum
+    [1] In [16]: from psum import psum
+    [2] In [16]: from psum import psum
+    [3] In [16]: from psum import psum
+    
+    In [8]: px s = psum(a)
+    Parallel execution on engines: all
+    Out[8]: 
+    <Results List>
+    [0] In [17]: s = psum(a)
+    [1] In [17]: s = psum(a)
+    [2] In [17]: s = psum(a)
+    [3] In [17]: s = psum(a)
+    
+    In [9]: px print s
+    Parallel execution on engines: all
+    Out[9]: 
+    <Results List>
+    [0] In [18]: print s
+    [0] Out[18]: 187.451545803
+    
+    [1] In [18]: print s
+    [1] Out[18]: 187.451545803
+    
+    [2] In [18]: print s
+    [2] Out[18]: 187.451545803
+
+    [3] In [18]: print s
+    [3] Out[18]: 187.451545803
+
+Any Python code that makes calls to MPI can be used in this manner, including
+compiled C, C++ and Fortran libraries that have been exposed to Python.
+
+.. [MPI] Message Passing Interface.  http://www-unix.mcs.anl.gov/mpi/
+.. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/
+.. [OpenMPI] Open MPI. http://www.open-mpi.org/
+.. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/
  No newline at end of file

docs/source/parallel/parallel_multiengine.txt

@@ -1,58 +1,126 b''
 .. _parallelmultiengine:
 
-=================================
-IPython's MultiEngine interface
-=================================
-
-.. contents::
-
-The MultiEngine interface represents one possible way of working with a
-set of IPython engines. The basic idea behind the MultiEngine interface is
-that the capabilities of each engine are explicitly exposed to the user.
-Thus, in the MultiEngine interface, each engine is given an id that is
-used to identify the engine and give it work to do. This interface is very
-intuitive and is designed with interactive usage in mind, and is thus the
-best place for new users of IPython to begin.
+===============================
+IPython's multiengine interface
+===============================
+
+The multiengine interface represents one possible way of working with a set of
+IPython engines. The basic idea behind the multiengine interface is that the
+capabilities of each engine are directly and explicitly exposed to the user.
+Thus, in the multiengine interface, each engine is given an id that is used to
+identify the engine and give it work to do. This interface is very intuitive
+and is designed with interactive usage in mind, and is thus the best place for
+new users of IPython to begin.
 
 Starting the IPython controller and engines
 ===========================================
 
 To follow along with this tutorial, you will need to start the IPython
-controller and four IPython engines. The simplest way of doing this is to
-use the ``ipcluster`` command::
+controller and four IPython engines. The simplest way of doing this is to use
+the :command:`ipcluster` command::
 
-	$ ipcluster -n 4
+	$ ipcluster local -n 4
 	
-For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+For more detailed information about starting the controller and engines, see
+our :ref:`introduction <ip1par>` to using IPython for parallel computing.
 
 Creating a ``MultiEngineClient`` instance
 =========================================
 
-The first step is to import the IPython ``client`` module and then create a ``MultiEngineClient`` instance::
+The first step is to import the IPython :mod:`IPython.kernel.client` module
+and then create a :class:`MultiEngineClient` instance:
+
+.. sourcecode:: ipython
 
 	In [1]: from IPython.kernel import client
 	
 	In [2]: mec = client.MultiEngineClient()
 
-To make sure there are engines connected to the controller, use can get a list of engine ids::
+This form assumes that the :file:`ipcontroller-mec.furl` is in the
+:file:`~./ipython/security` directory on the client's host. If not, the
+location of the FURL file must be given as an argument to the
+constructor:
+
+.. sourcecode:: ipython
+
+    In [2]: mec = client.MultiEngineClient('/path/to/my/ipcontroller-mec.furl')
+
+To make sure there are engines connected to the controller, use can get a list
+of engine ids:
+
+.. sourcecode:: ipython
 
 	In [3]: mec.get_ids()
 	Out[3]: [0, 1, 2, 3]
 
 Here we see that there are four engines ready to do work for us.
 
+Quick and easy parallelism
+==========================
+
+In many cases, you simply want to apply a Python function to a sequence of objects, but *in parallel*.  The multiengine interface provides two simple ways of accomplishing this:  a parallel version of :func:`map` and ``@parallel`` function decorator.
+
+Parallel map
+------------
+
+Python's builtin :func:`map` functions allows a function to be applied to a
+sequence element-by-element. This type of code is typically trivial to
+parallelize. In fact, the multiengine interface in IPython already has a
+parallel version of :meth:`map` that works just like its serial counterpart:
+
+.. sourcecode:: ipython
+
+	In [63]: serial_result = map(lambda x:x**10, range(32))
+
+	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
+
+	In [65]: serial_result==parallel_result
+	Out[65]: True
+
+.. note::
+
+    The multiengine interface version of :meth:`map` does not do any load
+    balancing.  For a load balanced version, see the task interface.
+
+.. seealso::
+
+    The :meth:`map` method has a number of options that can be controlled by
+    the :meth:`mapper` method.  See its docstring for more information.
+
+Parallel function decorator
+---------------------------
+
+Parallel functions are just like normal function, but they can be called on sequences and *in parallel*.  The multiengine interface provides a decorator that turns any Python function into a parallel function:
+
+.. sourcecode:: ipython
+
+    In [10]: @mec.parallel()
+       ....: def f(x):
+       ....:     return 10.0*x**4
+       ....: 
+
+    In [11]: f(range(32))    # this is done in parallel
+    Out[11]: 
+    [0.0,10.0,160.0,...]
+
+See the docstring for the :meth:`parallel` decorator for options.
+
 Running Python commands
 =======================
 
-The most basic type of operation that can be performed on the engines is to execute Python code. Executing Python code can be done in blocking or non-blocking mode (blocking is default) using the ``execute`` method.
+The most basic type of operation that can be performed on the engines is to
+execute Python code. Executing Python code can be done in blocking or
+non-blocking mode (blocking is default) using the :meth:`execute` method.
 
 Blocking execution
 ------------------
 
-In blocking mode, the ``MultiEngineClient`` object (called ``mec`` in
+In blocking mode, the :class:`MultiEngineClient` object (called ``mec`` in
 these examples) submits the command to the controller, which places the
-command in the engines' queues for execution. The ``execute`` call then
-blocks until the engines are done executing the command::
+command in the engines' queues for execution. The :meth:`execute` call then
+blocks until the engines are done executing the command:
+
+.. sourcecode:: ipython
 
 	# The default is to run on all engines
 	In [4]: mec.execute('a=5')
@@ -71,7 +139,10 b' blocks until the engines are done executing the command::'
 	[2] In [2]: b=10
 	[3] In [2]: b=10
 
-Python commands can be executed on specific engines by calling execute using the ``targets`` keyword argument::
+Python commands can be executed on specific engines by calling execute using
+the ``targets`` keyword argument:
+
+.. sourcecode:: ipython
 
 	In [6]: mec.execute('c=a+b',targets=[0,2])
 	Out[6]: 
@@ -102,7 +173,11 b' Python commands can be executed on specific engines by calling execute using the'
 	[3] In [4]: print c
 	[3] Out[4]: -5
 
-This example also shows one of the most important things about the IPython engines: they have a persistent user namespaces.  The ``execute`` method returns a Python ``dict`` that contains useful information::
+This example also shows one of the most important things about the IPython
+engines: they have a persistent user namespaces. The :meth:`execute` method
+returns a Python ``dict`` that contains useful information:
+
+.. sourcecode:: ipython
 
 	In [9]: result_dict = mec.execute('d=10; print d')
 
@@ -118,10 +193,14 b' This example also shows one of the most important things about the IPython engin'
 Non-blocking execution
 ----------------------
 
-In non-blocking mode, ``execute`` submits the command to be executed and then returns a
-``PendingResult`` object immediately. The ``PendingResult`` object gives you a way of getting a
-result at a later time through its ``get_result`` method or ``r`` attribute. This allows you to
-quickly submit long running commands without blocking your local Python/IPython session::
+In non-blocking mode, :meth:`execute` submits the command to be executed and
+then returns a :class:`PendingResult` object immediately. The
+:class:`PendingResult` object gives you a way of getting a result at a later
+time through its :meth:`get_result` method or :attr:`r` attribute. This allows
+you to quickly submit long running commands without blocking your local
+Python/IPython session:
+
+.. sourcecode:: ipython
 
 	# In blocking mode
 	In [6]: mec.execute('import time')
@@ -159,7 +238,12 b' quickly submit long running commands without blocking your local Python/IPython '
 	[2] In [3]: time.sleep(10)
 	[3] In [3]: time.sleep(10)
 
-Often, it is desirable to wait until a set of ``PendingResult`` objects are done.  For this, there is a the method ``barrier``.  This method takes a tuple of ``PendingResult`` objects and blocks until all of the associated results are ready::
+Often, it is desirable to wait until a set of :class:`PendingResult` objects
+are done. For this, there is a the method :meth:`barrier`. This method takes a
+tuple of :class:`PendingResult` objects and blocks until all of the associated
+results are ready:
+
+.. sourcecode:: ipython
 
 	In [72]: mec.block=False
 
@@ -182,16 +266,20 b' Often, it is desirable to wait until a set of ``PendingResult`` objects are done'
 The ``block`` and ``targets`` keyword arguments and attributes
 --------------------------------------------------------------
 
-Most commands in the multiengine interface (like ``execute``) accept ``block`` and ``targets``
-as keyword arguments. As we have seen above, these keyword arguments control the blocking mode
-and which engines the command is applied to. The ``MultiEngineClient`` class also has ``block``
-and ``targets`` attributes that control the default behavior when the keyword arguments are not
-provided. Thus the following logic is used for ``block`` and ``targets``:
+Most methods in the multiengine interface (like :meth:`execute`) accept
+``block`` and ``targets`` as keyword arguments. As we have seen above, these
+keyword arguments control the blocking mode and which engines the command is
+applied to. The :class:`MultiEngineClient` class also has :attr:`block` and
+:attr:`targets` attributes that control the default behavior when the keyword
+arguments are not provided. Thus the following logic is used for :attr:`block`
+and :attr:`targets`:
 
 	* If no keyword argument is provided, the instance attributes are used.
 	* Keyword argument, if provided override the instance attributes.
 
-The following examples demonstrate how to use the instance attributes::
+The following examples demonstrate how to use the instance attributes:
+
+.. sourcecode:: ipython
 
 	In [16]: mec.targets = [0,2]
 
@@ -225,14 +313,21 b' The following examples demonstrate how to use the instance attributes::'
 	[3] In [6]: b=10; print b
 	[3] Out[6]: 10
 
-The ``block`` and ``targets`` instance attributes also determine the behavior of the parallel
-magic commands...
+The :attr:`block` and :attr:`targets` instance attributes also determine the
+behavior of the parallel magic commands.
 
 
 Parallel magic commands
 -----------------------
 
-We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``) that make it more pleasant to execute Python commands on the engines interactively. These are simply shortcuts to ``execute`` and ``get_result``. The ``%px`` magic executes a single Python command on the engines specified by the `magicTargets``targets` attribute of the ``MultiEngineClient`` instance (by default this is 'all')::
+We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``)
+that make it more pleasant to execute Python commands on the engines
+interactively. These are simply shortcuts to :meth:`execute` and
+:meth:`get_result`. The ``%px`` magic executes a single Python command on the
+engines specified by the :attr:`targets` attribute of the
+:class:`MultiEngineClient` instance (by default this is ``'all'``):
+
+.. sourcecode:: ipython
 
 	# Make this MultiEngineClient active for parallel magic commands
 	In [23]: mec.activate()
@@ -277,7 +372,11 b' We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``) t'
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
-The ``%result`` magic gets and prints the stdin/stdout/stderr of the last command executed on each engine.  It is simply a shortcut to the ``get_result`` method::
+The ``%result`` magic gets and prints the stdin/stdout/stderr of the last
+command executed on each engine. It is simply a shortcut to the
+:meth:`get_result` method:
+
+.. sourcecode:: ipython
 
 	In [29]: %result
 	Out[29]:
@@ -294,7 +393,10 b' The ``%result`` magic gets and prints the stdin/stdout/stderr of the last comman'
 	[3] In [9]: print numpy.linalg.eigvals(a)
 	[3] Out[9]: [ 0.83664764 -0.25602658]
 
-The ``%autopx`` magic switches to a mode where everything you type is executed on the engines given by the ``targets`` attribute::
+The ``%autopx`` magic switches to a mode where everything you type is executed
+on the engines given by the :attr:`targets` attribute:
+
+.. sourcecode:: ipython
 
 	In [30]: mec.block=False
 
@@ -335,51 +437,21 b' The ``%autopx`` magic switches to a mode where everything you type is executed o'
 	[3] In [12]: print "Average max eigenvalue is: ", sum(max_evals)/len(max_evals)
 	[3] Out[12]: Average max eigenvalue is:  10.1158837784
 
-Using the ``with`` statement of Python 2.5
-------------------------------------------
-
-Python 2.5 introduced the ``with`` statement.  The ``MultiEngineClient`` can be used with the ``with`` statement to execute a block of code on the engines indicated by the ``targets`` attribute::
-
-	In [3]: with mec:
-	   ...:     client.remote()    # Required so the following code is not run locally
-	   ...:     a = 10
-	   ...:     b = 30
-	   ...:     c = a+b
-	   ...:     
-	   ...:     
-
-	In [4]: mec.get_result()
-	Out[4]: 
-	<Results List>
-	[0] In [1]: a = 10
-	b = 30
-	c = a+b
-
-	[1] In [1]: a = 10
-	b = 30
-	c = a+b
 
-	[2] In [1]: a = 10
-	b = 30
-	c = a+b
+Moving Python objects around
+============================
 
-	[3] In [1]: a = 10
-	b = 30
-	c = a+b
-
-This is basically another way of calling execute, but one with allows you to avoid writing code in strings.  When used in this way, the attributes ``targets`` and ``block`` are used to control how the code is executed.  For now, if you run code in non-blocking mode you won't have access to the ``PendingResult``.
-
-Moving Python object around
-===========================
-
-In addition to executing code on engines, you can transfer Python objects to and from your
-IPython session and the engines. In IPython, these operations are called ``push`` (sending an
-object to the engines) and ``pull`` (getting an object from the engines).
+In addition to executing code on engines, you can transfer Python objects to
+and from your IPython session and the engines. In IPython, these operations
+are called :meth:`push` (sending an object to the engines) and :meth:`pull`
+(getting an object from the engines).
 
 Basic push and pull
 -------------------
 
-Here are some examples of how you use ``push`` and ``pull``::
+Here are some examples of how you use :meth:`push` and :meth:`pull`:
+
+.. sourcecode:: ipython
 
 	In [38]: mec.push(dict(a=1.03234,b=3453))
 	Out[38]: [None, None, None, None]
@@ -415,7 +487,10 b' Here are some examples of how you use ``push`` and ``pull``::'
 	[3] In [13]: print c
 	[3] Out[13]: speed
 
-In non-blocking mode ``push`` and ``pull`` also return ``PendingResult`` objects::
+In non-blocking mode :meth:`push` and :meth:`pull` also return
+:class:`PendingResult` objects:
+
+.. sourcecode:: ipython
 
 	In [47]: mec.block=False
 
@@ -428,7 +503,12 b' In non-blocking mode ``push`` and ``pull`` also return ``PendingResult`` objects'
 Push and pull for functions
 ---------------------------
 
-Functions can also be pushed and pulled using ``push_function`` and ``pull_function``::
+Functions can also be pushed and pulled using :meth:`push_function` and
+:meth:`pull_function`:
+
+.. sourcecode:: ipython
+
+    In [52]: mec.block=True
 
 	In [53]: def f(x):
 	   ....:     return 2.0*x**4
@@ -466,7 +546,12 b' Functions can also be pushed and pulled using ``push_function`` and ``pull_funct'
 Dictionary interface
 --------------------
 
-As a shorthand to ``push`` and ``pull``, the ``MultiEngineClient`` class implements some of the Python dictionary interface. This make the remote namespaces of the engines appear as a local dictionary. Underneath, this uses ``push`` and ``pull``::
+As a shorthand to :meth:`push` and :meth:`pull`, the
+:class:`MultiEngineClient` class implements some of the Python dictionary
+interface. This make the remote namespaces of the engines appear as a local
+dictionary. Underneath, this uses :meth:`push` and :meth:`pull`:
+
+.. sourcecode:: ipython
 
 	In [50]: mec.block=True
 
@@ -478,11 +563,15 b' As a shorthand to ``push`` and ``pull``, the ``MultiEngineClient`` class impleme'
 Scatter and gather
 ------------------
 
-Sometimes it is useful to partition a sequence and push the partitions to different engines. In
-MPI language, this is know as scatter/gather and we follow that terminology. However, it is
-important to remember that in IPython ``scatter`` is from the interactive IPython session to
-the engines and ``gather`` is from the engines back to the interactive IPython session. For
-scatter/gather operations between engines, MPI should be used::
+Sometimes it is useful to partition a sequence and push the partitions to
+different engines. In MPI language, this is know as scatter/gather and we
+follow that terminology. However, it is important to remember that in
+IPython's :class:`MultiEngineClient` class, :meth:`scatter` is from the
+interactive IPython session to the engines and :meth:`gather` is from the
+engines back to the interactive IPython session. For scatter/gather operations
+between engines, MPI should be used:
+
+.. sourcecode:: ipython
 
 	In [58]: mec.scatter('a',range(16))
 	Out[58]: [None, None, None, None]
@@ -510,24 +599,14 b' scatter/gather operations between engines, MPI should be used::'
 Other things to look at
 =======================
 
-Parallel map
-------------
-
-Python's builtin ``map`` functions allows a function to be applied to a sequence element-by-element.  This type of code is typically trivial to parallelize.  In fact, the MultiEngine interface in IPython already has a parallel version of ``map`` that works just like its serial counterpart::
-
-	In [63]: serial_result = map(lambda x:x**10, range(32))
-
-	In [64]: parallel_result = mec.map(lambda x:x**10, range(32))
-
-	In [65]: serial_result==parallel_result
-	Out[65]: True
-
-As you would expect, the parallel version of ``map`` is also influenced by the ``block`` and ``targets`` keyword arguments and attributes.
-
 How to do parallel list comprehensions
 --------------------------------------
 
-In many cases list comprehensions are nicer than using the map function.  While we don't have fully parallel list comprehensions, it is simple to get the basic effect using ``scatter`` and ``gather``::
+In many cases list comprehensions are nicer than using the map function. While
+we don't have fully parallel list comprehensions, it is simple to get the
+basic effect using :meth:`scatter` and :meth:`gather`:
+
+.. sourcecode:: ipython
 
 	In [66]: mec.scatter('x',range(64))
 	Out[66]: [None, None, None, None]
@@ -547,10 +626,18 b' In many cases list comprehensions are nicer than using the map function. While '
 	In [69]: print y
 	[0, 1, 1024, 59049, 1048576, 9765625, 60466176, 282475249, 1073741824,...]
 
-Parallel Exceptions
+Parallel exceptions
 -------------------
 
-In the MultiEngine interface, parallel commands can raise Python exceptions, just like serial commands.  But, it is a little subtle, because a single parallel command can actually raise multiple exceptions (one for each engine the command was run on).  To express this idea, the MultiEngine interface has a ``CompositeError`` exception class that will be raised in most cases.  The ``CompositeError`` class is a special type of exception that wraps one or more other types of exceptions.  Here is how it works::
+In the multiengine interface, parallel commands can raise Python exceptions,
+just like serial commands. But, it is a little subtle, because a single
+parallel command can actually raise multiple exceptions (one for each engine
+the command was run on). To express this idea, the MultiEngine interface has a
+:exc:`CompositeError` exception class that will be raised in most cases. The
+:exc:`CompositeError` class is a special type of exception that wraps one or
+more other types of exceptions. Here is how it works:
+
+.. sourcecode:: ipython
 
 	In [76]: mec.block=True
 
@@ -580,7 +667,9 b' In the MultiEngine interface, parallel commands can raise Python exceptions, jus'
 	[2:execute]: ZeroDivisionError: integer division or modulo by zero
 	[3:execute]: ZeroDivisionError: integer division or modulo by zero
 
-Notice how the error message printed when ``CompositeError`` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions::
+Notice how the error message printed when :exc:`CompositeError` is raised has information about the individual exceptions that were raised on each engine.  If you want, you can even raise one of these original exceptions:
+
+.. sourcecode:: ipython
 
 	In [80]: try:
 	   ....:     mec.execute('1/0')
@@ -602,7 +691,11 b' Notice how the error message printed when ``CompositeError`` is raised has infor'
 
 	ZeroDivisionError: integer division or modulo by zero
 
-If you are working in IPython, you can simple type ``%debug`` after one of these ``CompositeError`` is raised, and inspect the exception instance::
+If you are working in IPython, you can simple type ``%debug`` after one of
+these :exc:`CompositeError` exceptions is raised, and inspect the exception
+instance:
+
+.. sourcecode:: ipython
 
 	In [81]: mec.execute('1/0')
 	---------------------------------------------------------------------------
@@ -679,7 +772,14 b' If you are working in IPython, you can simple type ``%debug`` after one of these'
 
 	ZeroDivisionError: integer division or modulo by zero
 
-All of this same error handling magic even works in non-blocking mode::
+.. note::
+
+    The above example appears to be broken right now because of a change in 
+    how we are using Twisted.
+
+All of this same error handling magic even works in non-blocking mode:
+
+.. sourcecode:: ipython
 
 	In [83]: mec.block=False