main branch resync

IPython/external/argparse.py

@@ -0,0 +1,1867 b''
+# -*- coding: utf-8 -*-
+
+# Copyright � 2006 Steven J. Bethard <[e-mail]>.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted under the terms of the 3-clause BSD
+# license. No warranty expressed or implied.
+# For details, see the accompanying file LICENSE.txt.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+* handles both optional and positional arguments
+* produces highly informative usage messages
+* supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file:
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    HelpFormatter, RawDescriptionHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default, while
+        RawDescriptionHelpFormatter tells the parser not to perform any
+        line-wrapping on description text.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '0.8.0'
+
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = '==PARSER=='
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format:
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+    
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+    
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+        
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join(func(*args) for func, args in self.items)
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty            
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, optionals, positionals, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, optionals, positionals, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations            
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+                    
+            # update the maximum item length
+            invocation_length = max(len(s) for s in invocations)
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list        
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+
+    def format_help(self):
+        help = self._root_section.format_help() % dict(prog=self._prog)
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join(part
+                       for part in part_strings
+                       if part and part is not SUPPRESS)
+
+    def _format_usage(self, usage, optionals, positionals, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if no optionals or positionals are available, usage is just prog
+        if usage is None and not optionals and not positionals:
+            usage = '%(prog)s'
+            
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            usage = '%(prog)s' % dict(prog=self._prog)
+                
+            # determine width of "usage: PROG" and width of text
+            prefix_width = len(prefix) + len(usage) + 1
+            prefix_indent = self._current_indent + prefix_width
+            text_width = self._width - self._current_indent
+
+            # put them on one line if they're short enough
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals)
+            if prefix_width + len(action_usage) + 1 < text_width:
+                usage = '%s %s' % (usage, action_usage)
+            
+            # if they're long, wrap optionals and positionals individually
+            else:
+                optional_usage = format(optionals)
+                positional_usage = format(positionals)
+                indent = ' ' * prefix_indent
+
+                # usage is made of PROG, optionals and positionals
+                parts = [usage, ' ']
+                
+                # options always get added right after PROG
+                if optional_usage:
+                    parts.append(_textwrap.fill(
+                        optional_usage, text_width,
+                        initial_indent=indent,
+                        subsequent_indent=indent).lstrip())
+
+                # if there were options, put arguments on the next line
+                # otherwise, start them right after PROG
+                if positional_usage:
+                    part = _textwrap.fill(
+                        positional_usage, text_width,
+                        initial_indent=indent,
+                        subsequent_indent=indent).lstrip()
+                    if optional_usage:
+                        part = '\n' + indent + part
+                    parts.append(part)
+                usage = ''.join(parts)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions):
+        parts = []
+        for action in actions:
+            if action.help is SUPPRESS:
+                continue
+
+            # produce all arg strings        
+            if not action.option_strings:
+                parts.append(self._format_args(action, action.dest))
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required
+                if not action.required:
+                    part = '[%s]' % part
+                parts.append(part)
+
+        return ' '.join(parts)
+
+    def _format_text(self, text):
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string            
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            return self._format_metavar(action, action.dest)
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _format_metavar(self, action, default_metavar):
+        if action.metavar is not None:
+            name = action.metavar
+        elif action.choices is not None:
+            choice_strs = (str(choice) for choice in action.choices)
+            name = '{%s}' % ','.join(choice_strs)
+        else:
+            name = default_metavar
+        return name
+
+    def _format_args(self, action, default_metavar):
+        name = self._format_metavar(action, default_metavar)
+        if action.nargs is None:
+            result = name
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % name
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % (name, name)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % (name, name)
+        elif action.nargs is PARSER:
+            result = '%s ...' % name
+        else:
+            result = ' '.join([name] * action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name, value in params.items():
+            if value is SUPPRESS:
+                del params[name]
+        if params.get('choices') is not None:
+            choices_str = ', '.join(str(c) for c in params['choices'])
+            params['choices'] = choices_str
+        return action.help % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    
+    def _fill_text(self, text, width, indent):
+        return ''.join(indent + line for line in text.splitlines(True))
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+# =====================
+# Options and Arguments
+# =====================
+
+class ArgumentError(Exception):
+    """ArgumentError(message, argument)
+
+    Raised whenever there was an error creating or using an argument
+    (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+    
+    def __init__(self, argument, message):
+        if argument.option_strings:
+            self.argument_name =  '/'.join(argument.option_strings)
+        elif argument.metavar not in (None, SUPPRESS):
+            self.argument_name = argument.metavar
+        elif argument.dest not in (None, SUPPRESS):
+            self.argument_name = argument.dest
+        else:
+            self.argument_name = None
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Action(*strings, **options)
+
+    Action objects hold the information necessary to convert a
+    set of command-line arguments (possibly including an initial option
+    string) into the desired Python object(s).
+
+    Keyword Arguments:
+
+    option_strings -- A list of command-line option strings which
+        should be associated with this action.
+    
+    dest -- The name of the attribute to hold the created object(s)
+    
+    nargs -- The number of command-line arguments that should be consumed.
+        By default, one argument will be consumed and a single value will
+        be produced.  Other values include:
+            * N (an integer) consumes N arguments (and produces a list)
+            * '?' consumes zero or one arguments
+            * '*' consumes zero or more arguments (and produces a list)
+            * '+' consumes one or more arguments (and produces a list)
+        Note that the difference between the default and nargs=1 is that
+        with the default, a single value will be produced, while with
+        nargs=1, a list containing a single value will be produced.
+
+    const -- The value to be produced if the option is specified and the
+        option uses an action that takes no values.
+
+    default -- The value to be produced if the option is not specified.
+
+    type -- The type which the command-line arguments should be converted
+        to, should be one of 'string', 'int', 'float', 'complex' or a
+        callable object that accepts a single string argument. If None,
+        'string' is assumed.
+        
+    choices -- A container of values that should be allowed. If not None,
+        after a command-line argument has been converted to the appropriate
+        type, an exception will be raised if it is not a member of this
+        collection.
+
+    required -- True if the action must always be specified at the command
+        line. This is only meaningful for optional command-line arguments.
+
+    help -- The help string describing the argument.
+
+    metavar -- The name to be used for the option's argument with the help
+        string. If None, the 'dest' value will be used as the name.
+    """
+
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar'
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+class _StoreAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs must be > 0')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+class _StoreConstAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+class _StoreTrueAction(_StoreConstAction):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+class _StoreFalseAction(_StoreConstAction):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+    
+class _AppendAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs must be > 0')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        _ensure_value(namespace, self.dest, []).append(values)
+
+class _AppendConstAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        _ensure_value(namespace, self.dest, []).append(self.const)
+
+class _CountAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+class _HelpAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+class _VersionAction(Action):
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_version()
+        parser.exit()
+        
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+        def __init__(self, name, help):
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=name, help=help)
+
+        
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+        
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = {}
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser        
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            tup = parser_name, ', '.join(self._name_parser_map)
+            msg = _('unknown parser %r (choices: %s)' % tup)
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        parser.parse_args(arg_strings, namespace)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+    mode -- A string indicating how the file is to be opened. Accepts the
+        same values as the builtin open() function.
+    bufsize -- The file's desired buffer size. Accepts the same values as
+        the builtin open() function.
+    """    
+    def __init__(self, mode='r', bufsize=None):
+        self._mode = mode
+        self._bufsize = bufsize
+    
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if self._mode == 'r':
+                return _sys.stdin
+            elif self._mode == 'w':
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r' % self._mode)
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        if self._bufsize:
+            return open(string, self._mode, self._bufsize)
+        else:
+            return open(string, self._mode)
+
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+
+    def __init__(self, **kwargs):
+        for name, value in kwargs.iteritems():
+            setattr(self, name, value)
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class _ActionsContainer(object):
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+        
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._optional_actions_list = []
+        self._positional_actions_list = []
+        self._positional_actions_full_list = []
+        self._option_strings = {}
+
+        # defaults storage
+        self._defaults = {}
+
+    # ====================
+    # Registration methods
+    # ====================
+
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default settings methods
+    # ==================================
+
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action_list in [self._option_strings.values(),
+                            self._positional_actions_full_list]:
+            for action in action_list:
+                if action.dest in kwargs:
+                    action.default = kwargs[action.dest]
+
+    # =======================
+    # Adding argument actions
+    # =======================
+
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+            
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        action = action_class(**kwargs)
+        return self._add_action(action)
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to optional or positional list
+        if action.option_strings:
+            self._optional_actions_list.append(action)
+        else:
+            self._positional_actions_list.append(action)
+            self._positional_actions_full_list.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_strings[option_string] = action
+
+        # return the created action
+        return action
+
+    def _add_container_actions(self, container):
+        for action in container._optional_actions_list:
+            self._add_action(action)
+        for action in container._positional_actions_list:
+            self._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+        
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on one-or-fewer-character option strings
+            if len(option_string) < 2:
+                msg = _('invalid option string %r: '
+                        'must be at least two characters long')
+                raise ValueError(msg % option_string)
+
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                msg = _('invalid option string %r: '
+                        'must start with a character %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
+
+            # error on strings that are all prefix characters
+            if not (set(option_string) - set(self.prefix_chars)):
+                msg = _('invalid option string %r: '
+                        'must contain characters other than %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)            
+            if option_string[0] in self.prefix_chars:
+                if option_string[1] in self.prefix_chars:
+                    long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option        
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_strings:
+                confl_optional = self._option_strings[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = _('conflicting option string(s): %s')
+        conflict_string = ', '.join(option_string
+                                    for option_string, action
+                                    in conflicting_actions)
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options        
+        for option_string, action in conflicting_actions:
+            
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_strings.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._optional_actions_list.remove(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+        
+        self.title = title
+        self._registries = container._registries
+        self._positional_actions_full_list = container._positional_actions_full_list
+        self._option_strings = container._option_strings
+        self._defaults = container._defaults
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.add_help = add_help
+
+        self._argument_group_class = _ArgumentGroup
+        self._has_subparsers = False
+        self._argument_groups = []
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        if self.add_help:
+            self.add_argument(
+                '-h', '--help', action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                '-v', '--version', action='version', default=SUPPRESS,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+|-\d*.\d+$')
+                
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+
+    def add_argument_group(self, *args, **kwargs):
+        group = self._argument_group_class(self, *args, **kwargs)
+        self._argument_groups.append(group)
+        return group
+
+    def add_subparsers(self, **kwargs):
+        if self._has_subparsers:
+            self.error(_('cannot have multiple subparser arguments'))
+        
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            formatter.add_usage(self.usage, [],
+                                self._get_positional_actions(), '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._positional_actions_list.append(action)
+        self._positional_actions_full_list.append(action)
+        self._has_subparsers = True
+
+        # return the created parsers action
+        return action
+
+    def _add_container_actions(self, container):
+        super(ArgumentParser, self)._add_container_actions(container)
+        try:
+            groups = container._argument_groups
+        except AttributeError:
+            pass
+        else:
+            for group in groups:
+                new_group = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+                new_group._add_container_actions(group)
+
+    def _get_optional_actions(self):
+        actions = []
+        actions.extend(self._optional_actions_list)
+        for argument_group in self._argument_groups:
+            actions.extend(argument_group._optional_actions_list)
+        return actions
+
+    def _get_positional_actions(self):
+        return list(self._positional_actions_full_list)
+
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+
+    def parse_args(self, args=None, namespace=None):
+        # args default to the system args
+        if args is None:
+            args = _sys.argv[1:]
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+            
+        # add any action defaults that aren't present
+        optional_actions = self._get_optional_actions()
+        positional_actions = self._get_positional_actions()
+        for action in optional_actions + positional_actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        default = action.default
+                        if isinstance(action.default, basestring):
+                            default = self._get_value(action, default)
+                        setattr(namespace, action.dest, default)
+
+        # add any parser defaults that aren't present
+        for dest, value in self._defaults.iteritems():
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, value)
+            
+        # parse the arguments and exit if there are any errors
+        try:
+            result = self._parse_args(args, namespace)
+        except ArgumentError, err:
+            self.error(str(err))
+
+        # make sure all required optionals are present
+        for action in self._get_optional_actions():
+            if action.required:
+                if getattr(result, action.dest, None) is None:
+                    opt_strs = '/'.join(action.option_strings)
+                    msg = _('option %s is required' % opt_strs)
+                    self.error(msg)
+
+        # return the parsed arguments                    
+        return result
+
+    def _parse_args(self, arg_strings, namespace):            
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        def take_action(action, argument_strings, option_string=None):
+            argument_values = self._get_values(action, argument_strings)
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)            
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+            
+            # determine the optional action and parse any explicit
+            # argument out of the option string
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # loop because single-dash options can be chained
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, raise an error
+                if action is None:
+                    self.error(_('no such option: %s') % option_string)
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        parse_optional = self._parse_optional
+                        for char in self.prefix_chars:
+                            option_string = char + explicit_arg
+                            option_tuple = parse_optional(option_string)
+                            if option_tuple[0] is not None:
+                                break
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                        # set the action, etc. for the next loop iteration
+                        action, option_string, explicit_arg = option_tuple
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+                
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+            
+            # consume any Positionals preceding the next option
+            next_option_string_index = min(
+                index
+                for index in option_string_indices
+                if index >= start_index)
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were unparseable
+            # arguments
+            if start_index not in option_string_indices:
+                msg = _('extra arguments found: %s')
+                extras = arg_strings[start_index:next_option_string_index]
+                self.error(msg % ' '.join(extras))
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were too
+        # many supplied
+        if stop_index != len(arg_strings):
+            extras = arg_strings[stop_index:]
+            self.error(_('extra arguments found: %s') % ' '.join(extras))
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # return the updated namespace
+        return namespace
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match        
+        if match is None:
+            nargs_errors = {
+                None:_('expected one argument'),
+                OPTIONAL:_('expected at most one argument'),
+                ONE_OR_MORE:_('expected at least one argument')
+            }
+            default = _('expected %s argument(s)') % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in xrange(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join(self._get_nargs_pattern(action)
+                              for action in actions_slice)
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend(len(string) for string in match.groups())
+                break
+
+        # return the list of arg string counts
+        return result
+    
+    def _parse_optional(self, arg_string):
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+        
+        # if it's just dashes, it was meant to be positional
+        if not arg_string.strip('-'):
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_strings:
+            action = self._option_strings[arg_string]
+            return action, arg_string, None
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = []
+        prefix_tuples = self._get_option_prefix_tuples(arg_string)
+        for option_string in self._option_strings:
+            for option_prefix, explicit_arg in prefix_tuples:
+                if option_string.startswith(option_prefix):
+                    action = self._option_strings[option_string]
+                    tup = action, option_string, explicit_arg
+                    option_tuples.append(tup)
+                    break
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join(opt_str for _, opt_str, _ in option_tuples)
+            tup = arg_string, options
+            self.error(_('ambiguous option: %s could match %s') % tup)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        if self._negative_number_matcher.match(arg_string):
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_prefix_tuples(self, option_string):
+        result = []
+        
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            tup = option_prefix, explicit_arg
+            result.append(tup)
+
+        # option strings starting with a single prefix character are
+        # split at all indices
+        else:
+            for first_index, char in enumerate(option_string):
+                if char not in self.prefix_chars:
+                    break
+            for i in xrange(len(option_string), first_index, -1):
+                tup = option_string[:i], option_string[i:] or None
+                result.append(tup)
+
+        # return the collected prefix tuples
+        return result                
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+        
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs is PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers            
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed        
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER args, strip out '--'
+        if action.nargs is not PARSER:
+            arg_strings = [s for s in arg_strings if s != '--']
+        
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, basestring):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+        
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs is PARSER:
+            value = list(self._get_value(action, v) for v in arg_strings)
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = list(self._get_value(action, v) for v in arg_strings)
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value            
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+        
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # TypeErrors or ValueErrors indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = _('invalid %s value: %r')
+            raise ArgumentError(action, msg % (name, arg_string))
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            tup = value, ', '.join(map(repr, action.choices))
+            msg = _('invalid choice: %r (choose from %s)') % tup
+            raise ArgumentError(action, msg)
+
+        
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage,
+                            self._get_optional_actions(),
+                            self._get_positional_actions())
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage,
+                            self._get_optional_actions(),
+                            self._get_positional_actions())
+
+        # description        
+        formatter.add_text(self.description)
+
+        # positionals
+        formatter.start_section(_('positional arguments'))
+        formatter.add_arguments(self._positional_actions_list)
+        formatter.end_section()
+
+        # optionals
+        formatter.start_section(_('optional arguments'))
+        formatter.add_arguments(self._optional_actions_list)
+        formatter.end_section()
+
+        # user-defined groups
+        for argument_group in self._argument_groups:
+            formatter.start_section(argument_group.title)
+            formatter.add_text(argument_group.description)
+            formatter.add_arguments(argument_group._positional_actions_list)
+            formatter.add_arguments(argument_group._optional_actions_list)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+
+    def print_usage(self, file=None):
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+
+    # ===============
+    # Exiting methods
+    # ===============
+
+    def exit(self, status=0, message=None):
+        if message:
+            _sys.stderr.write(message)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+        
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        self.exit(2, _('%s: error: %s\n') % (self.prog, message))

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -0,0 +1,155 b''
+# encoding: utf-8
+
+"""This file contains unittests for the frontendbase module."""
+
+__docformat__ = "restructuredtext en"
+
+#---------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                         
+#                                                                           
+#  Distributed under the terms of the BSD License.  The full license is in  
+#  the file COPYING, distributed as part of this software.                  
+#---------------------------------------------------------------------------
+                                                                            
+#---------------------------------------------------------------------------
+# Imports                                                                   
+#---------------------------------------------------------------------------
+
+import unittest
+
+try:
+    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+    from IPython.frontend import frontendbase 
+    from IPython.kernel.engineservice import EngineService
+except ImportError:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+from IPython.testing.decorators import skip
+
+class FrontEndCallbackChecker(AsyncFrontEndBase):
+    """FrontEndBase subclass for checking callbacks"""
+    def __init__(self, engine=None, history=None):
+        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
+                                                    history=history)
+        self.updateCalled = False
+        self.renderResultCalled = False
+        self.renderErrorCalled = False
+   
+    def update_cell_prompt(self, result, blockID=None):
+        self.updateCalled = True
+        return result
+   
+    def render_result(self, result):
+        self.renderResultCalled = True
+        return result
+    
+    
+    def render_error(self, failure):
+        self.renderErrorCalled = True
+        return failure
+    
+
+
+    
+class TestAsyncFrontendBase(unittest.TestCase):
+    def setUp(self):
+        """Setup the EngineService and FrontEndBase"""
+        
+        self.fb = FrontEndCallbackChecker(engine=EngineService())
+    
+    def test_implements_IFrontEnd(self):
+        assert(frontendbase.IFrontEnd.implementedBy(
+                                    AsyncFrontEndBase))
+    
+    def test_is_complete_returns_False_for_incomplete_block(self):
+        """"""
+        
+        block = """def test(a):"""
+        
+        assert(self.fb.is_complete(block) == False)
+    
+    def test_is_complete_returns_True_for_complete_block(self):
+        """"""
+        
+        block = """def test(a): pass"""
+            
+        assert(self.fb.is_complete(block))
+        
+        block = """a=3"""
+        
+        assert(self.fb.is_complete(block))
+    
+    def test_blockID_added_to_result(self):
+        block = """3+3"""
+        
+        d = self.fb.execute(block, blockID='TEST_ID')
+        
+        d.addCallback(self.checkBlockID, expected='TEST_ID')
+    
+    def test_blockID_added_to_failure(self):
+        block = "raise Exception()"
+        
+        d = self.fb.execute(block,blockID='TEST_ID')
+        d.addErrback(self.checkFailureID, expected='TEST_ID')
+    
+    def checkBlockID(self, result, expected=""):
+        assert(result['blockID'] == expected)
+    
+    
+    def checkFailureID(self, failure, expected=""):
+        assert(failure.blockID == expected)
+    
+    
+    def test_callbacks_added_to_execute(self):
+        """test that
+        update_cell_prompt
+        render_result
+        
+        are added to execute request
+        """
+        
+        d = self.fb.execute("10+10")
+        d.addCallback(self.checkCallbacks)
+    
+    def checkCallbacks(self, result):
+        assert(self.fb.updateCalled)
+        assert(self.fb.renderResultCalled)
+    
+    @skip("This test fails and lead to an unhandled error in a Deferred.")
+    def test_error_callback_added_to_execute(self):
+        """test that render_error called on execution error"""
+        
+        d = self.fb.execute("raise Exception()")
+        d.addCallback(self.checkRenderError)
+    
+    def checkRenderError(self, result):
+        assert(self.fb.renderErrorCalled)
+    
+    def test_history_returns_expected_block(self):
+        """Make sure history browsing doesn't fail"""
+        
+        blocks = ["a=1","a=2","a=3"]
+        for b in blocks:
+            d = self.fb.execute(b)
+        
+        # d is now the deferred for the last executed block
+        d.addCallback(self.historyTests, blocks)
+        
+    
+    def historyTests(self, result, blocks):
+        """historyTests"""
+        
+        assert(len(blocks) >= 3)
+        assert(self.fb.get_history_previous("") == blocks[-2])
+        assert(self.fb.get_history_previous("") == blocks[-3])
+        assert(self.fb.get_history_next() == blocks[-2])
+    
+    
+    def test_history_returns_none_at_startup(self):
+        """test_history_returns_none_at_startup"""
+        
+        assert(self.fb.get_history_previous("")==None)
+        assert(self.fb.get_history_next()==None)
+    
+    

IPython/kernel/core/tests/test_interpreter.py

@@ -0,0 +1,26 b''
+# encoding: utf-8
+
+"""This file contains unittests for the interpreter.py module."""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team                           
+#                                                                             
+#  Distributed under the terms of the BSD License.  The full license is in    
+#  the file COPYING, distributed as part of this software.                    
+#-----------------------------------------------------------------------------
+                                                                              
+#-----------------------------------------------------------------------------
+# Imports                                                                     
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.core.interpreter import Interpreter
+
+def test_unicode():
+    """ Test unicode handling with the interpreter.
+    """
+    i = Interpreter()
+    i.execute_python(u'print "ù"')
+    i.execute_python('print "ù"')
+

IPython/testing/iptest.py

@@ -0,0 +1,53 b''
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""IPython Test Suite Runner.
+"""
+
+import sys
+import warnings
+
+from nose.core import TestProgram
+import nose.plugins.builtin
+
+from IPython.testing.plugin.ipdoctest import IPythonDoctest
+
+def main():
+    """Run the IPython test suite.
+    """
+
+    warnings.filterwarnings('ignore', 
+        'This will be removed soon.  Use IPython.testing.util instead')
+
+    
+    # construct list of plugins, omitting the existing doctest plugin
+    plugins = [IPythonDoctest()]
+    for p in nose.plugins.builtin.plugins:
+        plug = p()
+        if plug.name == 'doctest':
+            continue
+
+        #print 'adding plugin:',plug.name  # dbg
+        plugins.append(plug)
+
+    argv = sys.argv + ['--doctest-tests','--doctest-extension=txt',
+                       '--detailed-errors',
+                       
+                       # We add --exe because of setuptools' imbecility (it
+                       # blindly does chmod +x on ALL files).  Nose does the
+                       # right thing and it tries to avoid executables,
+                       # setuptools unfortunately forces our hand here.  This
+                       # has been discussed on the distutils list and the
+                       # setuptools devs refuse to fix this problem!
+                       '--exe',
+                       ]
+
+    has_ip = False
+    for arg in sys.argv:
+        if 'IPython' in arg:
+            has_ip = True
+            break
+
+    if not has_ip:
+        argv.append('IPython')
+
+    TestProgram(argv=argv,plugins=plugins)

IPython/testing/tests/test_decorators.py

@@ -0,0 +1,161 b''
+"""Tests for the decorators we've created for IPython.
+"""
+
+# Module imports
+# Std lib
+import inspect
+import sys
+
+# Third party
+import nose.tools as nt
+
+# Our own
+from IPython.testing import decorators as dec
+
+
+#-----------------------------------------------------------------------------
+# Utilities
+
+# Note: copied from OInspect, kept here so the testing stuff doesn't create
+# circular dependencies and is easier to reuse.
+def getargspec(obj):
+    """Get the names and default values of a function's arguments.
+
+    A tuple of four things is returned: (args, varargs, varkw, defaults).
+    'args' is a list of the argument names (it may contain nested lists).
+    'varargs' and 'varkw' are the names of the * and ** arguments or None.
+    'defaults' is an n-tuple of the default values of the last n arguments.
+
+    Modified version of inspect.getargspec from the Python Standard
+    Library."""
+
+    if inspect.isfunction(obj):
+        func_obj = obj
+    elif inspect.ismethod(obj):
+        func_obj = obj.im_func
+    else:
+        raise TypeError, 'arg is not a Python function'
+    args, varargs, varkw = inspect.getargs(func_obj.func_code)
+    return args, varargs, varkw, func_obj.func_defaults
+
+#-----------------------------------------------------------------------------
+# Testing functions
+
[e-mail]
+def test_deliberately_broken():
+    """A deliberately broken test - we want to skip this one."""
+    1/0
+
[e-mail]('foo')
+def test_deliberately_broken2():
+    """Another deliberately broken test - we want to skip this one."""
+    1/0
+
+
+# Verify that we can correctly skip the doctest for a function at will, but
+# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
+def doctest_bad(x,y=1,**k):
+    """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    print 'x:',x
+    print 'y:',y
+    print 'k:',k
+
+
+def call_doctest_bad():
+    """Check that we can still call the decorated functions.
+    
+    >>> doctest_bad(3,y=4)
+    x: 3
+    y: 4
+    k: {}
+    """
+    pass
+
+
+def test_skip_dt_decorator():
+    """Doctest-skipping decorator should preserve the docstring.
+    """
+    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
+    check = """A function whose doctest we need to skip.
+
+    >>> 1+1
+    3
+    """
+    # Fetch the docstring from doctest_bad after decoration.
+    val = doctest_bad.__doc__
+    
+    assert check==val,"doctest_bad docstrings don't match"
+
+# Doctest skipping should work for class methods too
+class foo(object):
+    """Foo
+
+    Example:
+
+    >>> 1+1
+    2
+    """
+
+    @dec.skip_doctest
+    def __init__(self,x):
+        """Make a foo.
+
+        Example:
+
+        >>> f = foo(3)
+        junk
+        """
+        print 'Making a foo.'
+        self.x = x
+        
+    @dec.skip_doctest
+    def bar(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        >>> f.bar(0)
+        boom!
+        >>> 1/0
+        bam!
+        """
+        return 1/y
+
+    def baz(self,y):
+        """Example:
+
+        >>> f = foo(3)
+        Making a foo.
+        >>> f.baz(3)
+        True
+        """
+        return self.x==y
+
+
+
+def test_skip_dt_decorator2():
+    """Doctest-skipping decorator should preserve function signature.
+    """
+    # Hardcoded correct answer
+    dtargs = (['x', 'y'], None, 'k', (1,))
+    # Introspect out the value
+    dtargsr = getargspec(doctest_bad)
+    assert dtargsr==dtargs, \
+           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
+
+
[e-mail]_linux
+def test_linux():
+    nt.assert_not_equals(sys.platform,'linux2',"This test can't run under linux")
+
[e-mail]_win32
+def test_win32():
+    nt.assert_not_equals(sys.platform,'win32',"This test can't run under windows")
+
[e-mail]_osx
+def test_osx():
+    nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")

IPython/tests/test_genutils.py

@@ -0,0 +1,32 b''
+# encoding: utf-8
+
+"""Tests for genutils.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython import genutils
+
+
+def test_get_home_dir():
+    """Make sure we can get the home directory."""
+    home_dir = genutils.get_home_dir()
+
+def test_get_ipython_dir():
+    """Make sure we can get the ipython directory."""
+    ipdir = genutils.get_ipython_dir()
+
+def test_get_security_dir():
+    """Make sure we can get the ipython/security directory."""
+    sdir = genutils.get_security_dir()
+    
  No newline at end of file

IPython/tests/test_magic.py

@@ -0,0 +1,21 b''
+""" Tests for various magic functions
+
+Needs to be run by nose (to make ipython session available)
+
+"""
+def test_rehashx():
+    # clear up everything
+    _ip.IP.alias_table.clear()
+    del _ip.db['syscmdlist']
+    
+    _ip.magic('rehashx')
+    # Practically ALL ipython development systems will have more than 10 aliases
+
+    assert len(_ip.IP.alias_table) > 10
+    for key, val in _ip.IP.alias_table.items():
+        # we must strip dots from alias names
+        assert '.' not in key
+
+    # rehashx must fill up syscmdlist
+    scoms = _ip.db['syscmdlist']
+    assert len(scoms) > 10

docs/examples/core/ipython-sh.desktop

@@ -0,0 +1,22 b''
+# If you want ipython to appear in a linux app launcher ("start menu"), install this by doing:
+# sudo desktop-file-install ipython-sh.desktop
+
+[Desktop Entry]
+Comment=Perform shell-like tasks in interactive ipython session
+Exec=ipython -p sh
+GenericName[en_US]=IPython shell mode
+GenericName=IPython shell mode
+Icon=gnome-netstatus-idle
+MimeType=
+Name[en_US]=ipython-sh
+Name=ipython-sh
+Path=
+Categories=Development;Utility;
+StartupNotify=false
+Terminal=true
+TerminalOptions=
+Type=Application
+X-DBUS-ServiceName=
+X-DBUS-StartupType=none
+X-KDE-SubstituteUID=false
+X-KDE-Username=

docs/examples/core/ipython.desktop

@@ -0,0 +1,22 b''
+# If you want ipython to appear in a linux app launcher ("start menu"), install this by doing:
+# sudo desktop-file-install ipython.desktop
+
+[Desktop Entry]
+Comment=Enhanced interactive Python shell
+Exec=ipython
+GenericName[en_US]=IPython
+GenericName=IPython
+Icon=gnome-netstatus-idle
+MimeType=
+Name[en_US]=ipython
+Name=ipython
+Path=
+Categories=Development;Utility;
+StartupNotify=false
+Terminal=true
+TerminalOptions=
+Type=Application
+X-DBUS-ServiceName=
+X-DBUS-StartupType=none
+X-KDE-SubstituteUID=false
+X-KDE-Username=

docs/source/attic/parallel_task_old.txt

@@ -0,0 +1,240 b''
+.. _paralleltask:
+
+==========================
+The IPython task interface
+==========================
+
+.. contents::
+
+The ``Task`` interface to the controller presents the engines as a fault tolerant, dynamic load-balanced system or workers. Unlike the ``MultiEngine`` interface, in the ``Task`` interface, the user have no direct access to individual engines. In some ways, this interface is simpler, but in other ways it is more powerful. Best of all the user can use both of these interfaces at the same time to take advantage or both of their strengths.  When the user can break up the user's work into segments that do not depend on previous execution, the ``Task`` interface is ideal.  But it also has more power and flexibility, allowing the user to guide the distribution of jobs, without having to assign Tasks to engines explicitly.
+
+Starting the IPython controller and engines
+===========================================
+
+To follow along with this tutorial, the user will need to start the IPython
+controller and four IPython engines. The simplest way of doing this is to
+use the ``ipcluster`` command::
+
+	$ ipcluster -n 4
+	
+For more detailed information about starting the controller and engines, see our :ref:`introduction <ip1par>` to using IPython for parallel computing.
+
+The magic here is that this single controller and set of engines is running both the MultiEngine and ``Task`` interfaces simultaneously.
+
+QuickStart Task Farming
+=======================
+
+First, a quick example of how to start running the most basic Tasks.
+The first step is to import the IPython ``client`` module and then create a ``TaskClient`` instance::
+    
+    In [1]: from IPython.kernel import client
+    
+    In [2]: tc = client.TaskClient()
+
+Then the user wrap the commands the user want to run in Tasks::
+
+    In [3]: tasklist = []
+    In [4]: for n in range(1000):
+        ...     tasklist.append(client.Task("a = %i"%n, pull="a"))
+
+The first argument of the ``Task`` constructor is a string, the command to be executed.  The most important optional keyword argument is ``pull``, which can be a string or list of strings, and it specifies the variable names to be saved as results of the ``Task``.
+
+Next, the user need to submit the Tasks to the ``TaskController`` with the ``TaskClient``::
+
+    In [5]: taskids = [ tc.run(t) for t in tasklist ]
+
+This will give the user a list of the TaskIDs used by the controller to keep track of the Tasks and their results.  Now at some point the user are going to want to get those results back.  The ``barrier`` method allows the user to wait for the Tasks to finish running::
+
+    In [6]: tc.barrier(taskids)
+
+This command will block until all the Tasks in ``taskids`` have finished.  Now, the user probably want to look at the user's results::
+
+    In [7]: task_results = [ tc.get_task_result(taskid) for taskid in taskids ]
+
+Now the user have a list of ``TaskResult`` objects, which have the actual result as a dictionary, but also keep track of some useful metadata about the ``Task``::
+
+    In [8]: tr = ``Task``_results[73]
+    
+    In [9]: tr
+    Out[9]: ``TaskResult``[ID:73]:{'a':73}
+    
+    In [10]: tr.engineid
+    Out[10]: 1
+    
+    In [11]: tr.submitted, tr.completed, tr.duration
+    Out[11]: ("2008/03/08 03:41:42", "2008/03/08 03:41:44", 2.12345)
+
+The actual results are stored in a dictionary, ``tr.results``, and a namespace object ``tr.ns`` which accesses the result keys by attribute::
+
+    In [12]: tr.results['a']
+    Out[12]: 73
+    
+    In [13]: tr.ns.a
+    Out[13]: 73
+
+That should cover the basics of running simple Tasks.  There are several more powerful things the user can do with Tasks covered later.  The most useful probably being using a ``MutiEngineClient`` interface to initialize all the engines with the import dependencies necessary to run the user's Tasks.
+
+There are many options for running and managing Tasks. The best way to learn further about the ``Task`` interface is to study the examples in ``docs/examples``. If the user do so and learn a lots about this interface, we encourage the user to expand this documentation about the ``Task`` system.
+
+Overview of the Task System
+===========================
+
+The user's view of the ``Task`` system has three basic objects:  The ``TaskClient``, the ``Task``, and the ``TaskResult``.  The names of these three objects well indicate their role.
+
+The ``TaskClient`` is the user's ``Task`` farming connection to the IPython cluster.  Unlike the ``MultiEngineClient``, the ``TaskControler`` handles all the scheduling and distribution of work, so the ``TaskClient`` has no notion of engines, it just submits Tasks and requests their results.  The Tasks are described as ``Task`` objects, and their results are wrapped in ``TaskResult`` objects.  Thus, there are very few necessary methods for the user to manage.
+
+Inside the task system is a Scheduler object, which assigns tasks to workers.  The default scheduler is a simple FIFO queue.  Subclassing the Scheduler should be easy, just implementing your own priority system.
+
+The TaskClient
+==============
+
+The ``TaskClient`` is the object the user use to connect to the ``Controller`` that is managing the user's Tasks.  It is the analog of the ``MultiEngineClient`` for the standard IPython multiplexing interface.  As with all client interfaces, the first step is to import the IPython Client Module::
+
+    In [1]: from IPython.kernel import client
+
+Just as with the ``MultiEngineClient``, the user create the ``TaskClient`` with a tuple, containing the ip-address and port of the ``Controller``.  the ``client`` module conveniently has the default address of the ``Task`` interface of the controller.  Creating a default ``TaskClient`` object would be done with this::
+
+    In [2]: tc = client.TaskClient(client.default_task_address)
+
+or, if the user want to specify a non default location of the ``Controller``, the user can specify explicitly::
+
+    In [3]: tc = client.TaskClient(("192.168.1.1", 10113))
+
+As discussed earlier, the ``TaskClient`` only has a few basic methods.
+
+ * ``tc.run(task)``
+    ``run`` is the method by which the user submits Tasks.  It takes exactly one argument, a ``Task`` object.  All the advanced control of ``Task`` behavior is handled by properties of the ``Task`` object, rather than the submission command, so they will be discussed later in the `Task`_ section.  ``run`` returns an integer, the ``Task``ID by which the ``Task`` and its results can be tracked and retrieved::
+    
+        In [4]: ``Task``ID = tc.run(``Task``)
+    
+ * ``tc.get_task_result(taskid, block=``False``)``
+    ``get_task_result`` is the method by which results are retrieved.  It takes a single integer argument, the ``Task``ID`` of the result the user wish to retrieve.  ``get_task_result`` also takes a keyword argument ``block``.  ``block`` specifies whether the user actually want to wait for the result.  If ``block`` is false, as it is by default, ``get_task_result`` will return immediately.  If the ``Task`` has completed, it will return the ``TaskResult`` object for that ``Task``.  But if the ``Task`` has not completed, it will return ``None``.  If the user specify ``block=``True``, then ``get_task_result`` will wait for the ``Task`` to complete, and always return the ``TaskResult`` for the requested ``Task``.
+ * ``tc.barrier(taskid(s))``
+    ``barrier`` is a synchronization method.  It takes exactly one argument, a ``Task``ID or list of taskIDs.  ``barrier`` will block until all the specified Tasks have completed.  In practice, a barrier is often called between the ``Task`` submission section of the code and the result gathering section::
+        
+        In [5]: taskIDs = [ tc.run(``Task``) for ``Task`` in myTasks ]
+        
+        In [6]: tc.get_task_result(taskIDs[-1]) is None
+        Out[6]: ``True``
+        
+        In [7]: tc.barrier(``Task``ID)
+        
+        In [8]: results = [ tc.get_task_result(tid) for tid in taskIDs ]
+        
+ * ``tc.queue_status(verbose=``False``)``
+    ``queue_status`` is a method for querying the state of the ``TaskControler``.  ``queue_status`` returns a dict of the form::
+        
+        {'scheduled': Tasks that have been submitted but yet run
+         'pending'  : Tasks that are currently running
+         'succeeded': Tasks that have completed successfully
+         'failed'   : Tasks that have finished with a failure
+        }
+    
+    if @verbose is not specified (or is ``False``), then the values of the dict are integers - the number of Tasks in each state.  if @verbose is ``True``, then each element in the dict is a list of the taskIDs in that state::
+    
+        In [8]: tc.queue_status()
+        Out[8]: {'scheduled': 4,
+                'pending'  : 2,
+                'succeeded': 5,
+                'failed'   : 1
+                }
+        
+        In [9]: tc.queue_status(verbose=True)
+        Out[9]: {'scheduled': [8,9,10,11],
+                'pending'  : [6,7],
+                'succeeded': [0,1,2,4,5],
+                'failed'   : [3]
+                }
+        
+ * ``tc.abort(taskid)``
+    ``abort`` allows the user to abort Tasks that have already been submitted.  ``abort`` will always return immediately.  If the ``Task`` has completed, ``abort`` will raise an ``IndexError ``Task`` Already Completed``.  An obvious case for ``abort`` would be where the user submits a long-running ``Task`` with a number of retries (see ``Task``_ section for how to specify retries) in an interactive session, but realizes there has been a typo.  The user can then abort the ``Task``, preventing certain failures from cluttering up the queue.  It can also be used for parallel search-type problems, where only one ``Task`` will give the solution, so once the user find the solution, the user would want to abort all remaining Tasks to prevent wasted work.
+ * ``tc.spin()``
+    ``spin`` simply triggers the scheduler in the ``TaskControler``.  Under most normal circumstances, this will do nothing.  The primary known usage case involves the ``Task`` dependency (see `Dependencies`_).  The dependency is a function of an Engine's ``properties``, but changing the ``properties`` via the ``MutliEngineClient`` does not trigger a reschedule event.  The main example case for this requires the following event sequence:
+     * ``engine`` is available, ``Task`` is submitted, but ``engine`` does not have ``Task``'s dependencies.
+     * ``engine`` gets necessary dependencies while no new Tasks are submitted or completed.
+     * now ``engine`` can run ``Task``, but a ``Task`` event is required for the ``TaskControler`` to try scheduling ``Task`` again.
+     
+    ``spin`` is just an empty ping method to ensure that the Controller has scheduled all available Tasks, and should not be needed under most normal circumstances.
+
+That covers the ``TaskClient``, a simple interface to the cluster.  With this, the user can submit jobs (and abort if necessary), request their results, synchronize on arbitrary subsets of jobs.
+
+.. _task: The Task Object
+
+The Task Object
+===============
+
+The ``Task`` is the basic object for describing a job.  It can be used in a very simple manner, where the user just specifies a command string to be executed as the ``Task``.  The usage of this first argument is exactly the same as the ``execute`` method of the ``MultiEngine`` (in fact, ``execute`` is called to run the code)::
+
+    In [1]: t = client.Task("a = str(id)")
+    
+This ``Task`` would run, and store the string representation of the ``id`` element in ``a`` in each worker's namespace, but it is fairly useless because the user does not know anything about the state of the ``worker`` on which it ran at the time of retrieving results.  It is important that each ``Task`` not expect the state of the ``worker`` to persist after the ``Task`` is completed.  
+There are many different situations for using ``Task`` Farming, and the ``Task`` object has many attributes for use in customizing the ``Task`` behavior.  All of a ``Task``'s attributes may be specified in the constructor, through keyword arguments, or after ``Task`` construction through attribute assignment.
+
+Data Attributes
+***************
+It is likely that the user may want to move data around before or after executing the ``Task``.  We provide methods of sending data to initialize the worker's namespace, and specifying what data to bring back as the ``Task``'s results.
+
+ * pull = []
+    The obvious case is as above, where ``t`` would execute and store the result of ``myfunc`` in ``a``, it is likely that the user would want to bring ``a`` back to their namespace.  This is done through the ``pull`` attribute.  ``pull`` can be a string or list of strings, and it specifies the names of variables to be retrieved.  The ``TaskResult`` object retrieved by ``get_task_result`` will have a dictionary of keys and values, and the ``Task``'s ``pull`` attribute determines what goes into it::
+        
+        In [2]: t = client.Task("a = str(id)", pull = "a")
+        
+        In [3]: t = client.Task("a = str(id)", pull = ["a", "id"])
+        
+ * push = {}
+    A user might also want to initialize some data into the namespace before the code part of the ``Task`` is run.  Enter ``push``.  ``push`` is a dictionary of key/value pairs to be loaded from the user's namespace into the worker's immediately before execution::
+
+        In [4]: t = client.Task("a = f(submitted)", push=dict(submitted=time.time()), pull="a")
+
+push and pull result directly in calling an ``engine``'s ``push`` and ``pull`` methods before and after ``Task`` execution respectively, and thus their api is the same.
+
+Namespace Cleaning
+******************
+When a user is running a large number of Tasks, it is likely that the namespace of the worker's could become cluttered.  Some Tasks might be sensitive to clutter, while others might be known to cause namespace pollution.  For these reasons, Tasks have two boolean attributes for cleaning up the namespace.
+
+ * ``clear_after``
+    if clear_after is specified ``True``, the worker on which the ``Task`` was run will be reset (via ``engine.reset``) upon completion of the ``Task``.  This can be useful for both Tasks that produce clutter or Tasks whose intermediate data one might wish to be kept private::
+    
+        In [5]: t = client.Task("a = range(1e10)", pull = "a",clear_after=True)
+        
+    
+ * ``clear_before``
+    as one might guess, clear_before is identical to ``clear_after``, but it takes place before the ``Task`` is run.  This ensures that the ``Task`` runs on a fresh worker::
+
+        In [6]: t = client.Task("a = globals()", pull = "a",clear_before=True)
+
+Of course, a user can both at the same time, ensuring that all workers are clear except when they are currently running a job.  Both of these default to ``False``.
+
+Fault Tolerance
+***************
+It is possible that Tasks might fail, and there are a variety of reasons this could happen.  One might be that the worker it was running on disconnected, and there was nothing wrong with the ``Task`` itself.  With the fault tolerance attributes of the ``Task``, the user can specify how many times to resubmit the ``Task``, and what to do if it never succeeds.
+
+ * ``retries``
+    ``retries`` is an integer, specifying the number of times a ``Task`` is to be retried.  It defaults to zero.  It is often a good idea for this number to be 1 or 2, to protect the ``Task`` from disconnecting engines, but not a large number.  If a ``Task`` is failing 100 times, there is probably something wrong with the ``Task``.  The canonical bad example:
+    
+        In [7]: t = client.Task("os.kill(os.getpid(), 9)", retries=99)
+        
+    This would actually take down 100 workers.
+    
+ * ``recovery_task``
+    ``recovery_task`` is another ``Task`` object, to be run in the event of the original ``Task`` still failing after running out of retries.  Since ``recovery_task`` is another ``Task`` object, it can have its own ``recovery_task``.  The chain of Tasks is limitless, except loops are not allowed (that would be bad!).
+
+Dependencies
+************
+Dependencies are the most powerful part of the ``Task`` farming system, because it allows the user to do some classification of the workers, and guide the ``Task`` distribution without meddling with the controller directly.  It makes use of two objects - the ``Task``'s ``depend`` attribute, and the engine's ``properties``.  See the `MultiEngine`_ reference for how to use engine properties.  The engine properties api exists for extending IPython, allowing conditional execution and new controllers that make decisions based on properties of its engines.  Currently the ``Task`` dependency is the only internal use of the properties api.
+
+.. _MultiEngine: ./parallel_multiengine
+
+The ``depend`` attribute of a ``Task`` must be a function of exactly one argument, the worker's properties dictionary, and it should return ``True`` if the ``Task`` should be allowed to run on the worker and ``False`` if not.  The usage in the controller is fault tolerant, so exceptions raised by ``Task.depend`` will be ignored and functionally equivalent to always returning ``False``.  Tasks`` with invalid ``depend`` functions will never be assigned to a worker::
+
+    In [8]: def dep(properties):
+        ...     return properties["RAM"] > 2**32 # have at least 4GB
+    In [9]: t = client.Task("a = bigfunc()", depend=dep)
+    
+It is important to note that assignment of values to the properties dict is done entirely by the user, either locally (in the engine) using the EngineAPI, or remotely, through the ``MultiEngineClient``'s get/set_properties methods.
+
+
+
+
+
+    

docs/source/development/config_blueprint.txt

@@ -0,0 +1,33 b''
+=========================================
+Notes on the IPython configuration system
+=========================================
+
+This document has some random notes on the configuration system.
+
+To start, an IPython process needs:
+
+* Configuration files
+* Command line options
+* Additional files (FURL files, extra scripts, etc.)
+
+It feeds these things into the core logic of the process, and as output,
+produces:
+
+* Log files
+* Security files
+
+There are a number of things that complicate this:
+
+* A process may need to be started on a different host that doesn't have
+  any of the config files or additional files.  Those files need to be
+  moved over and put in a staging area.  The process then needs to be told
+  about them.
+* The location of the output files should somehow be set by config files or
+  command line options.
+* Our config files are very hierarchical, but command line options are flat,
+  making it difficult to relate command line options to config files.
+* Some processes (like ipcluster and the daemons) have to manage the input and
+  output files for multiple different subprocesses, each possibly on a
+  different host.  Ahhhh!
+* Our configurations are not singletons.  A given user will likely have
+  many different configurations for different clusters.

docs/source/install/install.txt

@@ -0,0 +1,217 b''
+Overview
+========
+
+This document describes the steps required to install IPython.  IPython is organized into a number of subpackages, each of which has its own dependencies.  All of the subpackages come with IPython, so you don't need to download and install them separately.  However, to use a given subpackage, you will need to install all of its dependencies.  
+
+
+Please let us know if you have problems installing IPython or any of its
+dependencies. IPython requires Python version 2.4 or greater.  We have not tested
+IPython with the upcoming 2.6 or 3.0 versions.
+
+.. warning::
+
+	IPython will not work with Python 2.3 or below.
+
+Some of the installation approaches use the :mod:`setuptools` package and its :command:`easy_install` command line program.  In many scenarios, this provides the most simple method of installing IPython and its dependencies.  It is not required though.  More information about :mod:`setuptools` can be found on its website.
+
+More general information about installing Python packages can be found in Python's documentation at http://www.python.org/doc/.
+
+Quickstart
+==========
+
+If you have :mod:`setuptools` installed and you are on OS X or Linux (not Windows), the following will download and install IPython *and* the main optional dependencies::
+
+    $ easy_install ipython[kernel,security,test]
+
+This will get Twisted, zope.interface and Foolscap, which are needed for IPython's parallel computing features as well as the nose package, which will enable you to run IPython's test suite.  To run IPython's test suite, use the :command:`iptest` command::
+
+    $ iptest
+
+Read on for more specific details and instructions for Windows.
+
+Installing IPython itself
+=========================
+
+Given a properly built Python, the basic interactive IPython shell will work with no external dependencies.  However, some Python distributions (particularly on Windows and OS X), don't come with a working :mod:`readline` module.  The IPython shell will work without :mod:`readline`, but will lack many features that users depend on, such as tab completion and command line editing.  See below for details of how to make sure you have a working :mod:`readline`.
+
+Installation using easy_install
+-------------------------------
+
+If you have :mod:`setuptools` installed, the easiest way of getting IPython is to simple use :command:`easy_install`::
+
+	$ easy_install ipython
+
+That's it.
+
+Installation from source
+------------------------
+
+If you don't want to use :command:`easy_install`, or don't have it installed, just grab the latest stable build of IPython from `here <http://ipython.scipy.org/dist/>`_.  Then do the following::
+
+	$ tar -xzf ipython.tar.gz
+	$ cd ipython
+	$ python setup.py install
+
+If you are installing to a location (like ``/usr/local``) that requires higher permissions, you may need to run the last command with :command:`sudo`.
+
+Windows
+-------
+
+There are a few caveats for Windows users.  The main issue is that a basic ``python setup.py install`` approach won't create ``.bat`` file or Start Menu shortcuts, which most users want.  To get an installation with these, there are two choices:
+
+1.  Install using :command:`easy_install`.
+
+2.  Install using our binary ``.exe`` Windows installer, which can be found at `here <http://ipython.scipy.org/dist/>`_
+
+3.  Install from source, but using :mod:`setuptools` (``python setupegg.py install``).
+
+Installing the development version
+----------------------------------
+
+It is also possible to install the development version of IPython from our `Bazaar <http://bazaar-vcs.org/>`_ source code
+repository.  To do this you will need to have Bazaar installed on your system.  Then just do::
+
+	$ bzr branch lp:ipython
+	$ cd ipython
+	$ python setup.py install
+
+Again, this last step on Windows won't create ``.bat`` files or Start Menu shortcuts, so you will have to use one of the other approaches listed above.
+
+Some users want to be able to follow the development branch as it changes.  If you have :mod:`setuptools` installed, this is easy. Simply replace the last step by::
+
+	$ python setupegg.py develop
+
+This creates links in the right places and installs the command line script to the appropriate places.  Then, if you want to update your IPython at any time, just do::
+
+	$ bzr pull
+
+Basic optional dependencies
+===========================
+
+There are a number of basic optional dependencies that most users will want to get.  These are:
+
+* readline (for command line editing, tab completion, etc.)
+* nose (to run the IPython test suite)
+* pexpect (to use things like irunner)
+
+If you are comfortable installing these things yourself, have at it, otherwise read on for more details.
+
+readline
+--------
+
+In principle, all Python distributions should come with a working :mod:`readline` module.  But, reality is not quite that simple.  There are two common situations where you won't have a working :mod:`readline` module:
+
+* If you are using the built-in Python on Mac OS X.
+
+* If you are running Windows, which doesn't have a :mod:`readline` module.
+
+On OS X, the built-in Python doesn't not have :mod:`readline` because of license issues.  Starting with OS X 10.5 (Leopard), Apple's built-in Python has a BSD-licensed not-quite-compatible readline replacement. As of IPython 0.9, many of the issues related to the differences between readline and libedit have been resolved.  For many users, libedit may be sufficient.
+
+Most users on OS X will want to get the full :mod:`readline` module.  To get a working :mod:`readline` module, just do (with :mod:`setuptools` installed)::
+
+	$ easy_install readline
+
+.. note:
+
+    Other Python distributions on OS X (such as fink, MacPorts and the 
+    official python.org binaries) already have readline installed so 
+    you don't have to do this step.
+
+If needed, the readline egg can be build and installed from source (see the wiki page at http://ipython.scipy.org/moin/InstallationOSXLeopard).
+
+On Windows, you will need the PyReadline module. PyReadline is a separate,
+Windows only implementation of readline that uses native Windows calls through
+:mod:`ctypes`. The easiest way of installing PyReadline is you use the binary
+installer available `here <http://ipython.scipy.org/dist/>`_. The
+:mod:`ctypes` module, which comes with Python 2.5 and greater, is required by
+PyReadline. It is available for Python 2.4 at
+http://python.net/crew/theller/ctypes.
+
+nose
+----
+
+To run the IPython test suite you will need the :mod:`nose` package.  Nose provides a great way of sniffing out and running all of the IPython tests.  The simplest way of getting nose, is to use :command:`easy_install`::
+
+	$ easy_install nose
+
+Another way of getting this is to do::
+
+    $ easy_install ipython[test]
+
+For more installation options, see the `nose website <http://somethingaboutorange.com/mrl/projects/nose/>`_.  Once you have nose installed, you can run IPython's test suite using the iptest command::
+
+	$ iptest
+
+
+pexpect
+-------
+
+The `pexpect <http://www.noah.org/wiki/Pexpect>`_ package is used in IPython's :command:`irunner` script.  On Unix platforms (including OS X), just do::
+
+	$ easy_install pexpect
+
+Windows users are out of luck as pexpect does not run there.
+
+Dependencies for IPython.kernel (parallel computing)
+====================================================
+
+The IPython kernel provides a nice architecture for parallel computing.  The main focus of this architecture is on interactive parallel computing.  These features require a number of additional packages:
+
+* zope.interface (yep, we use interfaces)
+* Twisted (asynchronous networking framework)
+* Foolscap (a nice, secure network protocol)
+* pyOpenSSL (security for network connections)
+
+On a Unix style platform (including OS X), if you want to use :mod:`setuptools`, you can just do::
+
+    $ easy_install ipython[kernel]    # the first three
+    $ easy_install ipython[security]  # pyOpenSSL
+
+zope.interface and Twisted
+--------------------------
+
+Twisted [Twisted]_ and zope.interface [ZopeInterface]_ are used for networking related things. On Unix
+style platforms (including OS X), the simplest way of getting the these is to
+use :command:`easy_install`::
+
+	$ easy_install zope.interface
+	$ easy_install Twisted
+
+Of course, you can also download the source tarballs from the `Twisted website <twistedmatrix.org>`_ and the `zope.interface page at PyPI <http://pypi.python.org/pypi/zope.interface>`_ and do the usual ``python setup.py install`` if you prefer.
+
+Windows is a bit different.  For zope.interface and Twisted, simply get the latest binary ``.exe`` installer from the Twisted website.  This installer includes both zope.interface and Twisted and should just work.
+
+Foolscap
+--------
+
+Foolscap [Foolscap]_ uses Twisted to provide a very nice secure RPC protocol that we use to implement our parallel computing features.
+
+On all platforms a simple::
+
+    $ easy_install foolscap
+
+should work.  You can also download the source tarballs from the `Foolscap website <http://foolscap.lothar.com/trac>`_ and do ``python setup.py install`` if you prefer.
+
+pyOpenSSL
+---------
+
+IPython requires an older version of pyOpenSSL [pyOpenSSL]_ (0.6 rather than the current 0.7).  There are a couple of options for getting this:
+
+1.  Most Linux distributions have packages for pyOpenSSL.
+2.  The built-in Python 2.5 on OS X 10.5 already has it installed.
+3.  There are source tarballs on the pyOpenSSL website.  On Unix-like
+    platforms, these can be built using ``python seutp.py install``.
+4.  There is also a binary ``.exe`` Windows installer on the `pyOpenSSL website <http://pyopenssl.sourceforge.net/>`_.
+
+Dependencies for IPython.frontend (the IPython GUI)
+===================================================
+
+wxPython
+--------
+
+Starting with IPython 0.9, IPython has a new IPython.frontend package that has a nice wxPython based IPython GUI.  As you would expect, this GUI requires wxPython.  Most Linux distributions have wxPython packages available and the built-in Python on OS X comes with wxPython preinstalled.  For Windows, a binary installer is available on the `wxPython website <http://www.wxpython.org/>`_.
+
+.. [Twisted] Twisted matrix.  http://twistedmatrix.org
+.. [ZopeInterface] http://pypi.python.org/pypi/zope.interface
+.. [Foolscap] Foolscap network protocol.  http://foolscap.lothar.com/trac
+.. [pyOpenSSL] pyOpenSSL.  http://pyopenssl.sourceforge.net
  No newline at end of file

IPython/ColorANSI.py

@@ -26,6 +26,7 b' def make_color_table(in_class):'
     Helper function for building the *TermColors classes."""
     
     color_templates = (
+        # Dark colors
         ("Black"       , "0;30"),
         ("Red"         , "0;31"),
         ("Green"       , "0;32"),
@@ -34,6 +35,7 b' def make_color_table(in_class):'
         ("Purple"      , "0;35"),
         ("Cyan"        , "0;36"),
         ("LightGray"   , "0;37"),
+        # Light colors
         ("DarkGray"    , "1;30"),
         ("LightRed"    , "1;31"),
         ("LightGreen"  , "1;32"),
@@ -41,7 +43,17 b' def make_color_table(in_class):'
         ("LightBlue"   , "1;34"),
         ("LightPurple" , "1;35"),
         ("LightCyan"   , "1;36"),
-        ("White"       , "1;37"),  )
+        ("White"       , "1;37"),  
+        # Blinking colors.  Probably should not be used in anything serious.
+        ("BlinkBlack"  , "5;30"),
+        ("BlinkRed"    , "5;31"),
+        ("BlinkGreen"  , "5;32"),
+        ("BlinkYellow" , "5;33"),
+        ("BlinkBlue"   , "5;34"),
+        ("BlinkPurple" , "5;35"),
+        ("BlinkCyan"   , "5;36"),
+        ("BlinkLightGray", "5;37"),
+        )
 
     for name,value in color_templates:
         setattr(in_class,name,in_class._base % value)

IPython/Extensions/ipy_completers.py

@@ -335,6 +335,12 b' def cd_completer(self, event):'
     if not found:
         if os.path.isdir(relpath):
             return [relpath]
+        # if no completions so far, try bookmarks
+        bks = self.db.get('bookmarks',{}).keys()
+        bkmatches = [s for s in bks if s.startswith(event.symbol)]
+        if bkmatches:
+            return bkmatches
+        
         raise IPython.ipapi.TryNext
 
 

IPython/Extensions/ipy_editors.py

@@ -28,7 +28,8 b' def install_editor(run_template, wait = False):'
             line = 0
         cmd = itplns(run_template, locals())
         print ">",cmd
-        os.system(cmd)
+        if os.system(cmd) != 0:
+            raise IPython.ipapi.TryNext()
         if wait:
             raw_input("Press Enter when done editing:")
 
@@ -64,7 +65,10 b' def idle(exe = None):'
         p = os.path.dirname(idlelib.__file__)
         exe = p + '/idle.py'
     install_editor(exe + ' "$file"')
-        
+
+def mate(exe = 'mate'):
+    """ TextMate, the missing editor"""
+    install_editor(exe + ' -w -l $line "$file"')
 
 # these are untested, report any problems
 

IPython/Extensions/ipy_profile_sh.py

@@ -8,7 +8,7 b' compatibility)'
 """
 
 from IPython import ipapi
-import os,textwrap
+import os,re,textwrap
 
 # The import below effectively obsoletes your old-style ipythonrc[.ini],
 # so consider yourself warned!
@@ -50,9 +50,15 b' def main():'
     ip.ex('import os')
     ip.ex("def up(): os.chdir('..')")
     ip.user_ns['LA'] = LastArgFinder() 
-    # Nice prompt
     
-    o.prompt_in1= r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
+    # You can assign to _prompt_title variable 
+    # to provide some extra information for prompt
+    # (e.g. the current mode, host/username...)
+
+    ip.user_ns['_prompt_title'] = ''
+    
+    # Nice prompt
+    o.prompt_in1= r'\C_Green${_prompt_title}\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Green|\#> '
     o.prompt_in2= r'\C_Green|\C_LightGreen\D\C_Green> '
     o.prompt_out= '<\#> '
     
@@ -98,9 +104,15 b' def main():'
     for cmd in syscmds:
         # print "sys",cmd #dbg
         noext, ext = os.path.splitext(cmd)
-        key = mapper(noext)
+        if ext.lower() == '.exe':
+            cmd = noext
+        
+        key = mapper(cmd)
         if key not in ip.IP.alias_table:
-            ip.defalias(key, cmd)
+            # Dots will be removed from alias names, since ipython
+            # assumes names with dots to be python code
+            
+            ip.defalias(key.replace('.',''), cmd)
 
     # mglob combines 'find', recursion, exclusion... '%mglob?' to learn more
     ip.load("IPython.external.mglob")    
@@ -117,7 +129,7 b' def main():'
             # and the next best thing to real 'ls -F'
             ip.defalias('d','dir /w /og /on')
     
-    ip.set_hook('input_prefilter', dotslash_prefilter_f)
+    ip.set_hook('input_prefilter', slash_prefilter_f)
     extend_shell_behavior(ip)
 
 class LastArgFinder:
@@ -139,13 +151,13 b' class LastArgFinder:'
             return parts[-1]
         return ""
 
-def dotslash_prefilter_f(self,line):
-    """ ./foo now runs foo as system command 
+def slash_prefilter_f(self,line):
+    """ ./foo, ~/foo and /bin/foo now run foo as system command
     
-    Removes the need for doing !./foo
+    Removes the need for doing !./foo, !~/foo or !/bin/foo
     """
     import IPython.genutils
-    if line.startswith("./"):
+    if re.match('(?:[.~]|/[a-zA-Z_0-9]+)/', line):
         return "_ip.system(" + IPython.genutils.make_quoted_expr(line)+")"
     raise ipapi.TryNext
 

IPython/Magic.py

@@ -422,7 +422,11 b' python-profiler package from non-free.""")'
                 else:
                     fndoc = 'No documentation'
             else:
-                fndoc = fn.__doc__.rstrip()
+                if fn.__doc__:
+                    fndoc = fn.__doc__.rstrip()       
+                else:
+                    fndoc = 'No documentation'
+                
                 
             if mode == 'rest':
                 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
@@ -2328,7 +2332,17 b' Currently the magic system has the following functions:\\n"""'
         # do actual editing here
         print 'Editing...',
         sys.stdout.flush()
-        self.shell.hooks.editor(filename,lineno)
+        try:
+            self.shell.hooks.editor(filename,lineno)
+        except IPython.ipapi.TryNext:
+            warn('Could not open editor')
+            return
+        
+        # XXX TODO: should this be generalized for all string vars?
+        # For now, this is special-cased to blocks created by cpaste
+        if args.strip() == 'pasted_block':
+            self.shell.user_ns['pasted_block'] = file_read(filename)
+        
         if opts.has_key('x'):  # -x prevents actual execution
             print
         else:
@@ -2338,6 +2352,8 b' Currently the magic system has the following functions:\\n"""'
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
+        
+                                                     
         if use_temp:
             try:
                 return open(filename).read()
@@ -2647,8 +2663,10 b' Defaulting color scheme to \'NoColor\'"""'
                         if isexec(ff) and ff not in self.shell.no_alias:
                             # each entry in the alias table must be (N,name),
                             # where N is the number of positional arguments of the
-                            # alias.
-                            alias_table[ff] = (0,ff)
+                            # alias.                            
+                            # Dots will be removed from alias names, since ipython
+                            # assumes names with dots to be python code
+                            alias_table[ff.replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             else:
                 for pdir in path:
@@ -2658,7 +2676,7 b' Defaulting color scheme to \'NoColor\'"""'
                         if isexec(ff) and base.lower() not in self.shell.no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
-                            alias_table[base.lower()] = (0,ff)
+                            alias_table[base.lower().replace('.','')] = (0,ff)
                             syscmdlist.append(ff)
             # Make sure the alias table doesn't contain keywords or builtins
             self.shell.alias_table_validate()
@@ -3210,14 +3228,24 b' Defaulting color scheme to \'NoColor\'"""'
         This assigns the pasted block to variable 'foo' as string, without 
         dedenting or executing it (preceding >>> and + is still stripped)
         
+        '%cpaste -r' re-executes the block previously entered by cpaste.
+        
         Do not be alarmed by garbled output on Windows (it's a readline bug). 
         Just press enter and type -- (and press enter again) and the block 
         will be what was just pasted.
         
         IPython statements (magics, shell escapes) are not supported (yet).
         """
-        opts,args = self.parse_options(parameter_s,'s:',mode='string')
+        opts,args = self.parse_options(parameter_s,'rs:',mode='string')
         par = args.strip()
+        if opts.has_key('r'):
+            b = self.user_ns.get('pasted_block', None)
+            if b is None:
+                raise UsageError('No previous pasted block available')
+            print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
+            exec b in self.user_ns
+            return
+        
         sentinel = opts.get('s','--')
 
         # Regular expressions that declare text we strip from the input:
@@ -3245,8 +3273,8 b' Defaulting color scheme to \'NoColor\'"""'
         #print "block:\n",block
         if not par:
             b = textwrap.dedent(block)
-            exec b in self.user_ns            
             self.user_ns['pasted_block'] = b
+            exec b in self.user_ns
         else:
             self.user_ns[par] = SList(block.splitlines())
             print "Block assigned to '%s'" % par

IPython/Release.py

@@ -1,7 +1,5 b''
 # -*- coding: utf-8 -*-
-"""Release data for the IPython project.
-
-$Id: Release.py 3002 2008-02-01 07:17:00Z fperez $"""
+"""Release data for the IPython project."""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
@@ -23,9 +21,9 b" name = 'ipython'"
 # bdist_deb does not accept underscores (a Debian convention).
 
 development = False    # change this to False to do a release
-version_base = '0.9.beta'
+version_base = '0.9.1'
 branch = 'ipython'
-revision = '1099'
+revision = '1143'
 
 if development:
     if branch == 'ipython':
@@ -36,45 +34,69 b' else:'
     version = version_base
 
 
-description = "Tools for interactive development in Python."
+description = "An interactive computing environment for Python"
 
 long_description = \
 """
-IPython provides a replacement for the interactive Python interpreter with
-extra functionality.
+The goal of IPython is to create a comprehensive environment for
+interactive and exploratory computing.  To support this goal, IPython
+has two main components:
+
+* An enhanced interactive Python shell.
+
+* An architecture for interactive parallel computing.
+
+The enhanced interactive Python shell has the following main features:
+
+* Comprehensive object introspection.
+
+* Input history, persistent across sessions.
 
-Main features:
+* Caching of output results during a session with automatically generated
+  references.
 
- * Comprehensive object introspection.
+* Readline based name completion.
 
- * Input history, persistent across sessions.
+* Extensible system of 'magic' commands for controlling the environment and
+  performing many tasks related either to IPython or the operating system.
 
- * Caching of output results during a session with automatically generated
-   references.
+* Configuration system with easy switching between different setups (simpler
+  than changing $PYTHONSTARTUP environment variables every time).
 
- * Readline based name completion.
+* Session logging and reloading.
 
- * Extensible system of 'magic' commands for controlling the environment and
-   performing many tasks related either to IPython or the operating system.
+* Extensible syntax processing for special purpose situations.
 
- * Configuration system with easy switching between different setups (simpler
-   than changing $PYTHONSTARTUP environment variables every time).
+* Access to the system shell with user-extensible alias system.
 
- * Session logging and reloading.
+* Easily embeddable in other Python programs and wxPython GUIs.
 
- * Extensible syntax processing for special purpose situations.
+* Integrated access to the pdb debugger and the Python profiler. 
 
- * Access to the system shell with user-extensible alias system.
+The parallel computing architecture has the following main features:
 
- * Easily embeddable in other Python programs.
+* Quickly parallelize Python code from an interactive Python/IPython session.
 
- * Integrated access to the pdb debugger and the Python profiler. 
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
 
- The latest development version is always available at the IPython subversion
- repository_.
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.
 
-.. _repository: http://ipython.scipy.org/svn/ipython/ipython/trunk#egg=ipython-dev
- """
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Share live parallel jobs with other users securely.  
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling in parallel code.
+
+The latest development version is always available from IPython's `Launchpad 
+site <http://launchpad.net/ipython>`_.
+"""
 
 license = 'BSD'
 

IPython/Shell.py

@@ -40,11 +40,12 b' except ImportError:'
 # IPython imports
 import IPython
 from IPython import ultraTB, ipapi
+from IPython.Magic import Magic
 from IPython.genutils import Term,warn,error,flag_calls, ask_yes_no
 from IPython.iplib import InteractiveShell
 from IPython.ipmaker import make_IPython
-from IPython.Magic import Magic
 from IPython.ipstruct import Struct
+from IPython.testing import decorators as testdec
 
 # Globals
 # global flag to pass around information about Ctrl-C without exceptions
@@ -384,7 +385,7 b' class MTInteractiveShell(InteractiveShell):'
 
         Modified version of code.py's runsource(), to handle threading issues.
         See the original for full docstring details."""
-        
+                
         global KBINT
         
         # If Ctrl-C was typed, we reset the flag and return right away
@@ -414,7 +415,7 b' class MTInteractiveShell(InteractiveShell):'
         if (self.worker_ident is None
             or self.worker_ident == thread.get_ident() ):
             InteractiveShell.runcode(self,code)
-            return
+            return False
 
         # Case 3
         # Store code in queue, so the execution thread can handle it.
@@ -607,7 +608,8 b' class MatplotlibShellBase:'
         # if a backend switch was performed, reverse it now
         if self.mpl_use._called:
             self.matplotlib.rcParams['backend'] = self.mpl_backend
-        
+
+    @testdec.skip_doctest
     def magic_run(self,parameter_s=''):
         Magic.magic_run(self,parameter_s,runner=self.mplot_exec)
 
@@ -774,6 +776,17 b' class IPShellGTK(IPThread):'
                  debug=1,shell_class=MTInteractiveShell):
 
         import gtk
+        # Check for set_interactive, coming up in new pygtk.
+        # Disable it so that this code works, but notify 
+        # the user that he has a better option as well.
+        # XXX TODO better support when set_interactive is released
+        try:
+            gtk.set_interactive(False)
+            print "Your PyGtk has set_interactive(), so you can use the"
+            print "more stable single-threaded Gtk mode."
+            print "See https://bugs.launchpad.net/ipython/+bug/270856"
+        except AttributeError:
+            pass
         
         self.gtk = gtk
         self.gtk_mainloop = hijack_gtk()

IPython/UserConfig/ipythonrc-physics

@@ -38,6 +38,8 b' ececute rad = pi/180.'
 execute print '*** q is an alias for PhysicalQuantityInteractive'
 execute print '*** g = 9.8 m/s^2 has been defined'
 execute print '*** rad = pi/180  has been defined'
+execute import ipy_constants as C
+execute print '*** C is the physical constants module'
 
 # Files to execute
 execfile 

IPython/completer.py

@@ -314,7 +314,10 b' class IPCompleter(Completer):'
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
-        protectables = ' '
+        if sys.platform == 'win32':
+            protectables = ' '
+        else:
+            protectables = ' ()'
 
         if text.startswith('!'):
             text = text[1:]

IPython/config/api.py

@@ -16,13 +16,11 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import os
-from IPython.config.cutils import get_home_dir, get_ipython_dir
+from os.path import join as pjoin
+
+from IPython.genutils import get_home_dir, get_ipython_dir
 from IPython.external.configobj import ConfigObj
 
-# Traitlets config imports
-from IPython.config import traitlets
-from IPython.config.config import *
-from traitlets import *
 
 class ConfigObjManager(object):
     
@@ -53,7 +51,7 b' class ConfigObjManager(object):'
 
     def write_default_config_file(self):
         ipdir = get_ipython_dir()
-        fname = ipdir + '/' + self.filename
+        fname = pjoin(ipdir, self.filename)
         if not os.path.isfile(fname):
             print "Writing the configuration file to: " + fname
             self.write_config_obj_to_file(fname)
@@ -87,11 +85,11 b' class ConfigObjManager(object):'
 
         # In ipythondir if it is set
         if ipythondir is not None:
-            trythis = ipythondir + '/' + filename
+            trythis = pjoin(ipythondir, filename)
             if os.path.isfile(trythis):
                 return trythis        
 
-        trythis = get_ipython_dir() + '/' + filename
+        trythis = pjoin(get_ipython_dir(), filename)
         if os.path.isfile(trythis):
             return trythis
 

IPython/config/cutils.py

@@ -22,71 +22,6 b' import sys'
 # Normal code begins
 #---------------------------------------------------------------------------
 
-class HomeDirError(Exception):
-    pass
-
-def get_home_dir():
-    """Return the closest possible equivalent to a 'home' directory.
-
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
-    Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
-
-    isdir = os.path.isdir
-    env = os.environ
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
-        else:
-            raise HomeDirError,'support for your operating system not implemented.'
-
-def get_ipython_dir():
-    ipdir_def = '.ipython'
-    home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir,ipdir_def)))
-    return ipdir
-
 def import_item(key):
     """
     Import and return bar given the string foo.bar.

IPython/demo.py

@@ -180,7 +180,7 b' def re_mark(mark):'
 
 class Demo(object):
 
-    re_stop     = re_mark('-?\s?stop\s?-?')
+    re_stop     = re_mark('-*\s?stop\s?-*')
     re_silent   = re_mark('silent')
     re_auto     = re_mark('auto')
     re_auto_all = re_mark('auto_all')

IPython/frontend/_process/killableprocess.py

@@ -50,7 +50,6 b' import subprocess'
 from subprocess import PIPE
 import sys
 import os
-import time
 import types
 
 try:
@@ -69,8 +68,16 b' except ImportError:'
 
 mswindows = (sys.platform == "win32")
 
+skip = False
+
 if mswindows:
-    import winprocess
+    import platform
+    if platform.uname()[3] == '' or platform.uname()[3] > '6.0.6000':
+        # Killable process does not work under vista when starting for
+        # something else than cmd.
+        skip = True
+    else:
+        import winprocess
 else:
     import signal
 
@@ -78,7 +85,11 b' if not mswindows:'
     def DoNothing(*args):
         pass
 
-class Popen(subprocess.Popen):
+
+if skip:
+ Popen = subprocess.Popen
+else:
+ class Popen(subprocess.Popen):
     if not mswindows:
         # Override __init__ to set a preexec_fn
         def __init__(self, *args, **kwargs):

IPython/frontend/_process/pipedprocess.py

@@ -50,7 +50,7 b' class PipedProcess(Thread):'
         """
         env = os.environ
         env['TERM'] = 'xterm'
-        process = Popen((self.command_string + ' 2>&1', ), shell=True,
+        process = Popen(self.command_string + ' 2>&1', shell=True,
                                 env=env,
                                 universal_newlines=True,
                                 stdout=PIPE, stdin=PIPE, )

IPython/frontend/asyncfrontendbase.py

@@ -14,30 +14,14 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-import uuid
+from IPython.external import guid
 
-try:
-    from zope.interface import Interface, Attribute, implements, classProvides
-except ImportError, e:
-    e.message = """%s
-________________________________________________________________________________
-zope.interface is required to run asynchronous frontends.""" % e.message
-    e.args = (e.message, ) + e.args[1:]
 
-from frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
-
-from IPython.kernel.engineservice import IEngineCore
+from zope.interface import Interface, Attribute, implements, classProvides
+from twisted.python.failure import Failure
+from IPython.frontend.frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
 from IPython.kernel.core.history import FrontEndHistory
-
-try:
-    from twisted.python.failure import Failure
-except ImportError, e:
-    e.message = """%s
-________________________________________________________________________________
-twisted is required to run asynchronous frontends.""" % e.message
-    e.args = (e.message, ) + e.args[1:]
-
-
+from IPython.kernel.engineservice import IEngineCore
 
 
 class AsyncFrontEndBase(FrontEndBase):
@@ -75,7 +59,7 b' class AsyncFrontEndBase(FrontEndBase):'
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate() 
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)

IPython/frontend/cocoa/cocoa_frontend.py

@@ -26,7 +26,7 b' __docformat__ = "restructuredtext en"'
 
 import sys
 import objc
-import uuid
+from IPython.external import guid
 
 from Foundation import NSObject, NSMutableArray, NSMutableDictionary,\
                         NSLog, NSNotificationCenter, NSMakeRange,\
@@ -361,7 +361,7 b' class IPythonCocoaController(NSObject, AsyncFrontEndBase):'
     
     def next_block_ID(self):
         
-        return uuid.uuid4()
+        return guid.generate() 
     
     def new_cell_block(self):
         """A new CellBlock at the end of self.textView.textStorage()"""

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -14,28 +14,31 b' __docformat__ = "restructuredtext en"'
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
-from IPython.kernel.core.interpreter import Interpreter                     
-import IPython.kernel.engineservice as es
-from IPython.testing.util import DeferredTestCase
-from twisted.internet.defer import succeed
-from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
-                                                    
-from Foundation import NSMakeRect
-from AppKit import NSTextView, NSScrollView                                                
+
+try:
+    from IPython.kernel.core.interpreter import Interpreter
+    import IPython.kernel.engineservice as es
+    from IPython.testing.util import DeferredTestCase
+    from twisted.internet.defer import succeed
+    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController                                                    
+    from Foundation import NSMakeRect
+    from AppKit import NSTextView, NSScrollView                                                
+except ImportError:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted, Foolscap and PyObjC")
 
 class TestIPythonCocoaControler(DeferredTestCase):
     """Tests for IPythonCocoaController"""
-    
+   
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
-    
-    
+   
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
-    
+   
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
@@ -47,45 +50,45 b' class TestIPythonCocoaControler(DeferredTestCase):'
         self.assertDeferredEquals(
             self.controller.execute(code).addCallback(removeNumberAndID), 
             expected)
-    
+   
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
-        
+       
         self.controller.execute(code).addCallback(testControllerUserNS)
-    
-    
+   
+   
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
-    
+   
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
-        
+       
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
-        
+       
         self.controller.execute(code).addCallback(testCompletes)
-    
-    
+   
+   
     def testCurrentIndent(self):
         """test that current_indent_string returns current indent or None.
         Uses _indent_for_block for direct unit testing.
         """
-        
+       
         self.controller.tabUsesSpaces = True
         self.assert_(self.controller._indent_for_block("""a=3""") == None)
         self.assert_(self.controller._indent_for_block("") == None)
         block = """def test():\n    a=3"""
         self.assert_(self.controller._indent_for_block(block) == \
                     ' ' * self.controller.tabSpaces)
-        
+       
         block = """if(True):\n%sif(False):\n%spass""" % \
                     (' '*self.controller.tabSpaces,
                      2*' '*self.controller.tabSpaces)
         self.assert_(self.controller._indent_for_block(block) == \
                     2*(' '*self.controller.tabSpaces))
-    
+   

IPython/frontend/frontendbase.py

@@ -21,14 +21,16 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 import string
-import uuid
-import _ast
+import codeop
+from IPython.external import guid
 
-from zopeinterface import Interface, Attribute, implements, classProvides
 
+from IPython.frontend.zopeinterface import (
+    Interface, 
+    Attribute, 
+)
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.core.util import Bunch
-from IPython.kernel.engineservice import IEngineCore
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
@@ -131,11 +133,7 b' class IFrontEnd(Interface):'
         
         pass
     
-    def compile_ast(block):
-        """Compiles block to an _ast.AST"""
-        
-        pass
-    
+   
     def get_history_previous(current_block):
         """Returns the block previous in  the history. Saves currentBlock if
         the history_cursor is currently at the end of the input history"""
@@ -217,28 +215,14 b' class FrontEndBase(object):'
         """
         
         try:
-            ast = self.compile_ast(block)
+            is_complete = codeop.compile_command(block.rstrip() + '\n\n',
+                            "<string>", "exec") 
         except:
             return False
         
         lines = block.split('\n')
-        return (len(lines)==1 or str(lines[-1])=='')
-    
-    
-    def compile_ast(self, block):
-        """Compile block to an AST
-        
-        Parameters:
-            block : str
-        
-        Result:
-            AST
-        
-        Throws:
-            Exception if block cannot be compiled
-        """
-        
-        return compile(block, "<string>", "exec", _ast.PyCF_ONLY_AST)
+        return ((is_complete is not None) 
+                    and (len(lines)==1 or str(lines[-1])==''))
     
     
     def execute(self, block, blockID=None):
@@ -258,7 +242,7 b' class FrontEndBase(object):'
             raise Exception("Block is not compilable")
         
         if(blockID == None):
-            blockID = uuid.uuid4() #random UUID
+            blockID = guid.generate()
         
         try:
             result = self.shell.execute(block)

IPython/frontend/linefrontendbase.py

@@ -20,6 +20,8 b' import re'
 
 import IPython
 import sys
+import codeop
+import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
@@ -76,6 +78,11 b' class LineFrontEndBase(FrontEndBase):'
 
         if banner is not None:
             self.banner = banner
+
+    def start(self):
+        """ Put the frontend in a state where it is ready for user
+            interaction.
+        """
         if self.banner is not None:
             self.write(self.banner, refresh=False)
 
@@ -141,9 +148,18 b' class LineFrontEndBase(FrontEndBase):'
                         and not re.findall(r"\n[\t ]*\n[\t ]*$", string)):
             return False
         else:
-            # Add line returns here, to make sure that the statement is
-            # complete.
-            return FrontEndBase.is_complete(self, string.rstrip() + '\n\n')
+            self.capture_output()
+            try:
+                # Add line returns here, to make sure that the statement is
+                # complete.
+                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                            "<string>", "exec")
+                self.release_output()
+            except Exception, e:
+                # XXX: Hack: return True so that the
+                # code gets executed and the error captured.
+                is_complete = True
+            return is_complete
 
 
     def write(self, string, refresh=True):
@@ -166,22 +182,35 b' class LineFrontEndBase(FrontEndBase):'
             raw_string = python_string
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
+
+        ## try:
+        ##     self.history.input_cache[-1] = raw_string.rstrip()
+        ##     result = self.shell.execute(python_string)
+        ##     self.last_result = result
+        ##     self.render_result(result)
+        ## except:
+        ##     self.show_traceback()
+        ## finally:
+        ##     self.after_execute()
+
         try:
-            self.history.input_cache[-1] = raw_string.rstrip()
-            result = self.shell.execute(python_string)
-            self.last_result = result
-            self.render_result(result)
-        except:
-            self.show_traceback()
+            try:
+                self.history.input_cache[-1] = raw_string.rstrip()
+                result = self.shell.execute(python_string)
+                self.last_result = result
+                self.render_result(result)
+            except:
+                self.show_traceback()
         finally:
             self.after_execute()
 
+
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, string):
-        """ Priflter the input to turn it in valid python.
+        """ Prefilter the input to turn it in valid python.
         """
         string = string.replace('\r\n', '\n')
         string = string.replace('\t', 4*' ')
@@ -210,9 +239,12 b' class LineFrontEndBase(FrontEndBase):'
         line = self.input_buffer
         new_line, completions = self.complete(line)
         if len(completions)>1:
-            self.write_completion(completions)
-        self.input_buffer = new_line
+            self.write_completion(completions, new_line=new_line)
+        elif not line == new_line:
+            self.input_buffer = new_line
         if self.debug:
+            print >>sys.__stdout__, 'line', line
+            print >>sys.__stdout__, 'new_line', new_line
             print >>sys.__stdout__, completions 
 
 
@@ -222,10 +254,15 b' class LineFrontEndBase(FrontEndBase):'
         return 80
 
 
-    def write_completion(self, possibilities):
+    def write_completion(self, possibilities, new_line=None):
         """ Write the list of possible completions.
+
+            new_line is the completed input line that should be displayed
+            after the completion are writen. If None, the input_buffer
+            before the completion is used.
         """
-        current_buffer = self.input_buffer
+        if new_line is None:
+            new_line = self.input_buffer
         
         self.write('\n')
         max_len = len(max(possibilities, key=len)) + 1
@@ -246,7 +283,7 b' class LineFrontEndBase(FrontEndBase):'
         self.write(''.join(buf))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
-        self.input_buffer = current_buffer
+        self.input_buffer = new_line
 
 
     def new_prompt(self, prompt):
@@ -275,6 +312,8 b' class LineFrontEndBase(FrontEndBase):'
         else:
             self.input_buffer += self._get_indent_string(
                                                 current_buffer[:-1])
+            if len(current_buffer.split('\n')) == 2:
+                self.input_buffer += '\t\t'
             if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
                 self.input_buffer += '\t'
 

IPython/frontend/prefilterfrontend.py

@@ -24,6 +24,7 b' __docformat__ = "restructuredtext en"'
 import sys
 
 from linefrontendbase import LineFrontEndBase, common_prefix
+from frontendbase import FrontEndBase
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
@@ -34,6 +35,7 b' from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap'
 from IPython.genutils import Term
 import pydoc
 import os
+import sys
 
 
 def mk_system_call(system_call_function, command):
@@ -57,6 +59,8 b' class PrefilterFrontEnd(LineFrontEndBase):'
     to execute the statements and the ipython0 used for code
     completion...
     """
+
+    debug = False
     
     def __init__(self, ipython0=None, *args, **kwargs):
         """ Parameters:
@@ -65,12 +69,24 b' class PrefilterFrontEnd(LineFrontEndBase):'
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
         """
+        LineFrontEndBase.__init__(self, *args, **kwargs)
+        self.shell.output_trap = RedirectorOutputTrap(
+                            out_callback=self.write,
+                            err_callback=self.write,
+                                            )
+        self.shell.traceback_trap = SyncTracebackTrap(
+                        formatters=self.shell.traceback_trap.formatters,
+                            )
+
+        # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[])
+            ipython0 = make_IPython(argv=[], 
+                                    user_ns=self.shell.user_ns,
+                                    user_global_ns=self.shell.user_global_ns)
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
@@ -86,24 +102,13 b' class PrefilterFrontEnd(LineFrontEndBase):'
                                                             'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
+
+
         if not 'banner' in kwargs and self.banner is None:
-            kwargs['banner'] = self.ipython0.BANNER + """
+            self.banner = self.ipython0.BANNER + """
 This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
 
-        LineFrontEndBase.__init__(self, *args, **kwargs)
-        # XXX: Hack: mix the two namespaces
-        self.shell.user_ns.update(self.ipython0.user_ns)
-        self.ipython0.user_ns = self.shell.user_ns 
-        self.shell.user_global_ns.update(self.ipython0.user_global_ns)
-        self.ipython0.user_global_ns = self.shell.user_global_ns
-
-        self.shell.output_trap = RedirectorOutputTrap(
-                            out_callback=self.write,
-                            err_callback=self.write,
-                                            )
-        self.shell.traceback_trap = SyncTracebackTrap(
-                        formatters=self.shell.traceback_trap.formatters,
-                            )
+        self.start()
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
@@ -113,7 +118,7 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         """ Use ipython0 to capture the last traceback and display it.
         """
         self.capture_output()
-        self.ipython0.showtraceback()
+        self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
 
@@ -164,6 +169,8 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
 
 
     def complete(self, line):
+        # FIXME: This should be factored out in the linefrontendbase
+        # method.
         word = line.split('\n')[-1].split(' ')[-1]
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
@@ -189,17 +196,33 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         # capture it.
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
+        
+        ## try:
+        ##     for line in input_string.split('\n'):
+        ##         filtered_lines.append(
+        ##                 self.ipython0.prefilter(line, False).rstrip())
+        ## except:
+        ##     # XXX: probably not the right thing to do.
+        ##     self.ipython0.showsyntaxerror()
+        ##     self.after_execute()
+        ## finally:
+        ##     self.release_output()
+
+
         try:
-            for line in input_string.split('\n'):
-                filtered_lines.append(
-                        self.ipython0.prefilter(line, False).rstrip())
-        except:
-            # XXX: probably not the right thing to do.
-            self.ipython0.showsyntaxerror()
-            self.after_execute()
+            try:
+                for line in input_string.split('\n'):
+                    filtered_lines.append(
+                            self.ipython0.prefilter(line, False).rstrip())
+            except:
+                # XXX: probably not the right thing to do.
+                self.ipython0.showsyntaxerror()
+                self.after_execute()
         finally:
             self.release_output()
 
+
+
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string

IPython/frontend/tests/test_frontendbase.py

@@ -1,152 +1,32 b''
 # encoding: utf-8
-
-"""This file contains unittests for the frontendbase module."""
+"""
+Test the basic functionality of frontendbase. 
+"""
 
 __docformat__ = "restructuredtext en"
 
-#---------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                         
-#                                                                           
-#  Distributed under the terms of the BSD License.  The full license is in  
-#  the file COPYING, distributed as part of this software.                  
-#---------------------------------------------------------------------------
-                                                                            
-#---------------------------------------------------------------------------
-# Imports                                                                   
-#---------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
 
-import unittest
-from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-from IPython.frontend import frontendbase 
-from IPython.kernel.engineservice import EngineService
+from IPython.frontend.frontendbase import FrontEndBase
 
-class FrontEndCallbackChecker(AsyncFrontEndBase):
-    """FrontEndBase subclass for checking callbacks"""
-    def __init__(self, engine=None, history=None):
-        super(FrontEndCallbackChecker, self).__init__(engine=engine, 
-                                                    history=history)
-        self.updateCalled = False
-        self.renderResultCalled = False
-        self.renderErrorCalled = False
-    
-    def update_cell_prompt(self, result, blockID=None):
-        self.updateCalled = True
-        return result
-    
-    def render_result(self, result):
-        self.renderResultCalled = True
-        return result
-    
-    
-    def render_error(self, failure):
-        self.renderErrorCalled = True
-        return failure
-    
+def test_iscomplete():
+    """ Check that is_complete works. 
+    """
+    f = FrontEndBase()
+    assert f.is_complete('(a + a)')
+    assert not f.is_complete('(a + a')
+    assert f.is_complete('1')
+    assert not f.is_complete('1 + ')
+    assert not f.is_complete('1 + \n\n')
+    assert f.is_complete('if True:\n  print 1\n')
+    assert not f.is_complete('if True:\n  print 1')
+    assert f.is_complete('def f():\n  print 1\n')
 
+if __name__ == '__main__':
+    test_iscomplete()
 
-    
-class TestAsyncFrontendBase(unittest.TestCase):
-    def setUp(self):
-        """Setup the EngineService and FrontEndBase"""
-        
-        self.fb = FrontEndCallbackChecker(engine=EngineService())
-    
-    
-    def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(
-                                    AsyncFrontEndBase))
-    
-    
-    def test_is_complete_returns_False_for_incomplete_block(self):
-        """"""
-        
-        block = """def test(a):"""
-        
-        assert(self.fb.is_complete(block) == False)
-    
-    def test_is_complete_returns_True_for_complete_block(self):
-        """"""
-        
-        block = """def test(a): pass"""
-            
-        assert(self.fb.is_complete(block))
-        
-        block = """a=3"""
-        
-        assert(self.fb.is_complete(block))
-    
-    
-    def test_blockID_added_to_result(self):
-        block = """3+3"""
-        
-        d = self.fb.execute(block, blockID='TEST_ID')
-        
-        d.addCallback(self.checkBlockID, expected='TEST_ID')
-    
-    def test_blockID_added_to_failure(self):
-        block = "raise Exception()"
-        
-        d = self.fb.execute(block,blockID='TEST_ID')
-        d.addErrback(self.checkFailureID, expected='TEST_ID')
-    
-    def checkBlockID(self, result, expected=""):
-        assert(result['blockID'] == expected)
-    
-    
-    def checkFailureID(self, failure, expected=""):
-        assert(failure.blockID == expected)
-    
-    
-    def test_callbacks_added_to_execute(self):
-        """test that
-        update_cell_prompt
-        render_result
-        
-        are added to execute request
-        """
-        
-        d = self.fb.execute("10+10")
-        d.addCallback(self.checkCallbacks)
-    
-    
-    def checkCallbacks(self, result):
-        assert(self.fb.updateCalled)
-        assert(self.fb.renderResultCalled)
-    
-    
-    def test_error_callback_added_to_execute(self):
-        """test that render_error called on execution error"""
-        
-        d = self.fb.execute("raise Exception()")
-        d.addCallback(self.checkRenderError)
-    
-    def checkRenderError(self, result):
-        assert(self.fb.renderErrorCalled)
-    
-    def test_history_returns_expected_block(self):
-        """Make sure history browsing doesn't fail"""
-        
-        blocks = ["a=1","a=2","a=3"]
-        for b in blocks:
-            d = self.fb.execute(b)
-        
-        # d is now the deferred for the last executed block
-        d.addCallback(self.historyTests, blocks)
-        
-    
-    def historyTests(self, result, blocks):
-        """historyTests"""
-        
-        assert(len(blocks) >= 3)
-        assert(self.fb.get_history_previous("") == blocks[-2])
-        assert(self.fb.get_history_previous("") == blocks[-3])
-        assert(self.fb.get_history_next() == blocks[-2])
-    
-    
-    def test_history_returns_none_at_startup(self):
-        """test_history_returns_none_at_startup"""
-        
-        assert(self.fb.get_history_previous("")==None)
-        assert(self.fb.get_history_next()==None)
-    
-    

IPython/frontend/tests/test_process.py

@@ -17,6 +17,8 b' from time import sleep'
 import sys
 
 from IPython.frontend._process import PipedProcess
+from IPython.testing import decorators as testdec
+
 
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
@@ -25,7 +27,8 b' def test_capture_out():'
     p = PipedProcess('echo 1', out_callback=s.write, )
     p.start()
     p.join()
-    assert s.getvalue() == '1\n'
+    result = s.getvalue().rstrip()
+    assert result == '1'
 
 
 def test_io():
@@ -40,7 +43,8 b' def test_io():'
         sleep(0.1)
     p.process.stdin.write(test_string)
     p.join()
-    assert s.getvalue() == test_string
+    result = s.getvalue()
+    assert result == test_string
 
 
 def test_kill():

IPython/frontend/wx/console_widget.py

@@ -23,6 +23,7 b' import wx'
 import wx.stc  as  stc
 
 from wx.py import editwindow
+import time
 import sys
 LINESEP = '\n'
 if sys.platform == 'win32':
@@ -35,7 +36,7 b' import re'
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
-    _DEFAULT_STYLE = 12
+    _DEFAULT_SIZE = 12
 
 _DEFAULT_STYLE = {
     'stdout'      : 'fore:#0000FF',
@@ -115,12 +116,15 b' class ConsoleWidget(editwindow.EditWindow):'
     # The color of the carret (call _apply_style() after setting)
     carret_color = 'BLACK'
     
+    # Store the last time a refresh was done
+    _last_refresh_time = 0
+
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
     
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
-                        size=wx.DefaultSize, style=0, ):
+                        size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
         self._configure_scintilla()
 
@@ -168,9 +172,14 b' class ConsoleWidget(editwindow.EditWindow):'
                 
         self.GotoPos(self.GetLength())
         if refresh:
-            # Maybe this is faster than wx.Yield()
-            self.ProcessEvent(wx.PaintEvent())
-            #wx.Yield()
+            current_time = time.time()
+            if current_time - self._last_refresh_time > 0.03:
+                if sys.platform == 'win32':
+                    wx.SafeYield()
+                else:
+                    wx.Yield()
+                #    self.ProcessEvent(wx.PaintEvent())
+                self._last_refresh_time = current_time 
 
    
     def new_prompt(self, prompt):
@@ -183,7 +192,6 b' class ConsoleWidget(editwindow.EditWindow):'
         # now we update our cursor giving end of prompt
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
-        wx.Yield()
         self.EnsureCaretVisible()
 
 

IPython/frontend/wx/wx_frontend.py

@@ -128,6 +128,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # while it is being swapped
     _out_buffer_lock = Lock()
 
+    # The different line markers used to higlight the prompts.
     _markers = dict()
 
     #--------------------------------------------------------------------------
@@ -135,12 +136,16 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     #--------------------------------------------------------------------------
  
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
-                 size=wx.DefaultSize, style=wx.CLIP_CHILDREN,
+                 size=wx.DefaultSize,
+                 style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
                  *args, **kwds):
         """ Create Shell instance.
         """
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
+        
+        # Stick in our own raw_input:
+        self.ipython0.raw_input = self.raw_input
 
         # Marker for complete buffer.
         self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
@@ -164,9 +169,11 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         # Inject self in namespace, for debug
         if self.debug:
             self.shell.user_ns['self'] = self
+        # Inject our own raw_input in namespace
+        self.shell.user_ns['raw_input'] = self.raw_input
 
 
-    def raw_input(self, prompt):
+    def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
         self.new_prompt(prompt)
@@ -174,15 +181,13 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if hasattr(self, '_cursor'):
             del self._cursor 
         self.SetCursor(wx.StockCursor(wx.CURSOR_CROSS))
-        self.waiting = True
         self.__old_on_enter = self._on_enter
+        event_loop = wx.EventLoop()
         def my_on_enter():
-            self.waiting = False
+            event_loop.Exit()
         self._on_enter = my_on_enter
-        # XXX: Busy waiting, ugly.
-        while self.waiting:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         self._on_enter = self.__old_on_enter
         self._input_state = 'buffering'
         self._cursor = wx.BusyCursor()
@@ -191,16 +196,18 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
 
     def system_call(self, command_string):
         self._input_state = 'subprocess'
+        event_loop = wx.EventLoop()
+        def _end_system_call():
+            self._input_state = 'buffering'
+            self._running_process = False
+            event_loop.Exit()
+
         self._running_process = PipedProcess(command_string, 
                     out_callback=self.buffered_write,
-                    end_callback = self._end_system_call)
+                    end_callback = _end_system_call)
         self._running_process.start()
-        # XXX: another one of these polling loops to have a blocking
-        # call
-        wx.Yield()
-        while self._running_process:
-            wx.Yield()
-            sleep(0.1)
+        # XXX: Running a separate event_loop. Ugly.
+        event_loop.Run() 
         # Be sure to flush the buffer.
         self._buffer_flush(event=None)
 
@@ -226,8 +233,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             for name in symbol_string.split('.')[1:] + ['__doc__']:
                 symbol = getattr(symbol, name)
             self.AutoCompCancel()
-            wx.Yield()
-            self.CallTipShow(self.GetCurrentPos(), symbol)
+            # Check that the symbol can indeed be converted to a string:
+            symbol += ''
+            wx.CallAfter(self.CallTipShow, self.GetCurrentPos(), symbol)
         except:
             # The retrieve symbol couldn't be converted to a string
             pass
@@ -238,9 +246,9 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             true, open the menu.
         """
         if self.debug:
-            print >>sys.__stdout__, "_popup_completion", 
+            print >>sys.__stdout__, "_popup_completion" 
         line = self.input_buffer
-        if (self.AutoCompActive() and not line[-1] == '.') \
+        if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
             offset=0
@@ -284,19 +292,21 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _COMPLETE_BUFFER_MARKER)
-        # Update the display:
-        wx.Yield()
-        self.GotoPos(self.GetLength())
-        PrefilterFrontEnd.execute(self, python_string, raw_string=raw_string)
+        # Use a callafter to update the display robustly under windows
+        def callback():
+            self.GotoPos(self.GetLength())
+            PrefilterFrontEnd.execute(self, python_string, 
+                                            raw_string=raw_string)
+        wx.CallAfter(callback)
 
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
 
     def capture_output(self):
-        __builtin__.raw_input = self.raw_input
         self.SetLexer(stc.STC_LEX_NULL)
         PrefilterFrontEnd.capture_output(self)
+        __builtin__.raw_input = self.raw_input
         
     
     def release_output(self):
@@ -316,12 +326,24 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     def show_traceback(self):
         start_line = self.GetCurrentLine()
         PrefilterFrontEnd.show_traceback(self)
-        wx.Yield()
+        self.ProcessEvent(wx.PaintEvent())
+        #wx.Yield()
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
     
     #--------------------------------------------------------------------------
+    # FrontEndBase interface 
+    #--------------------------------------------------------------------------
+    
+    def render_error(self, e):
+        start_line = self.GetCurrentLine()
+        self.write('\n' + e + '\n')
+        for i in range(start_line, self.GetCurrentLine()):
+            self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
+
+
+    #--------------------------------------------------------------------------
     # ConsoleWidget interface 
     #--------------------------------------------------------------------------
 
@@ -351,7 +373,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             if self._input_state == 'subprocess':
                 if self.debug:
                     print >>sys.__stderr__, 'Killing running process'
-                self._running_process.process.kill()
+                if hasattr(self._running_process, 'process'):
+                    self._running_process.process.kill()
             elif self._input_state == 'buffering':
                 if self.debug:
                     print >>sys.__stderr__, 'Raising KeyboardInterrupt'
@@ -376,7 +399,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                 char = '\04'
             self._running_process.process.stdin.write(char)
             self._running_process.process.stdin.flush()
-        elif event.KeyCode in (ord('('), 57):
+        elif event.KeyCode in (ord('('), 57, 53):
             # Calltips
             event.Skip()
             self.do_calltip()
@@ -410,8 +433,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                last_line = self.input_buffer.split('\n')[-1]
-                if not re.match(r'^\s*$', last_line):
+                current_line, current_line_number = self.CurLine
+                if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
@@ -427,7 +450,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if event.KeyCode in (59, ord('.')):
             # Intercepting '.'
             event.Skip()
-            self._popup_completion(create=True)
+            wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
 
@@ -456,13 +479,6 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     # Private API
     #--------------------------------------------------------------------------
  
-    def _end_system_call(self):
-        """ Called at the end of a system call.
-        """
-        self._input_state = 'buffering'
-        self._running_process = False
-
-
     def _buffer_flush(self, event):
         """ Called by the timer to flush the write buffer.
             

IPython/frontend/zopeinterface.py

@@ -16,13 +16,6 b' __docformat__ = "restructuredtext en"'
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-import string
-import uuid
-import _ast
-
 try:
     from zope.interface import Interface, Attribute, implements, classProvides
 except ImportError:

IPython/genutils.py

@@ -979,6 +979,38 b' def get_home_dir():'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds either .ipython or _ipython to the end of the path.
+    """
+    if os.name == 'posix':
+         ipdir_def = '.ipython'
+    else:
+         ipdir_def = '_ipython'
+    home_dir = get_home_dir()
+    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
+                                           os.path.join(home_dir,ipdir_def)))
+    return ipdir
+
+def get_security_dir():
+    """Get the IPython security directory.
+    
+    This directory is the default location for all security related files,
+    including SSL/TLS certificates and FURL files.
+    
+    If the directory does not exist, it is created with 0700 permissions.
+    If it exists, permissions are set to 0700.
+    """
+    security_dir = os.path.join(get_ipython_dir(), 'security')
+    if not os.path.isdir(security_dir):
+        os.mkdir(security_dir, 0700)
+    else:
+        os.chmod(security_dir, 0700)
+    return security_dir
+        
 #****************************************************************************
 # strings and text
 

IPython/history.py

@@ -8,6 +8,7 b' import os'
 
 # IPython imports
 from IPython.genutils import Term, ask_yes_no
+import IPython.ipapi
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
@@ -222,6 +223,7 b' class ShadowHist:'
         # cmd => idx mapping
         self.curidx = 0
         self.db = db
+        self.disabled = False
     
     def inc_idx(self):
         idx = self.db.get('shadowhist_idx', 1)
@@ -229,12 +231,19 b' class ShadowHist:'
         return idx
         
     def add(self, ent):
-        old = self.db.hget('shadowhist', ent, _sentinel)
-        if old is not _sentinel:
+        if self.disabled:
             return
-        newidx = self.inc_idx()
-        #print "new",newidx # dbg
-        self.db.hset('shadowhist',ent, newidx)
+        try:
+            old = self.db.hget('shadowhist', ent, _sentinel)
+            if old is not _sentinel:
+                return
+            newidx = self.inc_idx()
+            #print "new",newidx # dbg
+            self.db.hset('shadowhist',ent, newidx)
+        except:
+            IPython.ipapi.get().IP.showtraceback()
+            print "WARNING: disabling shadow history"
+            self.disabled = True
     
     def all(self):
         d = self.db.hdict('shadowhist')

IPython/hooks.py

@@ -25,7 +25,8 b' ip = IPython.ipapi.get()'
 def calljed(self,filename, linenum):
     "My editor hook calls the jed editor directly."
     print "Calling my own editor, jed ..."
-    os.system('jed +%d %s' % (linenum,filename))
+    if os.system('jed +%d %s' % (linenum,filename)) != 0:
+        raise ipapi.TryNext()
 
 ip.set_hook('editor', calljed)
 
@@ -84,7 +85,8 b' def editor(self,filename, linenum=None):'
         editor = '"%s"' % editor
         
     # Call the actual editor
-    os.system('%s %s %s' % (editor,linemark,filename))
+    if os.system('%s %s %s' % (editor,linemark,filename)) != 0:
+        raise ipapi.TryNext()
 
 import tempfile
 def fix_error_editor(self,filename,linenum,column,msg):
@@ -105,7 +107,8 b' def fix_error_editor(self,filename,linenum,column,msg):'
         return
     t = vim_quickfix_file()
     try:
-        os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name)
+        if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name):
+            raise ipapi.TryNext()
     finally:
         t.close()
 

IPython/iplib.py

@@ -1419,8 +1419,12 b' want to merge them back into the new files.""" % locals()'
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
-        self.hooks.fix_error_editor(e.filename,
-            int0(e.lineno),int0(e.offset),e.msg)
+        try:
+            self.hooks.fix_error_editor(e.filename,
+                int0(e.lineno),int0(e.offset),e.msg)
+        except IPython.ipapi.TryNext:
+            warn('Could not open editor')
+            return False
         return True
         
     def edit_syntax_error(self):
@@ -1572,6 +1576,11 b' want to merge them back into the new files.""" % locals()'
             else:                
                 banner = self.BANNER+self.banner2
 
+        # if you run stuff with -c <cmd>, raw hist is not updated
+        # ensure that it's in sync
+        if len(self.input_hist) != len (self.input_hist_raw):
+            self.input_hist_raw = InputList(self.input_hist)
+
         while 1:
             try:
                 self.interact(banner)

IPython/kernel/config/__init__.py

@@ -15,17 +15,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+from os.path import join as pjoin
+
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
-from IPython.config.cutils import get_ipython_dir
+from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
-try:
-    ipython_dir = get_ipython_dir() + '/'
-except:
-    # This will defaults to the cwd
-    ipython_dir = ''
+security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
@@ -33,7 +31,7 b' except:'
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
-    furl_file = ipython_dir + 'ipcontroller-engine.furl'
+    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
@@ -63,16 +61,17 b' controller_config = dict('
 
     logfile = '',    # Empty means log to stdout
     import_statement = '',
+    reuse_furls = False, # If False, old furl files are deleted
 
     engine_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-engine.pem',
+        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = ipython_dir + 'ipcontroller-engine.furl',
+    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
@@ -83,12 +82,12 b' controller_config = dict('
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
@@ -97,7 +96,7 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-client.pem'
+        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
@@ -108,10 +107,10 b' controller_config = dict('
 client_config = dict(
     client_interfaces = dict(
         task = dict(
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )

IPython/kernel/contexts.py

@@ -8,8 +8,6 b' which can also be useful as templates for writing new, application-specific'
 managers.
 """
 
-from __future__ import with_statement
-
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------

IPython/kernel/controllerservice.py

@@ -50,7 +50,7 b' from IPython.kernel.engineservice import \\'
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
@@ -170,7 +170,7 b' class ControllerService(object, service.Service):'
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
-        ipdir = cutils.get_ipython_dir()
+        ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):

IPython/kernel/core/interpreter.py

@@ -680,6 +680,13 b' class Interpreter(object):'
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
 
+        # The compiler module does not like unicode. We need to convert
+        # it encode it:
+        if isinstance(python, unicode):
+            # Use the utf-8-sig BOM so the compiler detects this a UTF-8
+            # encode string.
+            python = '\xef\xbb\xbf' + python.encode('utf-8')
+
         # The compiler module will parse the code into an abstract syntax tree.
         ast = compiler.parse(python)
 

IPython/kernel/core/tests/test_redirectors.py

@@ -13,10 +13,17 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 
+# Stdlib imports
 import os
 from cStringIO import StringIO
 
+# Our own imports
+from IPython.testing import decorators as dec
 
+#-----------------------------------------------------------------------------
+# Test functions
+
[e-mail]_win32
 def test_redirector():
     """ Checks that the redirector can be used to do synchronous capture.
     """
@@ -33,9 +40,12 b' def test_redirector():'
         r.stop()
         raise
     r.stop()
-    assert out.getvalue() == "".join("%ic\n%i\n" %(i, i) for i in range(10))
+    result1 = out.getvalue()
+    result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
+    assert result1 == result2
 
 
[e-mail]_win32
 def test_redirector_output_trap():
     """ This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
@@ -54,8 +64,7 b' def test_redirector_output_trap():'
         trap.unset()
         raise
     trap.unset()
-    assert out.getvalue() == "".join("%ic\n%ip\n%i\n" %(i, i, i) 
-                                                    for i in range(10))
-
+    result1 = out.getvalue()
+    result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
+    assert result1 == result2
     
-

IPython/kernel/engineconnector.py

@@ -18,7 +18,8 b' __docformat__ = "restructuredtext en"'
 import os
 import cPickle as pickle
 
-from twisted.python import log
+from twisted.python import log, failure
+from twisted.internet import defer
 
 from IPython.kernel.fcutil import find_furl
 from IPython.kernel.enginefc import IFCEngine
@@ -62,13 +63,17 b' class EngineConnector(object):'
             self.tub.startService()
         self.engine_service = engine_service
         self.engine_reference = IFCEngine(self.engine_service)
-        self.furl = find_furl(furl_or_file)
+        try:
+            self.furl = find_furl(furl_or_file)
+        except ValueError:
+            return defer.fail(failure.Failure())
+            # return defer.fail(failure.Failure(ValueError('not a valid furl or furl file: %r' % furl_or_file)))
         d = self.tub.getReference(self.furl)
         d.addCallbacks(self._register, self._log_failure)
         return d
     
     def _log_failure(self, reason):
-        log.err('engine registration failed:')
+        log.err('EngineConnector: engine registration failed:')
         log.err(reason)
         return reason
 

IPython/kernel/scripts/ipcluster.py

@@ -1,324 +1,486 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-"""Start an IPython cluster conveniently, either locally or remotely.
+"""Start an IPython cluster = (controller + engines)."""
 
-Basic usage
------------
-
-For local operation, the simplest mode of usage is:
-
-  %prog -n N
-
-where N is the number of engines you want started.
-
-For remote operation, you must call it with a cluster description file:
-
-  %prog -f clusterfile.py
-
-The cluster file is a normal Python script which gets run via execfile().  You
-can have arbitrary logic in it, but all that matters is that at the end of the
-execution, it declares the variables 'controller', 'engines', and optionally
-'sshx'.  See the accompanying examples for details on what these variables must
-contain.
-
-
-Notes
------
-
-WARNING: this code is still UNFINISHED and EXPERIMENTAL!  It is incomplete,
-some listed options are not really implemented, and all of its interfaces are
-subject to change.
-
-When operating over SSH for a remote cluster, this program relies on the
-existence of a particular script called 'sshx'.  This script must live in the
-target systems where you'll be running your controller and engines, and is
-needed to configure your PATH and PYTHONPATH variables for further execution of
-python code at the other end of an SSH connection.  The script can be as simple
-as:
-
-#!/bin/sh
-. $HOME/.bashrc
-"$@"
-
-which is the default one provided by IPython.  You can modify this or provide
-your own.  Since it's quite likely that for different clusters you may need
-this script to configure things differently or that it may live in different
-locations, its full path can be set in the same file where you define the
-cluster setup.  IPython's order of evaluation for this variable is the
-following:
-
-  a) Internal default: 'sshx'.  This only works if it is in the default system
-  path which SSH sets up in non-interactive mode.
-
-  b) Environment variable: if $IPYTHON_SSHX is defined, this overrides the
-  internal default.
-
-  c) Variable 'sshx' in the cluster configuration file: finally, this will
-  override the previous two values.
- 
-This code is Unix-only, with precious little hope of any of this ever working
-under Windows, since we need SSH from the ground up, we background processes,
-etc.  Ports of this functionality to Windows are welcome.
-
-
-Call summary
-------------
-
-    %prog [options]
-"""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-# Stdlib imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
 
 import os
-import signal
+import re
 import sys
-import time
+import signal
+pjoin = os.path.join
 
-from optparse import OptionParser
-from subprocess import Popen,call
+from twisted.internet import reactor, defer
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python import failure, log
+from twisted.internet.error import ProcessDone, ProcessTerminated
+from twisted.internet.utils import getProcessOutput
 
-#---------------------------------------------------------------------------
-# IPython imports
-#---------------------------------------------------------------------------
-from IPython.tools import utils
-from IPython.config import cutils
+from IPython.external import argparse
+from IPython.external import Itpl
+from IPython.kernel.twistedutil import gatherBoth
+from IPython.kernel.util import printer
+from IPython.genutils import get_ipython_dir, num_cpus
 
-#---------------------------------------------------------------------------
-# Normal code begins
-#---------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+# General process handling code
+#-----------------------------------------------------------------------------
 
-def parse_args():
-    """Parse command line and return opts,args."""
+def find_exe(cmd):
+    try:
+        import win32api
+    except ImportError:
+        raise ImportError('you need to have pywin32 installed for this to work')
+    else:
+        (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd)
+        return path
 
-    parser = OptionParser(usage=__doc__)
-    newopt = parser.add_option  # shorthand
+class ProcessStateError(Exception):
+    pass
 
-    newopt("--controller-port", type="int", dest="controllerport",
-           help="the TCP port the controller is listening on")
+class UnknownStatus(Exception):
+    pass
 
-    newopt("--controller-ip", type="string", dest="controllerip",
-           help="the TCP ip address of the controller")
+class LauncherProcessProtocol(ProcessProtocol):
+    """
+    A ProcessProtocol to go with the ProcessLauncher.
+    """
+    def __init__(self, process_launcher):
+        self.process_launcher = process_launcher
+    
+    def connectionMade(self):
+        self.process_launcher.fire_start_deferred(self.transport.pid)
+    
+    def processEnded(self, status):
+        value = status.value
+        if isinstance(value, ProcessDone):
+            self.process_launcher.fire_stop_deferred(0)
+        elif isinstance(value, ProcessTerminated):
+            self.process_launcher.fire_stop_deferred(
+                {'exit_code':value.exitCode,
+                 'signal':value.signal,
+                 'status':value.status
+                }
+            )
+        else:
+            raise UnknownStatus("unknown exit status, this is probably a bug in Twisted")
 
-    newopt("-n", "--num", type="int", dest="n",default=2,
-           help="the number of engines to start")
+    def outReceived(self, data):
+        log.msg(data)
 
-    newopt("--engine-port", type="int", dest="engineport",
-           help="the TCP port the controller will listen on for engine "
-           "connections")
-    
-    newopt("--engine-ip", type="string", dest="engineip",
-           help="the TCP ip address the controller will listen on "
-           "for engine connections")
+    def errReceived(self, data):
+        log.err(data)
 
-    newopt("--mpi", type="string", dest="mpi",
-           help="use mpi with package:  for instance --mpi=mpi4py")
+class ProcessLauncher(object):
+    """
+    Start and stop an external process in an asynchronous manner.
+    
+    Currently this uses deferreds to notify other parties of process state
+    changes.  This is an awkward design and should be moved to using
+    a formal NotificationCenter.
+    """
+    def __init__(self, cmd_and_args):
+        self.cmd = cmd_and_args[0]
+        self.args = cmd_and_args
+        self._reset()
+    
+    def _reset(self):
+        self.process_protocol = None
+        self.pid = None
+        self.start_deferred = None
+        self.stop_deferreds = []
+        self.state = 'before' # before, running, or after
     
-    newopt("-l", "--logfile", type="string", dest="logfile",
-           help="log file name")
+    @property
+    def running(self):
+        if self.state == 'running':
+            return True
+        else:
+            return False
+    
+    def fire_start_deferred(self, pid):
+        self.pid = pid
+        self.state = 'running'
+        log.msg('Process %r has started with pid=%i' % (self.args, pid))
+        self.start_deferred.callback(pid)
+    
+    def start(self):
+        if self.state == 'before':
+            self.process_protocol = LauncherProcessProtocol(self)
+            self.start_deferred = defer.Deferred()
+            self.process_transport = reactor.spawnProcess(
+                self.process_protocol,
+                self.cmd,
+                self.args,
+                env=os.environ
+            )
+            return self.start_deferred
+        else:
+            s = 'the process has already been started and has state: %r' % \
+                self.state
+            return defer.fail(ProcessStateError(s))
+    
+    def get_stop_deferred(self):
+        if self.state == 'running' or self.state == 'before':
+            d = defer.Deferred()
+            self.stop_deferreds.append(d)
+            return d
+        else:
+            s = 'this process is already complete'
+            return defer.fail(ProcessStateError(s))
+    
+    def fire_stop_deferred(self, exit_code):
+        log.msg('Process %r has stopped with %r' % (self.args, exit_code))
+        self.state = 'after'
+        for d in self.stop_deferreds:
+            d.callback(exit_code)
+    
+    def signal(self, sig):
+        """
+        Send a signal to the process.
+        
+        The argument sig can be ('KILL','INT', etc.) or any signal number.
+        """
+        if self.state == 'running':
+            self.process_transport.signalProcess(sig)
+    
+    # def __del__(self):
+    #     self.signal('KILL')
+    
+    def interrupt_then_kill(self, delay=1.0):
+        self.signal('INT')
+        reactor.callLater(delay, self.signal, 'KILL')
 
-    newopt('-f','--cluster-file',dest='clusterfile',
-           help='file describing a remote cluster')
 
-    return parser.parse_args()
+#-----------------------------------------------------------------------------
+# Code for launching controller and engines
+#-----------------------------------------------------------------------------
 
-def numAlive(controller,engines):
-    """Return the number of processes still alive."""
-    retcodes = [controller.poll()] + \
-               [e.poll() for e in engines]
-    return retcodes.count(None)
 
-stop = lambda pid: os.kill(pid,signal.SIGINT)
-kill = lambda pid: os.kill(pid,signal.SIGTERM)
+class ControllerLauncher(ProcessLauncher):
+    
+    def __init__(self, extra_args=None):
+        if sys.platform == 'win32':
+            args = [find_exe('ipcontroller.bat')]
+        else:
+            args = ['ipcontroller']
+        self.extra_args = extra_args
+        if extra_args is not None:
+            args.extend(extra_args)
+        
+        ProcessLauncher.__init__(self, args)
+
 
-def cleanup(clean,controller,engines):
-    """Stop the controller and engines with the given cleanup method."""
+class EngineLauncher(ProcessLauncher):
     
-    for e in engines:
-        if e.poll() is None:
-            print 'Stopping engine, pid',e.pid
-            clean(e.pid)
-    if controller.poll() is None:
-        print 'Stopping controller, pid',controller.pid
-        clean(controller.pid)
-
-
-def ensureDir(path):
-    """Ensure a directory exists or raise an exception."""
-    if not os.path.isdir(path):
-        os.makedirs(path)
-
-
-def startMsg(control_host,control_port=10105):
-    """Print a startup message"""
-    print
-    print 'Your cluster is up and running.'
-    print
-    print 'For interactive use, you can make a MultiEngineClient with:'
-    print
-    print 'from IPython.kernel import client'
-    print "mec = client.MultiEngineClient()"
-    print
-    print 'You can then cleanly stop the cluster from IPython using:'
-    print
-    print 'mec.kill(controller=True)'
-    print
+    def __init__(self, extra_args=None):
+        if sys.platform == 'win32':
+            args = [find_exe('ipengine.bat')]
+        else:
+            args = ['ipengine']
+        self.extra_args = extra_args
+        if extra_args is not None:
+            args.extend(extra_args)
+        
+        ProcessLauncher.__init__(self, args)
 
+
+class LocalEngineSet(object):
     
-def clusterLocal(opt,arg):
-    """Start a cluster on the local machine."""
+    def __init__(self, extra_args=None):
+        self.extra_args = extra_args
+        self.launchers = []
     
-    # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
-    pjoin = os.path.join
-
-    logfile = opt.logfile
-    if logfile is None:
-        logdir_base = pjoin(ipdir,'log')
-        ensureDir(logdir_base)
-        logfile = pjoin(logdir_base,'ipcluster-')
-
-    print 'Starting controller:',
-    controller = Popen(['ipcontroller','--logfile',logfile,'-x','-y'])
-    print 'Controller PID:',controller.pid
-
-    print 'Starting engines:   ',
-    time.sleep(5)
-
-    englogfile = '%s%s-' % (logfile,controller.pid)
-    mpi = opt.mpi
-    if mpi: # start with mpi - killing the engines with sigterm will not work if you do this
-        engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', 
-            mpi, '--logfile',englogfile])]
-        # engines = [Popen(['mpirun', '-np', str(opt.n), 'ipengine', '--mpi', mpi])]
-    else: # do what we would normally do
-        engines = [ Popen(['ipengine','--logfile',englogfile])
-                    for i in range(opt.n) ]
-    eids = [e.pid for e in engines]
-    print 'Engines PIDs:  ',eids
-    print 'Log files: %s*' % englogfile
+    def start(self, n):
+        dlist = []
+        for i in range(n):
+            el = EngineLauncher(extra_args=self.extra_args)
+            d = el.start()
+            self.launchers.append(el)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_start)
+        return dfinal
     
-    proc_ids = eids + [controller.pid]
-    procs = engines + [controller]
-
-    grpid = os.getpgrp()
-    try:
-        startMsg('127.0.0.1')
-        print 'You can also hit Ctrl-C to stop it, or use from the cmd line:'
-        print
-        print 'kill -INT',grpid
-        print
-        try:
-            while True:
-                time.sleep(5)
-        except:
-            pass
-    finally:
-        print 'Stopping cluster.  Cleaning up...'
-        cleanup(stop,controller,engines)
-        for i in range(4):
-            time.sleep(i+2)
-            nZombies = numAlive(controller,engines)
-            if  nZombies== 0:
-                print 'OK: All processes cleaned up.'
-                break
-            print 'Trying again, %d processes did not stop...' % nZombies
-            cleanup(kill,controller,engines)
-            if numAlive(controller,engines) == 0:
-                print 'OK: All processes cleaned up.'
-                break
-        else:
-            print '*'*75
-            print 'ERROR: could not kill some processes, try to do it',
-            print 'manually.'
-            zombies = []
-            if controller.returncode is None:
-                print 'Controller is alive: pid =',controller.pid
-                zombies.append(controller.pid)
-            liveEngines = [ e for e in engines if e.returncode is None ]
-            for e in liveEngines:
-                print 'Engine is alive:     pid =',e.pid
-                zombies.append(e.pid)
-            print
-            print 'Zombie summary:',' '.join(map(str,zombies))
-
-def clusterRemote(opt,arg):
-    """Start a remote cluster over SSH"""
-
-    # Load the remote cluster configuration
-    clConfig = {}
-    execfile(opt.clusterfile,clConfig)
-    contConfig = clConfig['controller']
-    engConfig = clConfig['engines']
-    # Determine where to find sshx:
-    sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
+    def _handle_start(self, r):
+        log.msg('Engines started with pids: %r' % r)
+        return r
     
-    # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
-    pjoin = os.path.join
-
-    logfile = opt.logfile
-    if logfile is None:
-        logdir_base = pjoin(ipdir,'log')
-        ensureDir(logdir_base)
-        logfile = pjoin(logdir_base,'ipcluster')
-
-    # Append this script's PID to the logfile name always
-    logfile = '%s-%s' % (logfile,os.getpid())
+    def _handle_stop(self, r):
+        log.msg('Engines received signal: %r' % r)
+        return r
     
-    print 'Starting controller:'
-    # Controller data:
-    xsys = os.system
-
-    contHost = contConfig['host']
-    contLog = '%s-con-%s-' % (logfile,contHost)
-    cmd = "ssh %s '%s' 'ipcontroller --logfile %s' &" % \
-          (contHost,sshx,contLog)
-    #print 'cmd:<%s>' % cmd  # dbg
-    xsys(cmd)
-    time.sleep(2)
-
-    print 'Starting engines:   '
-    for engineHost,engineData in engConfig.iteritems():
-        if isinstance(engineData,int):
-            numEngines = engineData
+    def signal(self, sig):
+        dlist = []
+        for el in self.launchers:
+            d = el.get_stop_deferred()
+            dlist.append(d)
+            el.signal(sig)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_stop)
+        return dfinal
+    
+    def interrupt_then_kill(self, delay=1.0):
+        dlist = []
+        for el in self.launchers:
+            d = el.get_stop_deferred()
+            dlist.append(d)
+            el.interrupt_then_kill(delay)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self._handle_stop)
+        return dfinal
+
+
+class BatchEngineSet(object):
+    
+    # Subclasses must fill these in.  See PBSEngineSet
+    submit_command = ''
+    delete_command = ''
+    job_id_regexp = ''
+    
+    def __init__(self, template_file, **kwargs):
+        self.template_file = template_file
+        self.context = {}
+        self.context.update(kwargs)
+        self.batch_file = self.template_file+'-run'
+
+    def parse_job_id(self, output):
+        m = re.match(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
         else:
-            raise NotImplementedError('port configuration not finished for engines')
-
-        print 'Sarting %d engines on %s' % (numEngines,engineHost)
-        engLog = '%s-eng-%s-' % (logfile,engineHost)
-        for i in range(numEngines):
-            cmd = "ssh %s '%s' 'ipengine --controller-ip %s --logfile %s' &" % \
-                      (engineHost,sshx,contHost,engLog)
-            #print 'cmd:<%s>' % cmd  # dbg
-            xsys(cmd)
-        # Wait after each host a little bit
-        time.sleep(1)
-            
-    startMsg(contConfig['host'])
+            raise Exception("job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+    
+    def write_batch_script(self, n):
+        self.context['n'] = n
+        template = open(self.template_file, 'r').read()
+        log.msg('Using template for batch script: %s' % self.template_file)
+        script_as_string = Itpl.itplns(template, self.context)
+        log.msg('Writing instantiated batch script: %s' % self.batch_file)
+        f = open(self.batch_file,'w')
+        f.write(script_as_string)
+        f.close()
+
+    def handle_error(self, f):
+        f.printTraceback()
+        f.raiseException()
+    
+    def start(self, n):
+        self.write_batch_script(n)
+        d = getProcessOutput(self.submit_command,
+            [self.batch_file],env=os.environ)
+        d.addCallback(self.parse_job_id)
+        d.addErrback(self.handle_error)
+        return d
         
-def main():
-    """Main driver for the two big options: local or remote cluster."""
+    def kill(self):
+        d = getProcessOutput(self.delete_command,
+            [self.job_id],env=os.environ)
+        return d
+
+class PBSEngineSet(BatchEngineSet):
+    
+    submit_command = 'qsub'
+    delete_command = 'qdel'
+    job_id_regexp = '\d+'
     
-    opt,arg = parse_args()
+    def __init__(self, template_file, **kwargs):
+        BatchEngineSet.__init__(self, template_file, **kwargs)
+
+
+#-----------------------------------------------------------------------------
+# Main functions for the different types of clusters
+#-----------------------------------------------------------------------------
+
+# TODO:
+# The logic in these codes should be moved into classes like LocalCluster
+# MpirunCluster, PBSCluster, etc.  This would remove alot of the duplications.
+# The main functions should then just parse the command line arguments, create
+# the appropriate class and call a 'start' method.
+
+def main_local(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+        cont_args.append('-x')
+    if args.y:
+        cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(cont_pid):
+        engine_args = []
+        engine_args.append('--logfile=%s' % \
+            pjoin(args.logdir,'ipengine%s-' % cont_pid))
+        eset = LocalEngineSet(extra_args=engine_args)
+        def shutdown(signum, frame):
+            log.msg('Stopping local cluster')
+            # We are still playing with the times here, but these seem
+            # to be reliable in allowing everything to exit cleanly.
+            eset.interrupt_then_kill(0.5)
+            cl.interrupt_then_kill(0.5)
+            reactor.callLater(1.0, reactor.stop)
+        signal.signal(signal.SIGINT,shutdown)
+        d = eset.start(args.n)
+        return d
+    def delay_start(cont_pid):
+        # This is needed because the controller doesn't start listening
+        # right when it starts and the controller needs to write
+        # furl files for the engine to pick up
+        reactor.callLater(1.0, start_engines, cont_pid)
+    dstart.addCallback(delay_start)
+    dstart.addErrback(lambda f: f.raiseException())
+
+def main_mpirun(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+        cont_args.append('-x')
+    if args.y:
+        cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(cont_pid):
+        raw_args = ['mpirun']
+        raw_args.extend(['-n',str(args.n)])
+        raw_args.append('ipengine')
+        raw_args.append('-l')
+        raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid))
+        if args.mpi:
+            raw_args.append('--mpi=%s' % args.mpi)
+        eset = ProcessLauncher(raw_args)
+        def shutdown(signum, frame):
+            log.msg('Stopping local cluster')
+            # We are still playing with the times here, but these seem
+            # to be reliable in allowing everything to exit cleanly.
+            eset.interrupt_then_kill(1.0)
+            cl.interrupt_then_kill(1.0)
+            reactor.callLater(2.0, reactor.stop)
+        signal.signal(signal.SIGINT,shutdown)
+        d = eset.start()
+        return d
+    def delay_start(cont_pid):
+        # This is needed because the controller doesn't start listening
+        # right when it starts and the controller needs to write
+        # furl files for the engine to pick up
+        reactor.callLater(1.0, start_engines, cont_pid)
+    dstart.addCallback(delay_start)
+    dstart.addErrback(lambda f: f.raiseException())
+
+def main_pbs(args):
+    cont_args = []
+    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
+    if args.x:
+	cont_args.append('-x')
+    if args.y:
+	cont_args.append('-y')
+    cl = ControllerLauncher(extra_args=cont_args)
+    dstart = cl.start()
+    def start_engines(r):
+        pbs_set =  PBSEngineSet(args.pbsscript)
+        def shutdown(signum, frame):
+            log.msg('Stopping pbs cluster')
+            d = pbs_set.kill()
+            d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
+            d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
+        signal.signal(signal.SIGINT,shutdown)
+        d = pbs_set.start(args.n)
+        return d
+    dstart.addCallback(start_engines)
+    dstart.addErrback(lambda f: f.raiseException())
+
+
+def get_args():
+    base_parser = argparse.ArgumentParser(add_help=False)
+    base_parser.add_argument(
+        '-x',
+        action='store_true',
+        dest='x',
+        help='turn off client security'
+    )
+    base_parser.add_argument(
+        '-y',
+        action='store_true',
+        dest='y',
+        help='turn off engine security'
+    )
+    base_parser.add_argument(
+        "--logdir", 
+        type=str,
+        dest="logdir",
+        help="directory to put log files (default=$IPYTHONDIR/log)",
+        default=pjoin(get_ipython_dir(),'log')
+    )
+    base_parser.add_argument(
+        "-n",
+        "--num", 
+        type=int,
+        dest="n",
+        default=2,
+        help="the number of engines to start"
+    )
+    
+    parser = argparse.ArgumentParser(
+        description='IPython cluster startup.  This starts a controller and\
+        engines using various approaches.  THIS IS A TECHNOLOGY PREVIEW AND\
+        THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.'
+    )
+    subparsers = parser.add_subparsers(
+        help='available cluster types.  For help, do "ipcluster TYPE --help"')
+    
+    parser_local = subparsers.add_parser(
+        'local',
+        help='run a local cluster',
+        parents=[base_parser]
+    )
+    parser_local.set_defaults(func=main_local)
+    
+    parser_mpirun = subparsers.add_parser(
+        'mpirun',
+        help='run a cluster using mpirun',
+        parents=[base_parser]
+    )
+    parser_mpirun.add_argument(
+        "--mpi",
+        type=str,
+        dest="mpi", # Don't put a default here to allow no MPI support
+        help="how to call MPI_Init (default=mpi4py)"
+    )
+    parser_mpirun.set_defaults(func=main_mpirun)
+    
+    parser_pbs = subparsers.add_parser(
+        'pbs', 
+        help='run a pbs cluster',
+        parents=[base_parser]
+    )
+    parser_pbs.add_argument(
+        '--pbs-script',
+        type=str, 
+        dest='pbsscript',
+        help='PBS script template',
+        default='pbs.template'
+    )
+    parser_pbs.set_defaults(func=main_pbs)
+    args = parser.parse_args()
+    return args
 
-    clusterfile = opt.clusterfile
-    if clusterfile:
-        clusterRemote(opt,arg)
-    else:
-        clusterLocal(opt,arg)
-        
-            
-if __name__=='__main__':
+def main():
+    args = get_args()
+    reactor.callWhenRunning(args.func, args)
+    log.startLogging(sys.stdout)
+    reactor.run()
+
+if __name__ == '__main__':
     main()

IPython/kernel/scripts/ipcontroller.py

@@ -64,7 +64,10 b' def make_tub(ip, port, secure, cert_file):'
         if have_crypto:
             tub = Tub(certFile=cert_file)
         else:
-            raise SecurityError("OpenSSL is not available, so we can't run in secure mode, aborting")
+            raise SecurityError("""
+OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
+Try running without security using 'ipcontroller -xy'.
+""")
     else:
         tub = UnauthenticatedTub()
     
@@ -202,6 +205,17 b' def start_controller():'
         except:
             log.msg("Error running import_statement: %s" % cis)
     
+    # Delete old furl files unless the reuse_furls is set
+    reuse = config['controller']['reuse_furls']
+    if not reuse:
+        paths = (config['controller']['engine_furl_file'],
+            config['controller']['controller_interfaces']['task']['furl_file'],
+            config['controller']['controller_interfaces']['multiengine']['furl_file']
+        )
+        for p in paths:
+            if os.path.isfile(p):
+                os.remove(p)
+    
     # Create the service hierarchy
     main_service = service.MultiService()
     # The controller service
@@ -316,6 +330,12 b' def init_config():'
         dest="ipythondir",
         help="look for config files and profiles in this directory"
     )
+    parser.add_option(
+        "-r",
+        action="store_true",
+        dest="reuse_furls",
+        help="try to reuse all furl files"
+    )
     
     (options, args) = parser.parse_args()
     
@@ -349,6 +369,8 b' def init_config():'
         config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
     if options.engine_furl_file is not None:
         config['controller']['engine_furl_file'] = options.engine_furl_file
+    if options.reuse_furls is not None:
+            config['controller']['reuse_furls'] = options.reuse_furls
 
     if options.logfile is not None:
         config['controller']['logfile'] = options.logfile

IPython/kernel/scripts/ipengine.py

@@ -91,7 +91,7 b' def start_engine():'
         try:
             engine_service.execute(shell_import_statement)
         except:
-            log.msg("Error running import_statement: %s" % sis)
+            log.msg("Error running import_statement: %s" % shell_import_statement)
     
     # Create the service hierarchy
     main_service = service.MultiService()
@@ -105,8 +105,13 b' def start_engine():'
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
+    log.msg("Using furl file: %s" % furl_file)
     d = engine_connector.connect_to_controller(engine_service, furl_file)
-    d.addErrback(lambda _: reactor.stop())
+    def handle_error(f):
+        log.err(f)
+        if reactor.running:
+            reactor.stop()
+    d.addErrback(handle_error)
     
     reactor.run()
 
@@ -168,4 +173,4 b' def main():'
 
 
 if __name__ == "__main__":
-    main()
  No newline at end of file
+    main()

IPython/kernel/tests/engineservicetest.py

@@ -245,7 +245,7 b' class IEngineSerializedTestCase(object):'
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireCore."""
+        """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
@@ -288,7 +288,7 b' class IEngineQueuedTestCase(object):'
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireQueued."""
+        """Does self.engine have the methods and attributes in IEngineQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
@@ -326,7 +326,7 b' class IEnginePropertiesTestCase(object):'
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireProperties."""
+        """Does self.engine have the methods and attributes in IEngineProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     

IPython/kernel/tests/multienginetest.py

@@ -20,7 +20,6 b' from twisted.internet import defer'
 from IPython.kernel import engineservice as es
 from IPython.kernel import multiengine as me
 from IPython.kernel import newserialized
-from IPython.kernel.error import NotDefined
 from IPython.testing import util
 from IPython.testing.parametric import parametric, Parametric
 from IPython.kernel import newserialized

IPython/kernel/tests/test_contexts.py

@@ -1,4 +1,6 b''
-from __future__ import with_statement
+#from __future__ import with_statement
+
+# XXX This file is currently disabled to preserve 2.4 compatibility.
 
 #def test_simple():
 if 0:
@@ -25,17 +27,17 b' if 0:'
 
     mec.pushAll()
 
-    with parallel as pr:
-        # A comment
-        remote()  # this means the code below only runs remotely
-        print 'Hello remote world'
-        x = range(10)
-        # Comments are OK
-        # Even misindented.
-        y = x+1
+    ## with parallel as pr:
+    ##     # A comment
+    ##     remote()  # this means the code below only runs remotely
+    ##     print 'Hello remote world'
+    ##     x = range(10)
+    ##     # Comments are OK
+    ##     # Even misindented.
+    ##     y = x+1
 
 
-    with pfor('i',sequence) as pr:
-        print x[i]
+    ## with pfor('i',sequence) as pr:
+    ##     print x[i]
 
     print pr.x + pr.y

IPython/kernel/tests/test_controllerservice.py

@@ -30,14 +30,15 b' try:'
     from controllertest import IControllerCoreTestCase
     from IPython.testing.util import DeferredTestCase
 except ImportError:
-    pass
-else:
-    class BasicControllerServiceTest(DeferredTestCase,
-        IControllerCoreTestCase):
-    
-        def setUp(self):
-            self.controller  = ControllerService()
-            self.controller.startService()
-    
-        def tearDown(self):
-            self.controller.stopService()
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+class BasicControllerServiceTest(DeferredTestCase,
+    IControllerCoreTestCase):
+
+    def setUp(self):
+        self.controller  = ControllerService()
+        self.controller.startService()
+
+    def tearDown(self):
+        self.controller.stopService()

IPython/kernel/tests/test_enginefc.py

@@ -37,56 +37,57 b' try:'
         IEngineSerializedTestCase, \
         IEngineQueuedTestCase
 except ImportError:
-    print "we got an error!!!"
-    raise
-else:
-    class EngineFCTest(DeferredTestCase, 
-                       IEngineCoreTestCase, 
-                       IEngineSerializedTestCase,
-                       IEngineQueuedTestCase
-                       ):
-        
-        zi.implements(IControllerBase)
-        
-        def setUp(self):
-        
-            # Start a server and append to self.servers
-            self.controller_reference = FCRemoteEngineRefFromService(self)
-            self.controller_tub = Tub()
-            self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-            self.controller_tub.setLocation('127.0.0.1:10105')
-            
-            furl = self.controller_tub.registerReference(self.controller_reference)
-            self.controller_tub.startService()
-            
-            # Start an EngineService and append to services/client
-            self.engine_service = es.EngineService()
-            self.engine_service.startService()
-            self.engine_tub = Tub()
-            self.engine_tub.startService()
-            engine_connector = EngineConnector(self.engine_tub)
-            d = engine_connector.connect_to_controller(self.engine_service, furl)
-            # This deferred doesn't fire until after register_engine has returned and
-            # thus, self.engine has been defined and the tets can proceed.
-            return d
-        
-        def tearDown(self):
-            dlist = []
-            # Shut down the engine
-            d = self.engine_tub.stopService()
-            dlist.append(d)
-            # Shut down the controller
-            d = self.controller_tub.stopService()
-            dlist.append(d)
-            return defer.DeferredList(dlist)
-        
-        #---------------------------------------------------------------------------
-        # Make me look like a basic controller
-        #---------------------------------------------------------------------------
-        
-        def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None):
-            self.engine = IEngineQueued(IEngineBase(engine_ref))
-            return {'id':id}
-        
-        def unregister_engine(self, id):
-            pass
  No newline at end of file
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+
+class EngineFCTest(DeferredTestCase, 
+                 IEngineCoreTestCase, 
+                 IEngineSerializedTestCase,
+                 IEngineQueuedTestCase
+                 ):
+ 
+  zi.implements(IControllerBase)
+ 
+  def setUp(self):
+ 
+      # Start a server and append to self.servers
+      self.controller_reference = FCRemoteEngineRefFromService(self)
+      self.controller_tub = Tub()
+      self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
+      self.controller_tub.setLocation('127.0.0.1:10105')
+     
+      furl = self.controller_tub.registerReference(self.controller_reference)
+      self.controller_tub.startService()
+     
+      # Start an EngineService and append to services/client
+      self.engine_service = es.EngineService()
+      self.engine_service.startService()
+      self.engine_tub = Tub()
+      self.engine_tub.startService()
+      engine_connector = EngineConnector(self.engine_tub)
+      d = engine_connector.connect_to_controller(self.engine_service, furl)
+      # This deferred doesn't fire until after register_engine has returned and
+      # thus, self.engine has been defined and the tets can proceed.
+      return d
+ 
+  def tearDown(self):
+      dlist = []
+      # Shut down the engine
+      d = self.engine_tub.stopService()
+      dlist.append(d)
+      # Shut down the controller
+      d = self.controller_tub.stopService()
+      dlist.append(d)
+      return defer.DeferredList(dlist)
+ 
+  #---------------------------------------------------------------------------
+  # Make me look like a basic controller
+  #---------------------------------------------------------------------------
+ 
+  def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None):
+      self.engine = IEngineQueued(IEngineBase(engine_ref))
+      return {'id':id}
+ 
+  def unregister_engine(self, id):
+      pass
  No newline at end of file

IPython/kernel/tests/test_engineservice.py

@@ -35,44 +35,46 b' try:'
         IEngineQueuedTestCase, \
         IEnginePropertiesTestCase
 except ImportError:
-    pass
-else:
-    class BasicEngineServiceTest(DeferredTestCase,
-                                 IEngineCoreTestCase, 
-                                 IEngineSerializedTestCase,
-                                 IEnginePropertiesTestCase):
-    
-        def setUp(self):
-            self.engine = es.EngineService()
-            self.engine.startService()
-    
-        def tearDown(self):
-            return self.engine.stopService()
-    
-    class ThreadedEngineServiceTest(DeferredTestCase,
-                                 IEngineCoreTestCase, 
-                                 IEngineSerializedTestCase,
-                                 IEnginePropertiesTestCase):
-    
-        def setUp(self):
-            self.engine = es.ThreadedEngineService()
-            self.engine.startService()
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")    
+
+
+class BasicEngineServiceTest(DeferredTestCase,
+                             IEngineCoreTestCase, 
+                             IEngineSerializedTestCase,
+                             IEnginePropertiesTestCase):
+
+    def setUp(self):
+        self.engine = es.EngineService()
+        self.engine.startService()
+
+    def tearDown(self):
+        return self.engine.stopService()
+
+class ThreadedEngineServiceTest(DeferredTestCase,
+                             IEngineCoreTestCase, 
+                             IEngineSerializedTestCase,
+                             IEnginePropertiesTestCase):
+
+    def setUp(self):
+        self.engine = es.ThreadedEngineService()
+        self.engine.startService()
+
+    def tearDown(self):
+        return self.engine.stopService()
+
+class QueuedEngineServiceTest(DeferredTestCase,
+                              IEngineCoreTestCase, 
+                              IEngineSerializedTestCase,
+                              IEnginePropertiesTestCase,
+                              IEngineQueuedTestCase):
+                          
+    def setUp(self):
+        self.rawEngine = es.EngineService()
+        self.rawEngine.startService()
+        self.engine = es.IEngineQueued(self.rawEngine)
     
-        def tearDown(self):
-            return self.engine.stopService()
-
-    class QueuedEngineServiceTest(DeferredTestCase,
-                                  IEngineCoreTestCase, 
-                                  IEngineSerializedTestCase,
-                                  IEnginePropertiesTestCase,
-                                  IEngineQueuedTestCase):
-                              
-        def setUp(self):
-            self.rawEngine = es.EngineService()
-            self.rawEngine.startService()
-            self.engine = es.IEngineQueued(self.rawEngine)
-        
-        def tearDown(self):
-            return self.rawEngine.stopService()
-
- 
+    def tearDown(self):
+        return self.rawEngine.stopService()
+
+

IPython/kernel/tests/test_multiengine.py

@@ -23,32 +23,34 b' try:'
     from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
         ISynchronousMultiEngineTestCase)
 except ImportError:
-    pass
-else:    
-    class BasicMultiEngineTestCase(DeferredTestCase, IMultiEngineTestCase):
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+ 
+class BasicMultiEngineTestCase(DeferredTestCase, IMultiEngineTestCase):
+
+    def setUp(self):
+        self.controller = ControllerService()
+        self.controller.startService()
+        self.multiengine = me.IMultiEngine(self.controller)
+        self.engines = []
     
-        def setUp(self):
-            self.controller = ControllerService()
-            self.controller.startService()
-            self.multiengine = me.IMultiEngine(self.controller)
-            self.engines = []
-        
-        def tearDown(self):
-            self.controller.stopService()
-            for e in self.engines:
-                e.stopService()
-
-
-    class SynchronousMultiEngineTestCase(DeferredTestCase, ISynchronousMultiEngineTestCase):
+    def tearDown(self):
+        self.controller.stopService()
+        for e in self.engines:
+            e.stopService()
+
+
+class SynchronousMultiEngineTestCase(DeferredTestCase, ISynchronousMultiEngineTestCase):
+
+    def setUp(self):
+        self.controller = ControllerService()
+        self.controller.startService()
+        self.multiengine = me.ISynchronousMultiEngine(me.IMultiEngine(self.controller))
+        self.engines = []
     
-        def setUp(self):
-            self.controller = ControllerService()
-            self.controller.startService()
-            self.multiengine = me.ISynchronousMultiEngine(me.IMultiEngine(self.controller))
-            self.engines = []
-        
-        def tearDown(self):
-            self.controller.stopService()
-            for e in self.engines:
-                e.stopService()
+    def tearDown(self):
+        self.controller.stopService()
+        for e in self.engines:
+            e.stopService()
 

IPython/kernel/tests/test_multienginefc.py

@@ -30,115 +30,115 b' try:'
     from IPython.kernel.error import CompositeError
     from IPython.kernel.util import printer
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
     
-    def _raise_it(f):
-        try:
-            f.raiseException()
-        except CompositeError, e:
-            e.raise_exception()
+def _raise_it(f):
+    try:
+        f.raiseException()
+    except CompositeError, e:
+        e.raise_exception()
+
+
+class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase):
+
+    def setUp(self):
     
+        self.engines = []
+            
+        self.controller = ControllerService()
+        self.controller.startService()
+        self.imultiengine = IMultiEngine(self.controller)
+        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
+
+        self.controller_tub = Tub()
+        self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
+        self.controller_tub.setLocation('127.0.0.1:10105')
     
-    class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase):
+        furl = self.controller_tub.registerReference(self.mec_referenceable)
+        self.controller_tub.startService()
     
-        def setUp(self):
-        
-            self.engines = []
-                
-            self.controller = ControllerService()
-            self.controller.startService()
-            self.imultiengine = IMultiEngine(self.controller)
-            self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
-
-            self.controller_tub = Tub()
-            self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-            self.controller_tub.setLocation('127.0.0.1:10105')
-        
-            furl = self.controller_tub.registerReference(self.mec_referenceable)
-            self.controller_tub.startService()
-        
-            self.client_tub = ClientConnector()
-            d = self.client_tub.get_multiengine_client(furl)
-            d.addCallback(self.handle_got_client)
-            return d
-        
-        def handle_got_client(self, client):
-            self.multiengine = client
+        self.client_tub = ClientConnector()
+        d = self.client_tub.get_multiengine_client(furl)
+        d.addCallback(self.handle_got_client)
+        return d
     
-        def tearDown(self):
-            dlist = []
-            # Shut down the multiengine client
-            d = self.client_tub.tub.stopService()
-            dlist.append(d)
-            # Shut down the engines
-            for e in self.engines:
-                e.stopService()
-            # Shut down the controller
-            d = self.controller_tub.stopService()
-            d.addBoth(lambda _: self.controller.stopService())
-            dlist.append(d)
-            return defer.DeferredList(dlist)
+    def handle_got_client(self, client):
+        self.multiengine = client
+
+    def tearDown(self):
+        dlist = []
+        # Shut down the multiengine client
+        d = self.client_tub.tub.stopService()
+        dlist.append(d)
+        # Shut down the engines
+        for e in self.engines:
+            e.stopService()
+        # Shut down the controller
+        d = self.controller_tub.stopService()
+        d.addBoth(lambda _: self.controller.stopService())
+        dlist.append(d)
+        return defer.DeferredList(dlist)
 
-        def test_mapper(self):
-            self.addEngine(4)
-            m = self.multiengine.mapper()
-            self.assertEquals(m.multiengine,self.multiengine)
-            self.assertEquals(m.dist,'b')
-            self.assertEquals(m.targets,'all')
-            self.assertEquals(m.block,True)
-        
-        def test_map_default(self):
-            self.addEngine(4)
-            m = self.multiengine.mapper()
-            d = m.map(lambda x: 2*x, range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, range(10)))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-        
-        def test_map_noblock(self):
-            self.addEngine(4)
-            m = self.multiengine.mapper(block=False)
-            d = m.map(lambda x: 2*x, range(10))
-            d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-                
-        def test_mapper_fail(self):
-            self.addEngine(4)
-            m = self.multiengine.mapper()
-            d = m.map(lambda x: 1/0, range(10))
-            d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-            return d
-        
-        def test_parallel(self):
-            self.addEngine(4)
-            p = self.multiengine.parallel()
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 2*x
-            d = f(range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-        
-        def test_parallel_noblock(self):
-            self.addEngine(1)
-            p = self.multiengine.parallel(block=False)
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 2*x
-            d = f(range(10))
-            d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-        
-        def test_parallel_fail(self):
-            self.addEngine(4)
-            p = self.multiengine.parallel()
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 1/0
-            d = f(range(10))
-            d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-            return d
  No newline at end of file
+    def test_mapper(self):
+        self.addEngine(4)
+        m = self.multiengine.mapper()
+        self.assertEquals(m.multiengine,self.multiengine)
+        self.assertEquals(m.dist,'b')
+        self.assertEquals(m.targets,'all')
+        self.assertEquals(m.block,True)
+    
+    def test_map_default(self):
+        self.addEngine(4)
+        m = self.multiengine.mapper()
+        d = m.map(lambda x: 2*x, range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, range(10)))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
+    
+    def test_map_noblock(self):
+        self.addEngine(4)
+        m = self.multiengine.mapper(block=False)
+        d = m.map(lambda x: 2*x, range(10))
+        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
+            
+    def test_mapper_fail(self):
+        self.addEngine(4)
+        m = self.multiengine.mapper()
+        d = m.map(lambda x: 1/0, range(10))
+        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
+        return d
+    
+    def test_parallel(self):
+        self.addEngine(4)
+        p = self.multiengine.parallel()
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 2*x
+        d = f(range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
+    
+    def test_parallel_noblock(self):
+        self.addEngine(1)
+        p = self.multiengine.parallel(block=False)
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 2*x
+        d = f(range(10))
+        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
+    
+    def test_parallel_fail(self):
+        self.addEngine(4)
+        p = self.multiengine.parallel()
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 1/0
+        d = f(range(10))
+        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
+        return d
  No newline at end of file

IPython/kernel/tests/test_newserialized.py

@@ -28,75 +28,75 b' try:'
         SerializeIt, \
         UnSerializeIt
 except ImportError:
-    pass
-else:
-    #-------------------------------------------------------------------------------
-    # Tests
-    #-------------------------------------------------------------------------------
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+
+#-------------------------------------------------------------------------------
+# Tests
+#-------------------------------------------------------------------------------
+
+class SerializedTestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
     
-    class SerializedTestCase(unittest.TestCase):
+    def tearDown(self):
+        pass
     
-        def setUp(self):
-            pass
-        
-        def tearDown(self):
-            pass
-        
-        def testSerializedInterfaces(self):
+    def testSerializedInterfaces(self):
 
-            us = UnSerialized({'a':10, 'b':range(10)})
-            s = ISerialized(us)
-            uss = IUnSerialized(s)
-            self.assert_(ISerialized.providedBy(s))
-            self.assert_(IUnSerialized.providedBy(us))
-            self.assert_(IUnSerialized.providedBy(uss))
-            for m in list(ISerialized):
-                self.assert_(hasattr(s, m))
-            for m in list(IUnSerialized):
-                self.assert_(hasattr(us, m))
-            for m in list(IUnSerialized):
-                self.assert_(hasattr(uss, m))
+        us = UnSerialized({'a':10, 'b':range(10)})
+        s = ISerialized(us)
+        uss = IUnSerialized(s)
+        self.assert_(ISerialized.providedBy(s))
+        self.assert_(IUnSerialized.providedBy(us))
+        self.assert_(IUnSerialized.providedBy(uss))
+        for m in list(ISerialized):
+            self.assert_(hasattr(s, m))
+        for m in list(IUnSerialized):
+            self.assert_(hasattr(us, m))
+        for m in list(IUnSerialized):
+            self.assert_(hasattr(uss, m))
 
-        def testPickleSerialized(self):
-            obj = {'a':1.45345, 'b':'asdfsdf', 'c':10000L}
-            original = UnSerialized(obj)
-            originalSer = ISerialized(original)
-            firstData = originalSer.getData()
-            firstTD = originalSer.getTypeDescriptor()
-            firstMD = originalSer.getMetadata()
-            self.assert_(firstTD == 'pickle')
-            self.assert_(firstMD == {})
-            unSerialized = IUnSerialized(originalSer)
-            secondObj = unSerialized.getObject()
-            for k, v in secondObj.iteritems():
-                self.assert_(obj[k] == v)
-            secondSer = ISerialized(UnSerialized(secondObj))
-            self.assert_(firstData == secondSer.getData())
-            self.assert_(firstTD == secondSer.getTypeDescriptor() )
-            self.assert_(firstMD == secondSer.getMetadata())
+    def testPickleSerialized(self):
+        obj = {'a':1.45345, 'b':'asdfsdf', 'c':10000L}
+        original = UnSerialized(obj)
+        originalSer = ISerialized(original)
+        firstData = originalSer.getData()
+        firstTD = originalSer.getTypeDescriptor()
+        firstMD = originalSer.getMetadata()
+        self.assert_(firstTD == 'pickle')
+        self.assert_(firstMD == {})
+        unSerialized = IUnSerialized(originalSer)
+        secondObj = unSerialized.getObject()
+        for k, v in secondObj.iteritems():
+            self.assert_(obj[k] == v)
+        secondSer = ISerialized(UnSerialized(secondObj))
+        self.assert_(firstData == secondSer.getData())
+        self.assert_(firstTD == secondSer.getTypeDescriptor() )
+        self.assert_(firstMD == secondSer.getMetadata())
+    
+    def testNDArraySerialized(self):
+        try:
+            import numpy
+        except ImportError:
+            pass
+        else:
+            a = numpy.linspace(0.0, 1.0, 1000)
+            unSer1 = UnSerialized(a)
+            ser1 = ISerialized(unSer1)
+            td = ser1.getTypeDescriptor()
+            self.assert_(td == 'ndarray')
+            md = ser1.getMetadata()
+            self.assert_(md['shape'] == a.shape)
+            self.assert_(md['dtype'] == a.dtype.str)
+            buff = ser1.getData()
+            self.assert_(buff == numpy.getbuffer(a))
+            s = Serialized(buff, td, md)
+            us = IUnSerialized(s)
+            final = us.getObject()
+            self.assert_(numpy.getbuffer(a) == numpy.getbuffer(final))
+            self.assert_(a.dtype.str == final.dtype.str)
+            self.assert_(a.shape == final.shape)
+        
         
-        def testNDArraySerialized(self):
-            try:
-                import numpy
-            except ImportError:
-                pass
-            else:
-                a = numpy.linspace(0.0, 1.0, 1000)
-                unSer1 = UnSerialized(a)
-                ser1 = ISerialized(unSer1)
-                td = ser1.getTypeDescriptor()
-                self.assert_(td == 'ndarray')
-                md = ser1.getMetadata()
-                self.assert_(md['shape'] == a.shape)
-                self.assert_(md['dtype'] == a.dtype.str)
-                buff = ser1.getData()
-                self.assert_(buff == numpy.getbuffer(a))
-                s = Serialized(buff, td, md)
-                us = IUnSerialized(s)
-                final = us.getObject()
-                self.assert_(numpy.getbuffer(a) == numpy.getbuffer(final))
-                self.assert_(a.dtype.str == final.dtype.str)
-                self.assert_(a.shape == final.shape)
-            
-            
-    
  No newline at end of file

IPython/kernel/tests/test_pendingdeferred.py

@@ -25,162 +25,162 b' try:'
     from IPython.kernel import error
     from IPython.kernel.util import printer
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
     
-    class Foo(object):
-    
-        def bar(self, bahz):
-            return defer.succeed('blahblah: %s' % bahz)
+class Foo(object):
 
-    class TwoPhaseFoo(pd.PendingDeferredManager):
-    
-        def __init__(self, foo):
-            self.foo = foo
-            pd.PendingDeferredManager.__init__(self)
+    def bar(self, bahz):
+        return defer.succeed('blahblah: %s' % bahz)
 
-        @pd.two_phase
-        def bar(self, bahz):
-            return self.foo.bar(bahz)
+class TwoPhaseFoo(pd.PendingDeferredManager):
 
-    class PendingDeferredManagerTest(DeferredTestCase):
-    
-        def setUp(self):
-            self.pdm = pd.PendingDeferredManager()
-        
-        def tearDown(self):
-            pass
-    
-        def testBasic(self):
-            dDict = {}
-            # Create 10 deferreds and save them
-            for i in range(10):
-                d = defer.Deferred()
-                did = self.pdm.save_pending_deferred(d)
-                dDict[did] = d
-            # Make sure they are begin saved
-            for k in dDict.keys():
-                self.assert_(self.pdm.quick_has_id(k))
-            # Get the pending deferred (block=True), then callback with 'foo' and compare
-            for did in dDict.keys()[0:5]:
-                d = self.pdm.get_pending_deferred(did,block=True)
-                dDict[did].callback('foo')
-                d.addCallback(lambda r: self.assert_(r=='foo'))
-            # Get the pending deferreds with (block=False) and make sure ResultNotCompleted is raised
-            for did in dDict.keys()[5:10]:
-                d = self.pdm.get_pending_deferred(did,block=False)
-                d.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
-            # Now callback the last 5, get them and compare.
-            for did in dDict.keys()[5:10]:
-                dDict[did].callback('foo')
-                d = self.pdm.get_pending_deferred(did,block=False)
-                d.addCallback(lambda r: self.assert_(r=='foo'))
-    
-        def test_save_then_delete(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            self.assert_(self.pdm.quick_has_id(did))
-            self.pdm.delete_pending_deferred(did)
-            self.assert_(not self.pdm.quick_has_id(did))
-    
-        def test_save_get_delete(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d2.addErrback(lambda f: self.assertRaises(error.AbortedPendingDeferredError, f.raiseException))
-            self.pdm.delete_pending_deferred(did)
-            return d2
-   
-        def test_double_get(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d3 = self.pdm.get_pending_deferred(did,True)
-            d3.addErrback(lambda f: self.assertRaises(error.InvalidDeferredID, f.raiseException))
-    
-        def test_get_after_callback(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d.callback('foo')
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d2.addCallback(lambda r: self.assertEquals(r,'foo'))
-            self.assert_(not self.pdm.quick_has_id(did))
+    def __init__(self, foo):
+        self.foo = foo
+        pd.PendingDeferredManager.__init__(self)
 
-        def test_get_before_callback(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d.callback('foo')
-            d2.addCallback(lambda r: self.assertEquals(r,'foo'))
-            self.assert_(not self.pdm.quick_has_id(did))
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d2.addCallback(lambda r: self.assertEquals(r,'foo'))
-            d.callback('foo')
-            self.assert_(not self.pdm.quick_has_id(did))
-    
-        def test_get_after_errback(self):
-            class MyError(Exception):
-                pass
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d.errback(failure.Failure(MyError('foo')))
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
-            self.assert_(not self.pdm.quick_has_id(did))
-
-        def test_get_before_errback(self):
-            class MyError(Exception):
-                pass
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d.errback(failure.Failure(MyError('foo')))
-            d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
-            self.assert_(not self.pdm.quick_has_id(did))
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
-            d.errback(failure.Failure(MyError('foo')))
-            self.assert_(not self.pdm.quick_has_id(did))
-        
-        def test_noresult_noblock(self):
-            d = defer.Deferred()
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,False)
-            d2.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
+    @pd.two_phase
+    def bar(self, bahz):
+        return self.foo.bar(bahz)
 
-        def test_with_callbacks(self):
-            d = defer.Deferred()
-            d.addCallback(lambda r: r+' foo')
-            d.addCallback(lambda r: r+' bar')
-            did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d.callback('bam')
-            d2.addCallback(lambda r: self.assertEquals(r,'bam foo bar'))
+class PendingDeferredManagerTest(DeferredTestCase):
+
+    def setUp(self):
+        self.pdm = pd.PendingDeferredManager()
     
-        def test_with_errbacks(self):
-            class MyError(Exception):
-                pass
+    def tearDown(self):
+        pass
+
+    def testBasic(self):
+        dDict = {}
+        # Create 10 deferreds and save them
+        for i in range(10):
             d = defer.Deferred()
-            d.addCallback(lambda r: 'foo')
-            d.addErrback(lambda f: 'caught error')
             did = self.pdm.save_pending_deferred(d)
-            d2 = self.pdm.get_pending_deferred(did,True)
-            d.errback(failure.Failure(MyError('bam')))
-            d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
+            dDict[did] = d
+        # Make sure they are begin saved
+        for k in dDict.keys():
+            self.assert_(self.pdm.quick_has_id(k))
+        # Get the pending deferred (block=True), then callback with 'foo' and compare
+        for did in dDict.keys()[0:5]:
+            d = self.pdm.get_pending_deferred(did,block=True)
+            dDict[did].callback('foo')
+            d.addCallback(lambda r: self.assert_(r=='foo'))
+        # Get the pending deferreds with (block=False) and make sure ResultNotCompleted is raised
+        for did in dDict.keys()[5:10]:
+            d = self.pdm.get_pending_deferred(did,block=False)
+            d.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
+        # Now callback the last 5, get them and compare.
+        for did in dDict.keys()[5:10]:
+            dDict[did].callback('foo')
+            d = self.pdm.get_pending_deferred(did,block=False)
+            d.addCallback(lambda r: self.assert_(r=='foo'))
+
+    def test_save_then_delete(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        self.assert_(self.pdm.quick_has_id(did))
+        self.pdm.delete_pending_deferred(did)
+        self.assert_(not self.pdm.quick_has_id(did))
+
+    def test_save_get_delete(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d2.addErrback(lambda f: self.assertRaises(error.AbortedPendingDeferredError, f.raiseException))
+        self.pdm.delete_pending_deferred(did)
+        return d2
+
+    def test_double_get(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d3 = self.pdm.get_pending_deferred(did,True)
+        d3.addErrback(lambda f: self.assertRaises(error.InvalidDeferredID, f.raiseException))
+
+    def test_get_after_callback(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d.callback('foo')
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d2.addCallback(lambda r: self.assertEquals(r,'foo'))
+        self.assert_(not self.pdm.quick_has_id(did))
+
+    def test_get_before_callback(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d.callback('foo')
+        d2.addCallback(lambda r: self.assertEquals(r,'foo'))
+        self.assert_(not self.pdm.quick_has_id(did))
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d2.addCallback(lambda r: self.assertEquals(r,'foo'))
+        d.callback('foo')
+        self.assert_(not self.pdm.quick_has_id(did))
+
+    def test_get_after_errback(self):
+        class MyError(Exception):
+            pass
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d.errback(failure.Failure(MyError('foo')))
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
+        self.assert_(not self.pdm.quick_has_id(did))
+
+    def test_get_before_errback(self):
+        class MyError(Exception):
+            pass
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d.errback(failure.Failure(MyError('foo')))
+        d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
+        self.assert_(not self.pdm.quick_has_id(did))
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
+        d.errback(failure.Failure(MyError('foo')))
+        self.assert_(not self.pdm.quick_has_id(did))
     
-        def test_nested_deferreds(self):
-            d = defer.Deferred()
-            d2 = defer.Deferred()
-            d.addCallback(lambda r: d2)
-            did = self.pdm.save_pending_deferred(d)
-            d.callback('foo')
-            d3 = self.pdm.get_pending_deferred(did,False)
-            d3.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
-            d2.callback('bar')
-            d3 = self.pdm.get_pending_deferred(did,False)
-            d3.addCallback(lambda r: self.assertEquals(r,'bar'))
+    def test_noresult_noblock(self):
+        d = defer.Deferred()
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,False)
+        d2.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
+
+    def test_with_callbacks(self):
+        d = defer.Deferred()
+        d.addCallback(lambda r: r+' foo')
+        d.addCallback(lambda r: r+' bar')
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d.callback('bam')
+        d2.addCallback(lambda r: self.assertEquals(r,'bam foo bar'))
+
+    def test_with_errbacks(self):
+        class MyError(Exception):
+            pass
+        d = defer.Deferred()
+        d.addCallback(lambda r: 'foo')
+        d.addErrback(lambda f: 'caught error')
+        did = self.pdm.save_pending_deferred(d)
+        d2 = self.pdm.get_pending_deferred(did,True)
+        d.errback(failure.Failure(MyError('bam')))
+        d2.addErrback(lambda f: self.assertRaises(MyError, f.raiseException))
+
+    def test_nested_deferreds(self):
+        d = defer.Deferred()
+        d2 = defer.Deferred()
+        d.addCallback(lambda r: d2)
+        did = self.pdm.save_pending_deferred(d)
+        d.callback('foo')
+        d3 = self.pdm.get_pending_deferred(did,False)
+        d3.addErrback(lambda f: self.assertRaises(error.ResultNotCompleted, f.raiseException))
+        d2.callback('bar')
+        d3 = self.pdm.get_pending_deferred(did,False)
+        d3.addCallback(lambda r: self.assertEquals(r,'bar'))
 

IPython/kernel/tests/test_task.py

@@ -26,25 +26,26 b' try:'
     from IPython.testing.util import DeferredTestCase
     from IPython.kernel.tests.tasktest import ITaskControllerTestCase
 except ImportError:
-    pass
-else:
-    #-------------------------------------------------------------------------------
-    # Tests
-    #-------------------------------------------------------------------------------
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
 
-    class BasicTaskControllerTestCase(DeferredTestCase, ITaskControllerTestCase):
+#-------------------------------------------------------------------------------
+# Tests
+#-------------------------------------------------------------------------------
+
+class BasicTaskControllerTestCase(DeferredTestCase, ITaskControllerTestCase):
+
+    def setUp(self):
+        self.controller  = cs.ControllerService()
+        self.controller.startService()
+        self.multiengine = IMultiEngine(self.controller)
+        self.tc = task.ITaskController(self.controller)
+        self.tc.failurePenalty = 0
+        self.engines=[]
     
-        def setUp(self):
-            self.controller  = cs.ControllerService()
-            self.controller.startService()
-            self.multiengine = IMultiEngine(self.controller)
-            self.tc = task.ITaskController(self.controller)
-            self.tc.failurePenalty = 0
-            self.engines=[]
-        
-        def tearDown(self):
-            self.controller.stopService()
-            for e in self.engines:
-                e.stopService()
+    def tearDown(self):
+        self.controller.stopService()
+        for e in self.engines:
+            e.stopService()
 
 

IPython/kernel/tests/test_taskfc.py

@@ -33,129 +33,130 b' try:'
     from IPython.kernel.error import CompositeError
     from IPython.kernel.parallelfunction import ParallelFunction
 except ImportError:
-    pass
-else:
+    import nose
+    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
 
-    #-------------------------------------------------------------------------------
-    # Tests
-    #-------------------------------------------------------------------------------
 
-    def _raise_it(f):
-        try:
-            f.raiseException()
-        except CompositeError, e:
-            e.raise_exception()
+#-------------------------------------------------------------------------------
+# Tests
+#-------------------------------------------------------------------------------
 
-    class TaskTest(DeferredTestCase, ITaskControllerTestCase):
+def _raise_it(f):
+    try:
+        f.raiseException()
+    except CompositeError, e:
+        e.raise_exception()
 
-        def setUp(self):
-        
-            self.engines = []
-                
-            self.controller = cs.ControllerService()
-            self.controller.startService()
-            self.imultiengine = me.IMultiEngine(self.controller)
-            self.itc = taskmodule.ITaskController(self.controller)
-            self.itc.failurePenalty = 0
-        
-            self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
-            self.tc_referenceable = IFCTaskController(self.itc)
-        
-            self.controller_tub = Tub()
-            self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
-            self.controller_tub.setLocation('127.0.0.1:10105')
-        
-            mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
-            tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
-            self.controller_tub.startService()
-        
-            self.client_tub = ClientConnector()
-            d = self.client_tub.get_multiengine_client(mec_furl)
-            d.addCallback(self.handle_mec_client)
-            d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
-            d.addCallback(self.handle_tc_client)
-            return d
-        
-        def handle_mec_client(self, client):
-            self.multiengine = client
+class TaskTest(DeferredTestCase, ITaskControllerTestCase):
+
+    def setUp(self):
+    
+        self.engines = []
+            
+        self.controller = cs.ControllerService()
+        self.controller.startService()
+        self.imultiengine = me.IMultiEngine(self.controller)
+        self.itc = taskmodule.ITaskController(self.controller)
+        self.itc.failurePenalty = 0
+    
+        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
+        self.tc_referenceable = IFCTaskController(self.itc)
+    
+        self.controller_tub = Tub()
+        self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
+        self.controller_tub.setLocation('127.0.0.1:10105')
+    
+        mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
+        tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
+        self.controller_tub.startService()
+    
+        self.client_tub = ClientConnector()
+        d = self.client_tub.get_multiengine_client(mec_furl)
+        d.addCallback(self.handle_mec_client)
+        d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
+        d.addCallback(self.handle_tc_client)
+        return d
+    
+    def handle_mec_client(self, client):
+        self.multiengine = client
+
+    def handle_tc_client(self, client):
+        self.tc = client
+
+    def tearDown(self):
+        dlist = []
+        # Shut down the multiengine client
+        d = self.client_tub.tub.stopService()
+        dlist.append(d)
+        # Shut down the engines
+        for e in self.engines:
+            e.stopService()
+        # Shut down the controller
+        d = self.controller_tub.stopService()
+        d.addBoth(lambda _: self.controller.stopService())
+        dlist.append(d)
+        return defer.DeferredList(dlist)
+    
+    def test_mapper(self):
+        self.addEngine(1)
+        m = self.tc.mapper()
+        self.assertEquals(m.task_controller,self.tc)
+        self.assertEquals(m.clear_before,False)
+        self.assertEquals(m.clear_after,False)
+        self.assertEquals(m.retries,0)
+        self.assertEquals(m.recovery_task,None)
+        self.assertEquals(m.depend,None)
+        self.assertEquals(m.block,True)
+    
+    def test_map_default(self):
+        self.addEngine(1)
+        m = self.tc.mapper()
+        d = m.map(lambda x: 2*x, range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
+    
+    def test_map_noblock(self):
+        self.addEngine(1)
+        m = self.tc.mapper(block=False)
+        d = m.map(lambda x: 2*x, range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
+        return d
+            
+    def test_mapper_fail(self):
+        self.addEngine(1)
+        m = self.tc.mapper()
+        d = m.map(lambda x: 1/0, range(10))
+        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
+        return d
+    
+    def test_parallel(self):
+        self.addEngine(1)
+        p = self.tc.parallel()
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 2*x
+        d = f(range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
+        return d
     
-        def handle_tc_client(self, client):
-            self.tc = client
+    def test_parallel_noblock(self):
+        self.addEngine(1)
+        p = self.tc.parallel(block=False)
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 2*x
+        d = f(range(10))
+        d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
+        return d
     
-        def tearDown(self):
-            dlist = []
-            # Shut down the multiengine client
-            d = self.client_tub.tub.stopService()
-            dlist.append(d)
-            # Shut down the engines
-            for e in self.engines:
-                e.stopService()
-            # Shut down the controller
-            d = self.controller_tub.stopService()
-            d.addBoth(lambda _: self.controller.stopService())
-            dlist.append(d)
-            return defer.DeferredList(dlist)
-        
-        def test_mapper(self):
-            self.addEngine(1)
-            m = self.tc.mapper()
-            self.assertEquals(m.task_controller,self.tc)
-            self.assertEquals(m.clear_before,False)
-            self.assertEquals(m.clear_after,False)
-            self.assertEquals(m.retries,0)
-            self.assertEquals(m.recovery_task,None)
-            self.assertEquals(m.depend,None)
-            self.assertEquals(m.block,True)
-        
-        def test_map_default(self):
-            self.addEngine(1)
-            m = self.tc.mapper()
-            d = m.map(lambda x: 2*x, range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-        
-        def test_map_noblock(self):
-            self.addEngine(1)
-            m = self.tc.mapper(block=False)
-            d = m.map(lambda x: 2*x, range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
-            return d
-                
-        def test_mapper_fail(self):
-            self.addEngine(1)
-            m = self.tc.mapper()
-            d = m.map(lambda x: 1/0, range(10))
-            d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-            return d
-        
-        def test_parallel(self):
-            self.addEngine(1)
-            p = self.tc.parallel()
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 2*x
-            d = f(range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
-            return d
-        
-        def test_parallel_noblock(self):
-            self.addEngine(1)
-            p = self.tc.parallel(block=False)
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 2*x
-            d = f(range(10))
-            d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
-            return d
-        
-        def test_parallel_fail(self):
-            self.addEngine(1)
-            p = self.tc.parallel()
-            self.assert_(isinstance(p, ParallelFunction))
-            @p
-            def f(x): return 1/0
-            d = f(range(10))
-            d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-            return d
  No newline at end of file
+    def test_parallel_fail(self):
+        self.addEngine(1)
+        p = self.tc.parallel()
+        self.assert_(isinstance(p, ParallelFunction))
+        @p
+        def f(x): return 1/0
+        d = f(range(10))
+        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
+        return d
  No newline at end of file

IPython/testing/decorators.py

@@ -8,6 +8,10 b' the decorator, in order to preserve metadata such as function name,'
 setup and teardown functions and so on - see nose.tools for more
 information.
 
+This module provides a set of useful decorators meant to be ready to use in
+your own tests.  See the bottom of the file for the ready-made ones, and if you
+find yourself writing a new one that may be of generic use, add it here.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
@@ -15,6 +19,7 b' code will be added at the bottom if we end up extending this.'
 
 # Stdlib imports
 import inspect
+import sys
 
 # Third-party imports
 
@@ -120,16 +125,36 b" skip_doctest = make_label_dec('skip_doctest',"
     omit from testing, while preserving the docstring for introspection, help,
     etc.""")                              
 
+def skip(msg=''):
+    """Decorator - mark a test function for skipping from test suite.
+
+    This function *is* already a decorator, it is not a factory like
+    make_label_dec or some of those in decorators_numpy.
+
+    :Parameters:
+
+      func : function
+        Test function to be skipped
 
-def skip(func):
-    """Decorator - mark a test function for skipping from test suite."""
+      msg : string
+        Optional message to be added.
+      """
 
     import nose
-    
-    def wrapper(*a,**k):
-        raise nose.SkipTest("Skipping test for function: %s" %
-                            func.__name__)
-    
-    return apply_wrapper(wrapper,func)
 
+    def inner(func):
+
+        def wrapper(*a,**k):
+            if msg: out = '\n'+msg
+            else: out = ''
+            raise nose.SkipTest("Skipping test for function: %s%s" %
+                                (func.__name__,out))
+
+        return apply_wrapper(wrapper,func)
+
+    return inner
 
+# Decorators to skip certain tests on specific platforms.
+skip_win32 = skipif(sys.platform=='win32',"This test does not run under Windows")
+skip_linux = skipif(sys.platform=='linux2',"This test does not run under Linux")
+skip_osx = skipif(sys.platform=='darwin',"This test does not run under OSX")

IPython/testing/plugin/Makefile

@@ -1,6 +1,5 b''
 # Set this prefix to where you want to install the plugin
-PREFIX=~/usr/local
-PREFIX=~/tmp/local
+PREFIX=/usr/local
 
 NOSE0=nosetests -vs --with-doctest --doctest-tests --detailed-errors
 NOSE=nosetests -vvs --with-ipdoctest --doctest-tests --doctest-extension=txt \

IPython/testing/plugin/ipdoctest.py

@@ -658,6 +658,24 b' class ExtensionDoctest(doctests.Doctest):'
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
+        parser.add_option('--doctest-tests', action='store_true',
+                          dest='doctest_tests',
+                          default=env.get('NOSE_DOCTEST_TESTS',True),
+                          help="Also look for doctests in test modules. "
+                          "Note that classes, methods and functions should "
+                          "have either doctests or non-doctest tests, "
+                          "not both. [NOSE_DOCTEST_TESTS]")
+        parser.add_option('--doctest-extension', action="append",
+                          dest="doctestExtension",
+                          help="Also look for doctests in files with "
+                          "this extension [NOSE_DOCTEST_EXTENSION]")
+        # Set the default as a list, if given in env; otherwise
+        # an additional value set on the command line will cause
+        # an error.
+        env_setting = env.get('NOSE_DOCTEST_EXTENSION')
+        if env_setting is not None:
+            parser.set_defaults(doctestExtension=tolist(env_setting))
+
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
@@ -743,16 +761,19 b' class ExtensionDoctest(doctests.Doctest):'
         Modified version that accepts extension modules as valid containers for
         doctests.
         """
-        #print 'Filename:',filename  # dbg
+        print 'Filename:',filename  # dbg
 
         # XXX - temporarily hardcoded list, will move to driver later
         exclude = ['IPython/external/',
-                   'IPython/Extensions/ipy_',
                    'IPython/platutils_win32',
                    'IPython/frontend/cocoa',
                    'IPython_doctest_plugin',
                    'IPython/Gnuplot',
-                   'IPython/Extensions/PhysicalQIn']
+                   'IPython/Extensions/ipy_',
+                   'IPython/Extensions/PhysicalQIn',
+                   'IPython/Extensions/scitedirector',
+                   'IPython/testing/plugin',
+                   ]
 
         for fex in exclude:
             if fex in filename:  # substring
@@ -782,3 +803,4 b' class IPythonDoctest(ExtensionDoctest):'
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
+

IPython/testing/plugin/test_refs.py

@@ -1,147 +1,19 b''
+"""Some simple tests for the plugin while running scripts.
+"""
 # Module imports
 # Std lib
 import inspect
 
-# Third party
-
 # Our own
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
-# Utilities
-
-# Note: copied from OInspect, kept here so the testing stuff doesn't create
-# circular dependencies and is easier to reuse.
-def getargspec(obj):
-    """Get the names and default values of a function's arguments.
-
-    A tuple of four things is returned: (args, varargs, varkw, defaults).
-    'args' is a list of the argument names (it may contain nested lists).
-    'varargs' and 'varkw' are the names of the * and ** arguments or None.
-    'defaults' is an n-tuple of the default values of the last n arguments.
-
-    Modified version of inspect.getargspec from the Python Standard
-    Library."""
-
-    if inspect.isfunction(obj):
-        func_obj = obj
-    elif inspect.ismethod(obj):
-        func_obj = obj.im_func
-    else:
-        raise TypeError, 'arg is not a Python function'
-    args, varargs, varkw = inspect.getargs(func_obj.func_code)
-    return args, varargs, varkw, func_obj.func_defaults
-
-#-----------------------------------------------------------------------------
 # Testing functions
 
 def test_trivial():
     """A trivial passing test."""
     pass
 
-
[e-mail]
-def test_deliberately_broken():
-    """A deliberately broken test - we want to skip this one."""
-    1/0
-
-
-# Verify that we can correctly skip the doctest for a function at will, but
-# that the docstring itself is NOT destroyed by the decorator.
[e-mail]_doctest
-def doctest_bad(x,y=1,**k):
-    """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    print 'x:',x
-    print 'y:',y
-    print 'k:',k
-
-
-def call_doctest_bad():
-    """Check that we can still call the decorated functions.
-    
-    >>> doctest_bad(3,y=4)
-    x: 3
-    y: 4
-    k: {}
-    """
-    pass
-
-
-# Doctest skipping should work for class methods too
-class foo(object):
-    """Foo
-
-    Example:
-
-    >>> 1+1
-    2
-    """
-
-    @dec.skip_doctest
-    def __init__(self,x):
-        """Make a foo.
-
-        Example:
-
-        >>> f = foo(3)
-        junk
-        """
-        print 'Making a foo.'
-        self.x = x
-        
-    @dec.skip_doctest
-    def bar(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        >>> f.bar(0)
-        boom!
-        >>> 1/0
-        bam!
-        """
-        return 1/y
-
-    def baz(self,y):
-        """Example:
-
-        >>> f = foo(3)
-        Making a foo.
-        >>> f.baz(3)
-        True
-        """
-        return self.x==y
-
-
-def test_skip_dt_decorator():
-    """Doctest-skipping decorator should preserve the docstring.
-    """
-    # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
-    check = """A function whose doctest we need to skip.
-
-    >>> 1+1
-    3
-    """
-    # Fetch the docstring from doctest_bad after decoration.
-    val = doctest_bad.__doc__
-    
-    assert check==val,"doctest_bad docstrings don't match"
-
-
-def test_skip_dt_decorator2():
-    """Doctest-skipping decorator should preserve function signature.
-    """
-    # Hardcoded correct answer
-    dtargs = (['x', 'y'], None, 'k', (1,))
-    # Introspect out the value
-    dtargsr = getargspec(doctest_bad)
-    assert dtargsr==dtargs, \
-           "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
-
-
 def doctest_run():
     """Test running a trivial script.
 
@@ -149,7 +21,6 b' def doctest_run():'
     x is: 1
     """
     
-#@dec.skip_doctest
 def doctest_runvars():
     """Test that variables defined in scripts get loaded correcly via %run.
 

IPython/testing/tutils.py

@@ -9,6 +9,7 b' Utilities for testing code.'
 # testing machinery from snakeoil that were good have already been merged into
 # the nose plugin, so this can be taken away soon.  Leave a warning for now,
 # we'll remove it in a later release (around 0.10 or so).
+
 from warnings import warn
 warn('This will be removed soon.  Use IPython.testing.util instead',
      DeprecationWarning)

IPython/ultraTB.py

@@ -445,7 +445,8 b' class ListTB(TBTools):'
         
         Also lifted nearly verbatim from traceback.py
         """
-        
+
+        have_filedata = False
         Colors = self.Colors
         list = []
         try:
@@ -459,8 +460,9 b' class ListTB(TBTools):'
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
-                    pass
+                    have_filedata = False
                 else:
+                    have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
@@ -492,7 +494,8 b' class ListTB(TBTools):'
                 list.append('%s\n' % str(stype))
 
         # vds:>>
-        __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        if have_filedata:
+            __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds:<<
 
         return list
@@ -809,10 +812,10 b' class VerboseTB(TBTools):'
 
         # vds: >>
         if records:
-             frame, file, lnum, func, lines, index = records[-1]
-             #print "file:", str(file), "linenb", str(lnum)         
-             file = abspath(file)
-             __IPYTHON__.hooks.synchronize_with_editor(file, lnum, 0)
+             filepath, lnum = records[-1][1:3]
+             #print "file:", str(file), "linenb", str(lnum) # dbg
+             filepath = os.path.abspath(filepath)
+             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
         # vds: <<
                 
         # return all our info assembled as a single string

MANIFEST.in

@@ -1,7 +1,6 b''
-include README_Windows.txt
-include win32_manual_post_install.py
 include ipython.py
 include setupbase.py
+include setupegg.py
 
 graft scripts
 
@@ -30,3 +29,4 b' global-exclude *.pyc'
 global-exclude .dircopy.log
 global-exclude .svn
 global-exclude .bzr
+global-exclude .hgignore

README.txt

@@ -1,11 +1,11 b''
-===============
-IPython1 README
-===============
- 
-.. contents::
+==============
+IPython README
+==============
 
 Overview
 ========
 
-Welcome to IPython.  New users should consult our documentation, which can be found
-in the docs/source subdirectory.
+Welcome to IPython. Our documentation can be found in the docs/source
+subdirectory. We also have ``.html`` and ``.pdf`` versions of this
+documentation available on the IPython `website <http://ipython.scipy.org>`_.
+

docs/Makefile

@@ -28,17 +28,16 b' help:'
 	@echo "dist         all, and then puts the results in dist/"
 
 clean:
-	-rm -rf build/*
+	-rm -rf build/* dist/*
 
 pdf: latex
 	cd build/latex && make all-pdf
 
 all: html pdf
 
-dist: all
+dist: clean all
 	mkdir -p dist
-	-rm -rf dist/*
-	ln build/latex/IPython.pdf dist/
+	ln build/latex/ipython.pdf dist/
 	cp -al build/html dist/
 	@echo "Build finished.  Final docs are in dist/"
 

docs/examples/kernel/mcpricer.py

@@ -40,3 +40,4 b' def main():'
 
 if __name__ == '__main__':
     main()
+

docs/examples/kernel/nwmerge.py

@@ -45,7 +45,10 b' def mergesort(list_of_lists, key=None):'
     for i, itr in enumerate(iter(pl) for pl in list_of_lists):
         try:
             item = itr.next()
-            toadd = (key(item), i, item, itr) if key else (item, i, itr)
+            if key:
+                toadd = (key(item), i, item, itr)
+            else:
+                toadd = (item, i, itr)
             heap.append(toadd)
         except StopIteration:
             pass

docs/examples/kernel/rmt.ipy

@@ -53,5 +53,5 b" if __name__ == '__main__':"
     %timeit -n1 -r1 parallelDiffs(rc, nmats, matsize)
 
     # Uncomment these to plot the histogram
-    import pylab
-    pylab.hist(parallelDiffs(rc,matsize,matsize))
+    # import pylab
+    # pylab.hist(parallelDiffs(rc,matsize,matsize))

docs/source/changes.txt

@@ -5,6 +5,89 b" What's new"
 ==========
 
 .. contents::
+..
+    1  Release 0.9.1
+    2  Release 0.9
+      2.1  New features
+      2.2  Bug fixes
+      2.3  Backwards incompatible changes
+      2.4  Changes merged in from IPython1
+        2.4.1  New features
+        2.4.2  Bug fixes
+        2.4.3  Backwards incompatible changes
+    3  Release 0.8.4
+    4  Release 0.8.3
+    5  Release 0.8.2
+    6  Older releases
+..
+
+Release dev
+===========
+
+New features
+------------
+
+* The wonderful TextMate editor can now be used with %edit on OS X.  Thanks
+  to Matt Foster for this patch.
+
+* Fully refactored :command:`ipcluster` command line program for starting
+  IPython clusters.  This new version is a complete rewrite and 1) is fully
+  cross platform (we now use Twisted's process management), 2) has much
+  improved performance, 3) uses subcommands for different types of clusters, 
+  4) uses argparse for parsing command line options, 5) has better support
+  for starting clusters using :command:`mpirun`, 6) has experimental support
+  for starting engines using PBS.  However, this new version of ipcluster
+  should be considered a technology preview.  We plan on changing the API
+  in significant ways before it is final.
+
+* The :mod:`argparse` module has been added to :mod:`IPython.external`.
+
+* Fully description of the security model added to the docs.
+
+* cd completer: show bookmarks if no other completions are available.
+
+* sh profile: easy way to give 'title' to prompt: assign to variable
+  '_prompt_title'. It looks like this::
+      
+        [~]|1> _prompt_title = 'sudo!'
+        sudo![~]|2>
+
+* %edit: If you do '%edit pasted_block', pasted_block
+  variable gets updated with new data (so repeated 
+  editing makes sense)
+
+Bug fixes
+---------
+
+* The ipengine and ipcontroller scripts now handle missing furl files
+  more gracefully by giving better error messages.
+
+* %rehashx: Aliases no longer contain dots. python3.0 binary
+  will create alias python30. Fixes:
+  #259716 "commands with dots in them don't work"
+
+* %cpaste: %cpaste -r repeats the last pasted block.
+  The block is assigned to pasted_block even if code
+  raises exception.
+
+Backwards incompatible changes
+------------------------------
+
+* The controller now has a ``-r`` flag that needs to be used if you want to
+  reuse existing furl files.  Otherwise they are deleted (the default).
+
+* Remove ipy_leo.py. "easy_install ipython-extension" to get it.
+  (done to decouple it from ipython release cycle)
+
+
+
+Release 0.9.1
+=============
+
+This release was quickly made to restore compatibility with Python 2.4, which
+version 0.9 accidentally broke.  No new features were introduced, other than
+some additional testing support for internal use.
+
 
 Release 0.9
 ===========
@@ -12,89 +95,170 b' Release 0.9'
 New features
 ------------
 
-	* The notion of a task has been completely reworked.  An `ITask` interface has
-	  been created.  This interface defines the methods that tasks need to implement.
-	  These methods are now responsible for things like submitting tasks and processing
-	  results.  There are two basic task types: :class:`IPython.kernel.task.StringTask`
-	  (this is the old `Task` object, but renamed) and the new 
-	  :class:`IPython.kernel.task.MapTask`, which is based on a function.
-	* A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to
-	  standardize the idea of a `map` method.  This interface has a single
-	  `map` method that has the same syntax as the built-in `map`.  We have also defined
-	  a `mapper` factory interface that creates objects that implement
-	  :class:`IPython.kernel.mapper.IMapper` for different controllers.  Both
-	  the multiengine and task controller now have mapping capabilties.
-	* The parallel function capabilities have been reworks.  The major changes are that
-	  i) there is now an `@parallel` magic that creates parallel functions, ii)
-	  the syntax for mulitple variable follows that of `map`, iii) both the
-	  multiengine and task controller now have a parallel function implementation.
-	* All of the parallel computing capabilities from `ipython1-dev` have been merged into
-	  IPython proper.  This resulted in the following new subpackages:  
-	  :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`,
-	  :mod:`IPython.tools` and :mod:`IPython.testing`.
-	* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and friends
-	  have been completely refactored.  Now we are checking for dependencies using
-	  the approach that matplotlib uses.
-	* The documentation has been completely reorganized to accept the documentation
-	  from `ipython1-dev`.
-	* We have switched to using Foolscap for all of our network protocols in
-	  :mod:`IPython.kernel`.  This gives us secure connections that are both encrypted
-	  and authenticated.
-	* We have a brand new `COPYING.txt` files that describes the IPython license
-	  and copyright.  The biggest change is that we are putting "The IPython
-	  Development Team" as the copyright holder.  We give more details about exactly
-	  what this means in this file.  All developer should read this and use the new
-	  banner in all IPython source code files.
-	* sh profile: ./foo runs foo as system command, no need to do !./foo anymore
-	* String lists now support 'sort(field, nums = True)' method (to easily
-	  sort system command output). Try it with 'a = !ls -l ; a.sort(1, nums=1)'
-	* '%cpaste foo' now assigns the pasted block as string list, instead of string
-	* The ipcluster script now run by default with no security.  This is done because
-	  the main usage of the script is for starting things on localhost.  Eventually
-	  when ipcluster is able to start things on other hosts, we will put security
-	  back.
-	* 'cd --foo' searches directory history for string foo, and jumps to that dir.
-	  Last part of dir name is checked first. If no matches for that are found,
-	  look at the whole path.
+* All furl files and security certificates are now put in a read-only
+  directory named ~./ipython/security.
+
+* A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that
+  determines the user's IPython directory in a robust manner.
+
+* Laurent's WX application has been given a top-level script called
+  ipython-wx, and it has received numerous fixes. We expect this code to be
+  architecturally better integrated with Gael's WX 'ipython widget' over the
+  next few releases.
+
+* The Editor synchronization work by Vivian De Smedt has been merged in.  This
+  code adds a number of new editor hooks to synchronize with editors under
+  Windows.
+
+* A new, still experimental but highly functional, WX shell by Gael Varoquaux.
+  This work was sponsored by Enthought, and while it's still very new, it is
+  based on a more cleanly organized arhictecture of the various IPython
+  components. We will continue to develop this over the next few releases as a
+  model for GUI components that use IPython.
+
+* Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework),
+  authored by Barry Wark.  Currently the WX and the Cocoa ones have slightly
+  different internal organizations, but the whole team is working on finding
+  what the right abstraction points are for a unified codebase.
+
+* As part of the frontend work, Barry Wark also implemented an experimental
+  event notification system that various ipython components can use.  In the
+  next release the implications and use patterns of this system regarding the
+  various GUI options will be worked out.
+
+* IPython finally has a full test system, that can test docstrings with
+  IPython-specific functionality.  There are still a few pieces missing for it
+  to be widely accessible to all users (so they can run the test suite at any
+  time and report problems), but it now works for the developers.  We are
+  working hard on continuing to improve it, as this was probably IPython's
+  major Achilles heel (the lack of proper test coverage made it effectively
+  impossible to do large-scale refactoring).  The full test suite can now
+  be run using the :command:`iptest` command line program.
+
+* The notion of a task has been completely reworked.  An `ITask` interface has
+  been created.  This interface defines the methods that tasks need to
+  implement.  These methods are now responsible for things like submitting
+  tasks and processing results.  There are two basic task types:
+  :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but
+  renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on
+  a function.
+
+* A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to
+  standardize the idea of a `map` method.  This interface has a single `map`
+  method that has the same syntax as the built-in `map`.  We have also defined
+  a `mapper` factory interface that creates objects that implement
+  :class:`IPython.kernel.mapper.IMapper` for different controllers.  Both the
+  multiengine and task controller now have mapping capabilties.
+
+* The parallel function capabilities have been reworks.  The major changes are
+  that i) there is now an `@parallel` magic that creates parallel functions,
+  ii) the syntax for mulitple variable follows that of `map`, iii) both the
+  multiengine and task controller now have a parallel function implementation.
 
+* All of the parallel computing capabilities from `ipython1-dev` have been
+  merged into IPython proper.  This resulted in the following new subpackages:
+  :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`,
+  :mod:`IPython.tools` and :mod:`IPython.testing`.
+
+* As part of merging in the `ipython1-dev` stuff, the `setup.py` script and
+  friends have been completely refactored.  Now we are checking for
+  dependencies using the approach that matplotlib uses.
+
+* The documentation has been completely reorganized to accept the 
+  documentation from `ipython1-dev`.
+
+* We have switched to using Foolscap for all of our network protocols in
+  :mod:`IPython.kernel`.  This gives us secure connections that are both
+  encrypted and authenticated.
+
+* We have a brand new `COPYING.txt` files that describes the IPython license
+  and copyright. The biggest change is that we are putting "The IPython
+  Development Team" as the copyright holder. We give more details about
+  exactly what this means in this file. All developer should read this and use
+  the new banner in all IPython source code files.
+
+* sh profile: ./foo runs foo as system command, no need to do !./foo anymore
+
+* String lists now support ``sort(field, nums = True)`` method (to easily sort
+  system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``.
+
+* '%cpaste foo' now assigns the pasted block as string list, instead of string
+
+* The ipcluster script now run by default with no security.  This is done
+  because the main usage of the script is for starting things on localhost.
+  Eventually when ipcluster is able to start things on other hosts, we will put
+  security back.
+
+* 'cd --foo' searches directory history for string foo, and jumps to that dir.
+  Last part of dir name is checked first. If no matches for that are found,
+  look at the whole path.
+
+  
 Bug fixes
 ---------
 
-	* The colors escapes in the multiengine client are now turned off on win32 as they
-	  don't print correctly.
-	* The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing mpi_import_statement
-	  incorrectly, which was leading the engine to crash when mpi was enabled.
-	* A few subpackages has missing `__init__.py` files.
-	* The documentation is only created is Sphinx is found.  Previously, the `setup.py`
-	  script would fail if it was missing.
-	* Greedy 'cd' completion has been disabled again (it was enabled in 0.8.4)
+* The Windows installer has been fixed.  Now all IPython scripts have ``.bat``
+  versions created.  Also, the Start Menu shortcuts have been updated.
+
+* The colors escapes in the multiengine client are now turned off on win32 as
+  they don't print correctly.
+
+* The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing
+  mpi_import_statement incorrectly, which was leading the engine to crash when
+  mpi was enabled.
+
+* A few subpackages had missing ``__init__.py`` files.
+
+* The documentation is only created if Sphinx is found.  Previously, the
+  ``setup.py`` script would fail if it was missing.
+
+* Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as
+  it caused problems on certain platforms.
           
 
 Backwards incompatible changes
 ------------------------------
 
-	* :class:`IPython.kernel.client.Task` has been renamed
-	  :class:`IPython.kernel.client.StringTask` to make way for new task types.
-	* The keyword argument `style` has been renamed `dist` in `scatter`, `gather`
-	  and `map`.
-	* Renamed the values that the rename `dist` keyword argument can have from
-	  `'basic'` to `'b'`.
-	* IPython has a larger set of dependencies if you want all of its capabilities.
-	  See the `setup.py` script for details.
-	* The constructors for :class:`IPython.kernel.client.MultiEngineClient` and 
-	  :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple.
-	  Instead they take the filename of a file that contains the FURL for that
-	  client.  If the FURL file is in your IPYTHONDIR, it will be found automatically
-	  and the constructor can be left empty.
-	* The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created 
-	  using the factory functions :func:`get_multiengine_client` and 
-	  :func:`get_task_client`.  These return a `Deferred` to the actual client.
-	* The command line options to `ipcontroller` and `ipengine` have changed to
-	  reflect the new Foolscap network protocol and the FURL files.  Please see the
-	  help for these scripts for details.
-	* The configuration files for the kernel have changed because of the Foolscap stuff.
-	  If you were using custom config files before, you should delete them and regenerate
-	  new ones.
+* The ``clusterfile`` options of the :command:`ipcluster` command has been
+  removed as it was not working and it will be replaced soon by something much
+  more robust.
+
+* The :mod:`IPython.kernel` configuration now properly find the user's
+  IPython directory.
+
+* In ipapi, the :func:`make_user_ns` function has been replaced with
+  :func:`make_user_namespaces`, to support dict subclasses in namespace
+  creation.
+
+* :class:`IPython.kernel.client.Task` has been renamed
+  :class:`IPython.kernel.client.StringTask` to make way for new task types.
+
+* The keyword argument `style` has been renamed `dist` in `scatter`, `gather`
+  and `map`.
+
+* Renamed the values that the rename `dist` keyword argument can have from
+  `'basic'` to `'b'`.
+
+* IPython has a larger set of dependencies if you want all of its capabilities.
+  See the `setup.py` script for details.
+
+* The constructors for :class:`IPython.kernel.client.MultiEngineClient` and 
+  :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple.
+  Instead they take the filename of a file that contains the FURL for that
+  client.  If the FURL file is in your IPYTHONDIR, it will be found automatically
+  and the constructor can be left empty.
+
+* The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created 
+  using the factory functions :func:`get_multiengine_client` and 
+  :func:`get_task_client`.  These return a `Deferred` to the actual client.
+
+* The command line options to `ipcontroller` and `ipengine` have changed to
+  reflect the new Foolscap network protocol and the FURL files.  Please see the
+  help for these scripts for details.
+
+* The configuration files for the kernel have changed because of the Foolscap
+  stuff.  If you were using custom config files before, you should delete them
+  and regenerate new ones.
 
 Changes merged in from IPython1
 -------------------------------
@@ -102,89 +266,109 b' Changes merged in from IPython1'
 New features
 ............
 
-	* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted
-	  and zope.interface are now easy installable, we can declare them as dependencies
-	  in our setupegg.py script.
-	* IPython is now compatible with Twisted 2.5.0 and 8.x.
-	* Added a new example of how to use :mod:`ipython1.kernel.asynclient`.
-	* Initial draft of a process daemon in :mod:`ipython1.daemon`.  This has not
-	  been merged into IPython and is still in `ipython1-dev`.
-	* The ``TaskController`` now has methods for getting the queue status.
-	* The ``TaskResult`` objects not have information about how long the task
-	  took to run.
-	* We are attaching additional attributes to exceptions ``(_ipython_*)`` that 
-	  we use to carry additional info around.
-	* New top-level module :mod:`asyncclient` that has asynchronous versions (that
-	  return deferreds) of the client classes.  This is designed to users who want
-	  to run their own Twisted reactor
-	* All the clients in :mod:`client` are now based on Twisted.  This is done by 
-	  running the Twisted reactor in a separate thread and using the
-	  :func:`blockingCallFromThread` function that is in recent versions of Twisted.
-	* Functions can now be pushed/pulled to/from engines using
-	  :meth:`MultiEngineClient.push_function` and :meth:`MultiEngineClient.pull_function`.
-	* Gather/scatter are now implemented in the client to reduce the work load
-	  of the controller and improve performance.
-	* Complete rewrite of the IPython docuementation.  All of the documentation
-	  from the IPython website has been moved into docs/source as restructured
-	  text documents.  PDF and HTML documentation are being generated using 
-	  Sphinx.
-	* New developer oriented documentation: development guidelines and roadmap. 
-	* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` file
-	  that is organized by release and is meant to provide something more relevant
-	  for users.
+* Much improved ``setup.py`` and ``setupegg.py`` scripts.  Because Twisted and
+  zope.interface are now easy installable, we can declare them as dependencies
+  in our setupegg.py script.
+
+* IPython is now compatible with Twisted 2.5.0 and 8.x.
+
+* Added a new example of how to use :mod:`ipython1.kernel.asynclient`.
+
+* Initial draft of a process daemon in :mod:`ipython1.daemon`.  This has not
+  been merged into IPython and is still in `ipython1-dev`.
+  
+* The ``TaskController`` now has methods for getting the queue status.
+
+* The ``TaskResult`` objects not have information about how long the task
+  took to run.
+  
+* We are attaching additional attributes to exceptions ``(_ipython_*)`` that 
+  we use to carry additional info around.
+  
+* New top-level module :mod:`asyncclient` that has asynchronous versions (that
+  return deferreds) of the client classes.  This is designed to users who want
+  to run their own Twisted reactor.
+  
+* All the clients in :mod:`client` are now based on Twisted.  This is done by 
+  running the Twisted reactor in a separate thread and using the
+  :func:`blockingCallFromThread` function that is in recent versions of Twisted.
+
+* Functions can now be pushed/pulled to/from engines using
+  :meth:`MultiEngineClient.push_function` and
+  :meth:`MultiEngineClient.pull_function`.
+
+* Gather/scatter are now implemented in the client to reduce the work load
+  of the controller and improve performance.
+
+* Complete rewrite of the IPython docuementation.  All of the documentation
+  from the IPython website has been moved into docs/source as restructured
+  text documents.  PDF and HTML documentation are being generated using 
+  Sphinx.
+
+* New developer oriented documentation: development guidelines and roadmap. 
+
+* Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt``
+  file that is organized by release and is meant to provide something more
+  relevant for users.
 
 Bug fixes
 .........
 
-	* Created a proper ``MANIFEST.in`` file to create source distributions.
-	* Fixed a bug in the ``MultiEngine`` interface.  Previously, multi-engine 
-	  actions were being collected with a :class:`DeferredList` with 
-	  ``fireononeerrback=1``.  This meant that methods were returning 
-	  before all engines had given their results.  This was causing extremely odd 
-	  bugs in certain cases. To fix this problem, we have 1) set 
-	  ``fireononeerrback=0`` to make sure all results (or exceptions) are in 
-	  before returning and 2) introduced a :exc:`CompositeError` exception 
-	  that wraps all of the engine exceptions.  This is a huge change as it means 
-	  that users will have to catch :exc:`CompositeError` rather than the actual
-	  exception.
+* Created a proper ``MANIFEST.in`` file to create source distributions.
+
+* Fixed a bug in the ``MultiEngine`` interface.  Previously, multi-engine 
+  actions were being collected with a :class:`DeferredList` with 
+  ``fireononeerrback=1``.  This meant that methods were returning 
+  before all engines had given their results.  This was causing extremely odd 
+  bugs in certain cases. To fix this problem, we have 1) set 
+  ``fireononeerrback=0`` to make sure all results (or exceptions) are in 
+  before returning and 2) introduced a :exc:`CompositeError` exception 
+  that wraps all of the engine exceptions.  This is a huge change as it means 
+  that users will have to catch :exc:`CompositeError` rather than the actual
+  exception.
 
 Backwards incompatible changes
 ..............................
 
-	* All names have been renamed to conform to the lowercase_with_underscore
-	  convention.  This will require users to change references to all names like
-	  ``queueStatus`` to ``queue_status``.
-	* Previously, methods like :meth:`MultiEngineClient.push` and    	
-	  :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``.  This was 
-	  becoming a problem as we weren't able to introduce new keyword arguments into
-	  the API.  Now these methods simple take a dict or sequence.  This has also allowed
-	  us to get rid of the ``*All`` methods like :meth:`pushAll` and :meth:`pullAll`.
-	  These things are now handled with the ``targets`` keyword argument that defaults
-	  to ``'all'``.
-	* The :attr:`MultiEngineClient.magicTargets` has been renamed to
-	  :attr:`MultiEngineClient.targets`. 
-	* All methods in the MultiEngine interface now accept the optional keyword argument
-	  ``block``.
-	* Renamed :class:`RemoteController` to :class:`MultiEngineClient` and 
-	  :class:`TaskController` to :class:`TaskClient`.
-	* Renamed the top-level module from :mod:`api` to :mod:`client`.
-	* Most methods in the multiengine interface now raise a :exc:`CompositeError` exception
-	  that wraps the user's exceptions, rather than just raising the raw user's exception.
-	* Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` 
-	  and ``pull``.
+* All names have been renamed to conform to the lowercase_with_underscore
+  convention.  This will require users to change references to all names like
+  ``queueStatus`` to ``queue_status``.
+
+* Previously, methods like :meth:`MultiEngineClient.push` and
+  :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``.  This was
+  becoming a problem as we weren't able to introduce new keyword arguments into
+  the API.  Now these methods simple take a dict or sequence.  This has also
+  allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and
+  :meth:`pullAll`.  These things are now handled with the ``targets`` keyword
+  argument that defaults to ``'all'``.
+
+* The :attr:`MultiEngineClient.magicTargets` has been renamed to
+  :attr:`MultiEngineClient.targets`. 
+
+* All methods in the MultiEngine interface now accept the optional keyword
+  argument ``block``.
+
+* Renamed :class:`RemoteController` to :class:`MultiEngineClient` and 
+  :class:`TaskController` to :class:`TaskClient`.
+
+* Renamed the top-level module from :mod:`api` to :mod:`client`.
 
+* Most methods in the multiengine interface now raise a :exc:`CompositeError`
+  exception that wraps the user's exceptions, rather than just raising the raw
+  user's exception.
+
+* Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` 
+  and ``pull``.
+
+  
 Release 0.8.4
 =============
 
-Someone needs to describe what went into 0.8.4.
+This was a quick release to fix an unfortunate bug that slipped into the 0.8.3
+release.  The ``--twisted`` option was disabled, as it turned out to be broken
+across several platforms.
 
-Release 0.8.2
-=============
 
-* %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
-  and jumps to /foo. The current behaviour is closer to the documented 
-  behaviour, and should not trip anyone.
-  
 Release 0.8.3
 =============
   
@@ -192,9 +376,18 b' Release 0.8.3'
   it by passing -pydb command line argument to IPython. Note that setting
   it in config file won't work.
 
+  
+Release 0.8.2
+=============
+
+* %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory 
+  and jumps to /foo. The current behaviour is closer to the documented 
+  behaviour, and should not trip anyone.
+
+  
 Older releases
 ==============
 
-Changes in earlier releases of IPython are described in the older file ``ChangeLog``.  
-Please refer to this document for details.
+Changes in earlier releases of IPython are described in the older file
+``ChangeLog``.  Please refer to this document for details.
 

docs/source/conf.py

@@ -1,7 +1,11 b''
 # -*- coding: utf-8 -*-
 #
-# IPython documentation build configuration file, created by
-# sphinx-quickstart on Thu May  8 16:45:02 2008.
+# IPython documentation build configuration file.
+
+# NOTE: This file has been edited manually from the auto-generated one from
+# sphinx.  Do NOT delete and re-generate.  If any changes from sphinx are
+# needed, generate a scratch one and merge by hand any new fields needed.
+
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
@@ -16,14 +20,26 b' import sys, os'
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-#sys.path.append(os.path.abspath('some/directory'))
+sys.path.append(os.path.abspath('../sphinxext'))
+
+# Import support for ipython console session syntax highlighting (lives
+# in the sphinxext directory defined above)
+import ipython_console_highlighting
+
+# We load the ipython release info into a dict by explicit execution
+iprelease = {}
+execfile('../../IPython/Release.py',iprelease)
 
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-#extensions = []
+extensions = ['sphinx.ext.autodoc',
+              'inheritance_diagram', 'only_directives',
+              'ipython_console_highlighting', 
+              # 'plot_directive', # disabled for now, needs matplotlib
+              ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -41,10 +57,11 b" copyright = '2008, The IPython Development Team'"
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
 #
-# The short X.Y version.
-version = '0.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.beta1'
+release = iprelease['version']
+# The short X.Y version.
+version = '.'.join(release.split('.',2)[:2])
+
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -57,7 +74,7 b" today_fmt = '%B %d, %Y'"
 
 # List of directories, relative to source directories, that shouldn't be searched
 # for source files.
-#exclude_dirs = []
+exclude_dirs = ['attic']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True
@@ -125,7 +142,7 b" html_last_updated_fmt = '%b %d, %Y'"
 #html_file_suffix = ''
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'IPythondoc'
+htmlhelp_basename = 'ipythondoc'
 
 
 # Options for LaTeX output
@@ -140,11 +157,8 b" latex_font_size = '11pt'"
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
-latex_documents = [ ('index', 'IPython.tex', 'IPython Documentation',
-                     ur"""Brian Granger, Fernando Pérez and Ville Vainio\\
-                     \ \\
-                     With contributions from:\\
-                     Benjamin Ragan-Kelley.""",
+latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
+                     ur"""The IPython Development Team""",
                      'manual'),
                     ]
 
@@ -164,3 +178,10 b" latex_documents = [ ('index', 'IPython.tex', 'IPython Documentation',"
 
 # If false, no module index is generated.
 #latex_use_modindex = True
+
+
+# Cleanup
+# -------
+# delete release info to avoid pickling errors from sphinx
+
+del iprelease

docs/source/config/customization.txt

@@ -18,6 +18,8 b' time. A hybrid approach of specifying a few options in ipythonrc and'
 doing the more advanced configuration in ipy_user_conf.py is also
 possible.
 
+.. _ipythonrc:
+
 The ipythonrc approach
 ======================
 
@@ -36,11 +38,11 b' fairly primitive). Note that these are not python files, and this is'
 deliberate, because it allows us to do some things which would be quite
 tricky to implement if they were normal python files.
 
-First, an rcfile can contain permanent default values for almost all
-command line options (except things like -help or -Version). Sec
-`command line options`_ contains a description of all command-line
-options. However, values you explicitly specify at the command line
-override the values defined in the rcfile.
+First, an rcfile can contain permanent default values for almost all command
+line options (except things like -help or -Version). :ref:`This section
+<command_line_options>` contains a description of all command-line
+options. However, values you explicitly specify at the command line override
+the values defined in the rcfile.
 
 Besides command line option values, the rcfile can specify values for
 certain extra special options which are not available at the command
@@ -266,13 +268,13 b' which look like this::'
 IPython profiles
 ================
 
-As we already mentioned, IPython supports the -profile command-line
-option (see sec. `command line options`_). A profile is nothing more
-than a particular configuration file like your basic ipythonrc one,
-but with particular customizations for a specific purpose. When you
-start IPython with 'ipython -profile <name>', it assumes that in your
-IPYTHONDIR there is a file called ipythonrc-<name> or
-ipy_profile_<name>.py, and loads it instead of the normal ipythonrc.
+As we already mentioned, IPython supports the -profile command-line option (see
+:ref:`here <command_line_options>`).  A profile is nothing more than a
+particular configuration file like your basic ipythonrc one, but with
+particular customizations for a specific purpose. When you start IPython with
+'ipython -profile <name>', it assumes that in your IPYTHONDIR there is a file
+called ipythonrc-<name> or ipy_profile_<name>.py, and loads it instead of the
+normal ipythonrc.
 
 This system allows you to maintain multiple configurations which load
 modules, set options, define functions, etc. suitable for different

docs/source/config/index.txt

@@ -3,7 +3,7 b' Configuration and customization'
 ===============================
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    initial_config.txt
    customization.txt

docs/source/config/initial_config.txt

@@ -11,28 +11,27 b' in a directory named by default $HOME/.ipython. You can change this by'
 defining the environment variable IPYTHONDIR, or at runtime with the
 command line option -ipythondir.
 
-If all goes well, the first time you run IPython it should
-automatically create a user copy of the config directory for you,
-based on its builtin defaults. You can look at the files it creates to
-learn more about configuring the system. The main file you will modify
-to configure IPython's behavior is called ipythonrc (with a .ini
-extension under Windows), included for reference in `ipythonrc`_
-section. This file is very commented and has many variables you can
-change to suit your taste, you can find more details in
-Sec. customization_. Here we discuss the basic things you will want to
-make sure things are working properly from the beginning.
+If all goes well, the first time you run IPython it should automatically create
+a user copy of the config directory for you, based on its builtin defaults. You
+can look at the files it creates to learn more about configuring the
+system. The main file you will modify to configure IPython's behavior is called
+ipythonrc (with a .ini extension under Windows), included for reference
+:ref:`here <ipythonrc>`. This file is very commented and has many variables you
+can change to suit your taste, you can find more details :ref:`here
+<customization>`. Here we discuss the basic things you will want to make sure
+things are working properly from the beginning.
 
 
-.. _Accessing help:
+.. _accessing_help:
 
 Access to the Python help system
 ================================
 
-This is true for Python in general (not just for IPython): you should
-have an environment variable called PYTHONDOCS pointing to the directory
-where your HTML Python documentation lives. In my system it's
-/usr/share/doc/python-docs-2.3.4/html, check your local details or ask
-your systems administrator.
+This is true for Python in general (not just for IPython): you should have an
+environment variable called PYTHONDOCS pointing to the directory where your
+HTML Python documentation lives. In my system it's
+:file:`/usr/share/doc/python-doc/html`, check your local details or ask your
+systems administrator.
 
 This is the directory which holds the HTML version of the Python
 manuals. Unfortunately it seems that different Linux distributions
@@ -40,8 +39,9 b' package these files differently, so you may have to look around a bit.'
 Below I show the contents of this directory on my system for reference::
 
     [html]> ls
-    about.dat acks.html dist/ ext/ index.html lib/ modindex.html
-    stdabout.dat tut/ about.html api/ doc/ icons/ inst/ mac/ ref/ style.css
+    about.html  dist/  icons/      lib/           python2.5.devhelp.gz  whatsnew/
+    acks.html   doc/   index.html  mac/           ref/
+    api/        ext/   inst/       modindex.html  tut/
 
 You should really make sure this variable is correctly set so that
 Python's pydoc-based help system works. It is a powerful and convenient
@@ -108,6 +108,8 b' The following terminals seem to handle the color sequences fine:'
       support under cygwin, please post to the IPython mailing list so
       this issue can be resolved for all users.
 
+.. _pyreadline: https://code.launchpad.net/pyreadline
+      
 These have shown problems:
 
     * Windows command prompt in WinXP/2k logged into a Linux machine via
@@ -157,13 +159,12 b' $HOME/.ipython/ipythonrc and set the colors option to the desired value.'
 Object details (types, docstrings, source code, etc.)
 =====================================================
 
-IPython has a set of special functions for studying the objects you
-are working with, discussed in detail in Sec. `dynamic object
-information`_. But this system relies on passing information which is
-longer than your screen through a data pager, such as the common Unix
-less and more programs. In order to be able to see this information in
-color, your pager needs to be properly configured. I strongly
-recommend using less instead of more, as it seems that more simply can
+IPython has a set of special functions for studying the objects you are working
+with, discussed in detail :ref:`here <dynamic_object_info>`. But this system
+relies on passing information which is longer than your screen through a data
+pager, such as the common Unix less and more programs. In order to be able to
+see this information in color, your pager needs to be properly configured. I
+strongly recommend using less instead of more, as it seems that more simply can
 not understand colored text correctly.
 
 In order to configure less as your default pager, do the following:

docs/source/credits.txt

@@ -4,24 +4,39 b''
 Credits
 =======
 
-IPython is mainly developed by Fernando Pérez
-<[e-mail]>, but the project was born from mixing in
-Fernando's code with the IPP project by Janko Hauser
-<jhauser-AT-zscout.de> and LazyPython by Nathan Gray
-<n8gray-AT-caltech.edu>. For all IPython-related requests, please
-contact Fernando.
-
-As of early 2006, the following developers have joined the core team:
-
-    * [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
-      Google Summer of Code project to develop python interactive
-      notebooks (XML documents) and graphical interface. This project
-      was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
-      Toni Alatalo <antont-AT-an.org>
-    * [Brian Granger] <bgranger-AT-scu.edu>: extending IPython to allow
-      support for interactive parallel computing.
-    * [Ville Vainio] <vivainio-AT-gmail.com>: Ville is the new
-      maintainer for the main trunk of IPython after version 0.7.1.
+IPython is led by Fernando Pérez.
+
+As of this writing, the following developers have joined the core team:
+
+* [Robert Kern] <rkern-AT-enthought.com>: co-mentored the 2005
+  Google Summer of Code project to develop python interactive
+  notebooks (XML documents) and graphical interface. This project
+  was awarded to the students Tzanko Matev <tsanko-AT-gmail.com> and
+  Toni Alatalo <antont-AT-an.org>.
+
+* [Brian Granger] <ellisonbg-AT-gmail.com>: extending IPython to allow
+  support for interactive parallel computing.
+
+* [Benjamin (Min) Ragan-Kelley]: key work on IPython's parallel
+  computing infrastructure.
+
+* [Ville Vainio] <vivainio-AT-gmail.com>: Ville has made many improvements
+  to the core of IPython and was the maintainer of the main IPython
+  trunk from version 0.7.1 to 0.8.4.
+
+* [Gael Varoquaux] <gael.varoquaux-AT-normalesup.org>: work on the merged
+  architecture for the interpreter as of version 0.9, implementing a new WX GUI
+  based on this system.
+
+* [Barry Wark] <barrywark-AT-gmail.com>: implementing a new Cocoa GUI, as well
+  as work on the new interpreter architecture and Twisted support.
+
+* [Laurent Dufrechou] <laurent.dufrechou-AT-gmail.com>: development of the WX
+  GUI support.
+
+* [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu>: maintainer of the
+  PyReadline project, necessary for IPython under windows.
+
 
 The IPython project is also very grateful to:
 
@@ -50,90 +65,143 b" an O'Reilly Python editor. His Oct/11/2001 article about IPP and"
 LazyPython, was what got this project started. You can read it at:
 http://www.onlamp.com/pub/a/python/2001/10/11/pythonnews.html.
 
-And last but not least, all the kind IPython users who have emailed new
-code, bug reports, fixes, comments and ideas. A brief list follows,
-please let me know if I have ommitted your name by accident:
-
-    * [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
-      color problem. This bug alone caused many lost hours and
-      frustration, many thanks to him for the fix. I've always been a
-      fan of Ogg & friends, now I have one more reason to like these folks.
-      Jack is also contributing with Debian packaging and many other
-      things.
-    * [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
-      reports, bug fixes, ideas, lots more. The ipython.el mode for
-      (X)Emacs is Alex's code, providing full support for IPython under
-      (X)Emacs.
-    * [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
-      information, Fink package management.
-    * [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
-      around the exception handling idiosyncracies of WxPython. Readline
-      and color support for Windows.
-    * [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
-      improved readline support, including fixes for Python 2.3.
-    * [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
-    * [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
-    * [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
-    * [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
-    * [Philip Hisley] <compsys-AT-starpower.net>
-    * [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
-      more.
-    * [Robin Siebler] <robinsiebler-AT-starband.net>
-    * [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
-    * [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
-    * [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
-    * [Syver Enstad] <syver-en-AT-online.no> Windows setup.
-    * [Richard] <rxe-AT-renre-europe.com> Global embedding.
-    * [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
-      compatibility.
-    * [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
-      installation.
-    * [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
-    * [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
-      documentation fixes.
-    * [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
-      ideas. Patches for Windows installer.
-    * [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
-    * [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
-    * [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
-      Win32/CygWin.
-    * [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
-      nice, lightweight string interpolation.
-    * [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
-    * [Gever Tulley] <gever-AT-helium.com> Code contributions.
-    * [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
-    * [Oliver Sander] <osander-AT-gmx.de> Bug reports.
-    * [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
-      logging module.
-    * [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
-      Fixes, enhancement suggestions for system shell use.
-    * [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
-      reports on Windows installation issues. Contributed a true Windows
-      binary installer.
-    * [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
-      to traceback printing.
-    * [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
-      prompt specials.
-    * [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
-      the multithreaded IPython.
-    * [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
-      author, helped with all the development of support for matplotlib
-      in IPyhton, including making necessary changes to matplotlib itself.
-    * [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
-    * [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
-      with (X)Emacs support, threading patches, ideas...
-    * [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
-      packaging and distribution.
-    * [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
-      tab-completing named arguments of user-defined functions.
-    * [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
-      support implementation for searching namespaces.
-    * [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
-      so that when pdb is activated from within IPython, coloring, tab
-      completion and other features continue to work seamlessly.
-    * [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
-      editor invocation on syntax errors (see
-      http://www.scipy.net/roundup/ipython/issue36).
-    * [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
-      paging system.
-    * [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
  No newline at end of file
+And last but not least, all the kind IPython users who have emailed new code,
+bug reports, fixes, comments and ideas. A brief list follows, please let us
+know if we have ommitted your name by accident:
+
+* Dan Milstein <danmil-AT-comcast.net>.  A bold refactoring of the 
+  core prefilter stuff in the IPython interpreter.
+
+* [Jack Moffit] <jack-AT-xiph.org> Bug fixes, including the infamous
+  color problem. This bug alone caused many lost hours and
+  frustration, many thanks to him for the fix. I've always been a
+  fan of Ogg & friends, now I have one more reason to like these folks.
+  Jack is also contributing with Debian packaging and many other
+  things.
+ 
+* [Alexander Schmolck] <a.schmolck-AT-gmx.net> Emacs work, bug
+  reports, bug fixes, ideas, lots more. The ipython.el mode for
+  (X)Emacs is Alex's code, providing full support for IPython under
+  (X)Emacs.
+
+* [Andrea Riciputi] <andrea.riciputi-AT-libero.it> Mac OSX
+  information, Fink package management.
+
+* [Gary Bishop] <gb-AT-cs.unc.edu> Bug reports, and patches to work
+  around the exception handling idiosyncracies of WxPython. Readline
+  and color support for Windows.
+
+* [Jeffrey Collins] <Jeff.Collins-AT-vexcel.com> Bug reports. Much
+  improved readline support, including fixes for Python 2.3.
+
+* [Dryice Liu] <dryice-AT-liu.com.cn> FreeBSD port.
+
+* [Mike Heeter] <korora-AT-SDF.LONESTAR.ORG>
+
+* [Christopher Hart] <hart-AT-caltech.edu> PDB integration.
+
+* [Milan Zamazal] <pdm-AT-zamazal.org> Emacs info.
+
+* [Philip Hisley] <compsys-AT-starpower.net>
+
+* [Holger Krekel] <pyth-AT-devel.trillke.net> Tab completion, lots
+  more.
+
+* [Robin Siebler] <robinsiebler-AT-starband.net>
+
+* [Ralf Ahlbrink] <ralf_ahlbrink-AT-web.de>
+
+* [Thorsten Kampe] <thorsten-AT-thorstenkampe.de>
+
+* [Fredrik Kant] <fredrik.kant-AT-front.com> Windows setup.
+
+* [Syver Enstad] <syver-en-AT-online.no> Windows setup.
+
+* [Richard] <rxe-AT-renre-europe.com> Global embedding.
+
+* [Hayden Callow] <h.callow-AT-elec.canterbury.ac.nz> Gnuplot.py 1.6
+  compatibility.
+  
+* [Leonardo Santagada] <retype-AT-terra.com.br> Fixes for Windows
+  installation.
+  
+* [Christopher Armstrong] <radix-AT-twistedmatrix.com> Bugfixes.
+
+* [Francois Pinard] <pinard-AT-iro.umontreal.ca> Code and
+  documentation fixes.
+  
+* [Cory Dodt] <cdodt-AT-fcoe.k12.ca.us> Bug reports and Windows
+  ideas. Patches for Windows installer.
+  
+* [Olivier Aubert] <oaubert-AT-bat710.univ-lyon1.fr> New magics.
+
+* [King C. Shu] <kingshu-AT-myrealbox.com> Autoindent patch.
+
+* [Chris Drexler] <chris-AT-ac-drexler.de> Readline packages for
+  Win32/CygWin.
+  
+* [Gustavo Cordova Avila] <gcordova-AT-sismex.com> EvalDict code for
+  nice, lightweight string interpolation.
+  
+* [Kasper Souren] <Kasper.Souren-AT-ircam.fr> Bug reports, ideas.
+
+* [Gever Tulley] <gever-AT-helium.com> Code contributions.
+
+* [Ralf Schmitt] <ralf-AT-brainbot.com> Bug reports & fixes.
+
+* [Oliver Sander] <osander-AT-gmx.de> Bug reports.
+
+* [Rod Holland] <rhh-AT-structurelabs.com> Bug reports and fixes to
+  logging module.
+
+* [Daniel 'Dang' Griffith] <pythondev-dang-AT-lazytwinacres.net>
+  Fixes, enhancement suggestions for system shell use.
+
+* [Viktor Ransmayr] <viktor.ransmayr-AT-t-online.de> Tests and
+  reports on Windows installation issues. Contributed a true Windows
+  binary installer.
+
+* [Mike Salib] <msalib-AT-mit.edu> Help fixing a subtle bug related
+  to traceback printing.
+
+* [W.J. van der Laan] <gnufnork-AT-hetdigitalegat.nl> Bash-like
+  prompt specials.
+
+* [Antoon Pardon] <Antoon.Pardon-AT-rece.vub.ac.be> Critical fix for
+  the multithreaded IPython.
+
+* [John Hunter] <jdhunter-AT-nitace.bsd.uchicago.edu> Matplotlib
+  author, helped with all the development of support for matplotlib
+  in IPyhton, including making necessary changes to matplotlib itself.
+
+* [Matthew Arnison] <maffew-AT-cat.org.au> Bug reports, '%run -d' idea.
+
+* [Prabhu Ramachandran] <prabhu_r-AT-users.sourceforge.net> Help
+  with (X)Emacs support, threading patches, ideas...
+
+* [Norbert Tretkowski] <tretkowski-AT-inittab.de> help with Debian
+  packaging and distribution.
+
+* [George Sakkis] <gsakkis-AT-eden.rutgers.edu> New matcher for
+  tab-completing named arguments of user-defined functions.
+
+* [Jörgen Stenarson] <jorgen.stenarson-AT-bostream.nu> Wildcard
+  support implementation for searching namespaces.
+
+* [Vivian De Smedt] <vivian-AT-vdesmedt.com> Debugger enhancements,
+  so that when pdb is activated from within IPython, coloring, tab
+  completion and other features continue to work seamlessly.
+
+* [Scott Tsai] <scottt958-AT-yahoo.com.tw> Support for automatic
+  editor invocation on syntax errors (see
+  http://www.scipy.net/roundup/ipython/issue36).
+
+* [Alexander Belchenko] <bialix-AT-ukr.net> Improvements for win32
+  paging system.
+
+* [Will Maier] <willmaier-AT-ml1.net> Official OpenBSD port.
+
+* [Ondrej Certik] <ondrej-AT-certik.cz>: set up the IPython docs to use the new
+  Sphinx system used by Python, Matplotlib and many more projects.
+
+* [Stefan van der Walt] <stefan-AT-sun.ac.za>: support for the new config system.

docs/source/development/development.txt

@@ -1,191 +1,142 b''
 .. _development:
 
-==================================
+==============================
 IPython development guidelines
-==================================
-
-.. contents::
+==============================
 
 
 Overview
 ========
 
-IPython is the next generation of IPython.  It is named such for two reasons:
-
-- Eventually, IPython will become IPython version 1.0.
-- This new code base needs to be able to co-exist with the existing IPython until
-  it is a full replacement for it.  Thus we needed a different name.  We couldn't
-  use ``ipython`` (lowercase) as some files systems are case insensitive.
-
-There are two, no three, main goals of the IPython effort:
-
-1. Clean up the existing codebase and write lots of tests.
-2. Separate the core functionality of IPython from the terminal to enable IPython
-   to be used from within a variety of GUI applications.
-3. Implement a system for interactive parallel computing.
-
-While the third goal may seem a bit unrelated to the main focus of IPython, it turns
-out that the technologies required for this goal are nearly identical with those
-required for goal two. This is the main reason the interactive parallel computing
-capabilities are being put into IPython proper. Currently the third of these goals is
-furthest along.
-
-This document describes IPython from the perspective of developers.  
+This document describes IPython from the perspective of developers. Most
+importantly, it gives information for people who want to contribute to the
+development of IPython. So if you want to help out, read on!
+
+How to contribute to IPython
+============================
+
+IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_.
+This makes it easy for people to contribute to the development of IPython.
+Here is a sketch of how to get going.
+
+Install Bazaar and create a Launchpad account
+---------------------------------------------
 
+First make sure you have installed Bazaar (see their `website
+<http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about
+you, try the following::
 
-Project organization
-====================
-
-Subpackages
------------
+    $ bzr whoami
+    Joe Coder <[e-mail]>
 
-IPython is organized into semi self-contained subpackages.  Each of the subpackages will have its own:
-
-- **Dependencies**.  One of the most important things to keep in mind in
-  partitioning code amongst subpackages, is that they should be used to cleanly
-  encapsulate dependencies.  
-- **Tests**.  Each subpackage shoud have its own ``tests`` subdirectory that
-  contains all of the tests for that package.  For information about writing tests
-  for IPython, see the `Testing System`_ section of this document.
-- **Configuration**.  Each subpackage should have its own ``config`` subdirectory
-  that contains the configuration information for the components of the 
-  subpackage.  For information about how the IPython configuration system
-  works, see the `Configuration System`_ section of this document.
-- **Scripts**.  Each subpackage should have its own ``scripts`` subdirectory that
-  contains all of the command line scripts associated with the subpackage.
+This should display your name and email. Next, you will want to create an
+account on the `Launchpad website <http://www.launchpad.net>`_ and setup your
+ssh keys. For more information of setting up your ssh keys, see `this link
+<https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_.
 
-Installation and dependencies
------------------------------
-   
-IPython will not use `setuptools`_ for installation. Instead, we will use standard
-``setup.py`` scripts that use `distutils`_. While there are a number a extremely nice
-features that `setuptools`_ has (like namespace packages), the current implementation
-of `setuptools`_ has performance problems, particularly on shared file systems. In
-particular, when Python packages are installed on NSF file systems, import times
-become much too long (up towards 10 seconds). 
+Get the main IPython branch from Launchpad
+------------------------------------------
 
-Because IPython is being used extensively in the context of high performance
-computing, where performance is critical but shared file systems are common, we feel
-these performance hits are not acceptable. Thus, until the performance problems
-associated with `setuptools`_ are addressed, we will stick with plain `distutils`_. We
-are hopeful that these problems will be addressed and that we will eventually begin
-using `setuptools`_. Because of this, we are trying to organize IPython in a way that
-will make the eventual transition to `setuptools`_ as painless as possible.
+Now, you can get a copy of the main IPython development branch (we call this
+the "trunk")::
 
-Because we will be using `distutils`_, there will be no method for automatically installing dependencies.  Instead, we are following the approach of `Matplotlib`_ which can be summarized as follows:
+    $ bzr branch lp:ipython
 
-- Distinguish between required and optional dependencies.  However, the required
-  dependencies for IPython should be only the Python standard library.
-- Upon installation check to see which optional dependencies are present and tell
-  the user which parts of IPython need which optional dependencies.
+Create a working branch
+-----------------------
 
-It is absolutely critical that each subpackage of IPython has a clearly specified set
-of dependencies and that dependencies are not carelessly inherited from other IPython
-subpackages. Furthermore, tests that have certain dependencies should not fail if
-those dependencies are not present. Instead they should be skipped and print a
-message.
+When working on IPython, you won't actually make edits directly to the
+:file:`lp:ipython` branch. Instead, you will create a separate branch for your
+changes. For now, let's assume you want to do your work in a branch named
+"ipython-mybranch". Create this branch by doing::
 
-.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
-.. _distutils: http://docs.python.org/lib/module-distutils.html
-.. _Matplotlib: http://matplotlib.sourceforge.net/
+    $ bzr branch ipython ipython-mybranch
 
-Specific subpackages
---------------------
+When you actually create a branch, you will want to give it a name that
+reflects the nature of the work that you will be doing in it, like
+"install-docs-update".
 
-``core``
-	This is the core functionality of IPython that is independent of the
-	terminal, network and GUIs.  Most of the code that is in the current
-	IPython trunk will be refactored, cleaned up and moved here.
+Make edits in your working branch
+---------------------------------
 
-``kernel``
-	The enables the IPython core to be expose to a the network.  This is
-	also where all of the parallel computing capabilities are to be found.
-	
-``config``
-	The configuration package used by IPython.
+Now you are ready to actually make edits in your :file:`ipython-mybranch`
+branch. Before doing this, it is helpful to install this branch so you can
+test your changes as you work. This is easiest if you have setuptools
+installed. Then, just do::
 
-``frontends``
-	The various frontends for IPython.  A frontend is the end-user application
-	that exposes the capabilities of IPython to the user.  The most basic frontend
-	will simply be a terminal based application that looks just like today 's 
-	IPython.  Other frontends will likely be more powerful and based on GUI toolkits.
-
-``notebook``
-	An application that allows users to work with IPython notebooks.
-
-``tools``
-	This is where general utilities go.
+    $ cd ipython-mybranch
+    $ python setupegg.py develop
 
+Now, make some changes. After a while, you will want to commit your changes.
+This let's Bazaar know that you like the changes you have made and gives you
+an opportunity to keep a nice record of what you have done. This looks like
+this::
 
-Version control
-===============
-
-In the past, IPython development has been done using `Subversion`__.  Recently, we made the transition to using `Bazaar`__ and `Launchpad`__.  This makes it much easier for people
-to contribute code to IPython.  Here is a sketch of how to use Bazaar for IPython
-development.  First, you should install Bazaar.  After you have done that, make
-sure that it is working by getting the latest main branch of IPython::
-
-	$ bzr branch lp:ipython
-
-Now you can create a new branch for you to do your work in::
+    $ ...do work in ipython-mybranch...
+    $ bzr commit -m "the commit message goes here"
 
-	$ bzr branch ipython ipython-mybranch
-
-The typical work cycle in this branch will be to make changes in `ipython-mybranch`
-and then commit those changes using the commit command::
-
-	$ ...do work in ipython-mybranch...
-	$ bzr ci -m "the commit message goes here"
-
-Please note that since we now don't use an old-style linear ChangeLog
-(that tends to cause problems with distributed version control
-systems), you should ensure that your log messages are reasonably
-detailed.  Use a docstring-like approach in the commit messages
-(including the second line being left *blank*)::
+Please note that since we now don't use an old-style linear ChangeLog (that
+tends to cause problems with distributed version control systems), you should
+ensure that your log messages are reasonably detailed.  Use a docstring-like
+approach in the commit messages (including the second line being left
+*blank*)::
 
   Single line summary of  changes being committed.
 
-  - more details when warranted ...
-  - including crediting outside contributors if they sent the
+  * more details when warranted ...
+  * including crediting outside contributors if they sent the
     code/bug/idea!
 
-If we couple this with a policy of making single commits for each
-reasonably atomic change, the bzr log should give an excellent view of
-the project, and the `--short` log option becomes a nice summary.
+As you work, you will repeat this edit/commit cycle many times. If you work on
+your branch for a long time, you will also want to get the latest changes from
+the :file:`lp:ipython` branch. This can be done with the following sequence of
+commands::
 
-While working with this branch, it is a good idea to merge in changes that have been
-made upstream in the parent branch.  This can be done by doing::
+    $ ls
+    ipython
+    ipython-mybranch
+    
+    $ cd ipython
+    $ bzr pull
+    $ cd ../ipython-mybranch
+    $ bzr merge ../ipython
+    $ bzr commit -m "Merging changes from trunk"
 
-	$ bzr pull
-	
-If this command shows that the branches have diverged, then you should do a merge
-instead::
+Along the way, you should also run the IPython test suite.  You can do this using the :command:`iptest` command::
 
-	$ bzr merge lp:ipython
+    $ cd
+    $ iptest
 
-If you want others to be able to see your branch, you can create an account with
-launchpad and push the branch to your own workspace::
+The :command:`iptest` command will also pick up and run any tests you have written.
 
-	$ bzr push bzr+ssh://<me>@bazaar.launchpad.net/~<me>/+junk/ipython-mybranch
+Post your branch and request a code review
+------------------------------------------
 
-Finally, once the work in your branch is done, you can merge your changes back into
-the `ipython` branch by using merge::
+Once you are done with your edits, you should post your branch on Launchpad so
+that other IPython developers can review the changes and help you merge your
+changes into the main development branch. To post your branch on Launchpad,
+do::
 
-	$ cd ipython
-	$ merge ../ipython-mybranch
-	[resolve any conflicts]
-	$ bzr ci -m "Fixing that bug"
-	$ bzr push
+    $ cd ipython-mybranch
+    $ bzr push lp:~yourusername/ipython/ipython-mybranch
 
-But this will require you to have write permissions to the `ipython` branch.  It you don't
-you can tell one of the IPython devs about your branch and they can do the merge for you.
+Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you
+should see your branch under the "Code" tab. If you click on your branch, you
+can provide a short description of the branch as well as mark its status. Most
+importantly, you should click the link that reads "Propose for merging into
+another branch". What does this do?
 
-More information about Bazaar workflows can be found `here`__.
+This let's the other IPython developers know that your branch is ready to be
+reviewed and merged into the main development branch. During this review
+process, other developers will give you feedback and help you get your code
+ready to be merged. What types of things will we be looking for:
 
-.. __: http://subversion.tigris.org/
-.. __: http://bazaar-vcs.org/
-.. __: http://www.launchpad.net/ipython
-.. __: http://doc.bazaar-vcs.org/bzr.dev/en/user-guide/index.html
+* All code is documented.
+* All code has tests.
+* The entire IPython test suite passes.
+
+Once your changes have been reviewed and approved, someone will merge them
+into the main development branch.
 
 Documentation
 =============
@@ -193,38 +144,32 b' Documentation'
 Standalone documentation
 ------------------------
 
-All standalone documentation should be written in plain text (``.txt``) files using
-`reStructuredText`_ for markup and formatting. All such documentation should be placed
-in the top level directory ``docs`` of the IPython source tree. Or, when appropriate,
-a suitably named subdirectory should be used. The documentation in this location will
-serve as the main source for IPython documentation and all existing documentation
-should be converted to this format.
+All standalone documentation should be written in plain text (``.txt``) files
+using reStructuredText [reStructuredText]_ for markup and formatting. All such
+documentation should be placed in directory :file:`docs/source` of the IPython
+source tree. The documentation in this location will serve as the main source
+for IPython documentation and all existing documentation should be converted
+to this format.
 
-In the future, the text files in the ``docs`` directory will be used to generate all
-forms of documentation for IPython. This include documentation on the IPython website
-as well as *pdf* documentation.
+To build the final documentation, we use Sphinx [Sphinx]_.  Once you have Sphinx installed, you can build the html docs yourself by doing::
 
-.. _reStructuredText: http://docutils.sourceforge.net/rst.html
+    $ cd ipython-mybranch/docs
+    $ make html
 
 Docstring format
 ----------------
 
-Good docstrings are very important. All new code will use `Epydoc`_ for generating API
-docs, so we will follow the `Epydoc`_ conventions. More specifically, we will use
-`reStructuredText`_ for markup and formatting, since it is understood by a wide
-variety of tools. This means that if in the future we have any reason to change from
-`Epydoc`_ to something else, we'll have fewer transition pains.
-
-Details about using `reStructuredText`_ for docstrings can be found `here
+Good docstrings are very important. All new code should have docstrings that
+are formatted using reStructuredText for markup and formatting, since it is
+understood by a wide variety of tools. Details about using reStructuredText
+for docstrings can be found `here
 <http://epydoc.sourceforge.net/manual-othermarkup.html>`_.
 
-.. _Epydoc: http://epydoc.sourceforge.net/
-
 Additional PEPs of interest regarding documentation of code:
 
-- `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
-- `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
-- `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
+* `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_
+* `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_
+* `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_
 
 
 Coding conventions
@@ -233,128 +178,133 b' Coding conventions'
 General
 -------
 
-In general, we'll try to follow the standard Python style conventions as described here:
+In general, we'll try to follow the standard Python style conventions as
+described here:
 
-- `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
+* `Style Guide for Python Code  <http://www.python.org/peps/pep-0008.html>`_  
 
 
 Other comments:
 
-- In a large file, top level classes and functions should be 
+* In a large file, top level classes and functions should be 
   separated by 2-3 lines to make it easier to separate them visually.
-- Use 4 spaces for indentation.
-- Keep the ordering of methods the same in classes that have the same 
-  methods.  This is particularly true for classes that implement 
-  similar interfaces and for interfaces that are similar.
+* Use 4 spaces for indentation.
+* Keep the ordering of methods the same in classes that have the same 
+  methods.  This is particularly true for classes that implement an interface.
 
 Naming conventions
 ------------------
 
-In terms of naming conventions, we'll follow the guidelines from the `Style Guide for
-Python Code`_.
+In terms of naming conventions, we'll follow the guidelines from the `Style
+Guide for Python Code`_.
 
 For all new IPython code (and much existing code is being refactored), we'll use:
 
-- All ``lowercase`` module names.
+* All ``lowercase`` module names.
 
-- ``CamelCase`` for class names.
+* ``CamelCase`` for class names.
 
-- ``lowercase_with_underscores`` for methods, functions, variables and attributes.
+* ``lowercase_with_underscores`` for methods, functions, variables and
+  attributes.
 
-This may be confusing as most of the existing IPython codebase uses a different convention (``lowerCamelCase`` for methods and attributes).  Slowly, we will move IPython over to the new
-convention, providing shadow names for backward compatibility in public interfaces. 
+There are, however, some important exceptions to these rules. In some cases,
+IPython code will interface with packages (Twisted, Wx, Qt) that use other
+conventions. At some level this makes it impossible to adhere to our own
+standards at all times. In particular, when subclassing classes that use other
+naming conventions, you must follow their naming conventions. To deal with
+cases like this, we propose the following policy:
 
-There are, however, some important exceptions to these rules.  In some cases, IPython
-code will interface with packages (Twisted, Wx, Qt) that use other conventions.  At some level this makes it impossible to adhere to our own standards at all times.  In particular, when subclassing classes that use other naming conventions, you must follow their naming conventions.  To deal with cases like this, we propose the following policy:
+* If you are subclassing a class that uses different conventions, use its
+  naming conventions throughout your subclass.  Thus, if you are creating a
+  Twisted Protocol class, used Twisted's
+  ``namingSchemeForMethodsAndAttributes.``
 
-- If you are subclassing a class that uses different conventions, use its
-  naming conventions throughout your subclass.  Thus, if you are creating a 
-  Twisted Protocol class, used Twisted's ``namingSchemeForMethodsAndAttributes.``
+* All IPython's official interfaces should use our conventions.  In some cases
+  this will mean that you need to provide shadow names (first implement
+  ``fooBar`` and then ``foo_bar = fooBar``).  We want to avoid this at all
+  costs, but it will probably be necessary at times.  But, please use this
+  sparingly!
 
-- All IPython's official interfaces should use our conventions.  In some cases 
-  this will mean that you need to provide shadow names (first implement ``fooBar``
-  and then ``foo_bar = fooBar``).  We want to avoid this at all costs, but it 
-  will probably be necessary at times.  But, please use this sparingly!
+Implementation-specific *private* methods will use
+``_single_underscore_prefix``. Names with a leading double underscore will
+*only* be used in special cases, as they makes subclassing difficult (such
+names are not easily seen by child classes).
 
-Implementation-specific *private* methods will use ``_single_underscore_prefix``.
-Names with a leading double underscore will *only* be used in special cases, as they
-makes subclassing difficult (such names are not easily seen by child classes).
-
-Occasionally some run-in lowercase names are used, but mostly for very short names or
-where we are implementing methods very similar to existing ones in a base class (like
-``runlines()`` where ``runsource()`` and ``runcode()`` had established precedent).
+Occasionally some run-in lowercase names are used, but mostly for very short
+names or where we are implementing methods very similar to existing ones in a
+base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had
+established precedent).
 
 The old IPython codebase has a big mix of classes and modules prefixed with an
-explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned upon, as
-namespaces offer cleaner prefixing. The only case where this approach is justified is
-for classes which are expected to be imported into external namespaces and a very
-generic name (like Shell) is too likely to clash with something else. We'll need to
-revisit this issue as we clean up and refactor the code, but in general we should
-remove as many unnecessary ``IP``/``ip`` prefixes as possible. However, if a prefix
-seems absolutely necessary the more specific ``IPY`` or ``ipy`` are preferred.
+explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned
+upon, as namespaces offer cleaner prefixing. The only case where this approach
+is justified is for classes which are expected to be imported into external
+namespaces and a very generic name (like Shell) is too likely to clash with
+something else. We'll need to revisit this issue as we clean up and refactor
+the code, but in general we should remove as many unnecessary ``IP``/``ip``
+prefixes as possible. However, if a prefix seems absolutely necessary the more
+specific ``IPY`` or ``ipy`` are preferred.
 
 .. _devel_testing:
 
 Testing system
 ==============
 
-It is extremely important that all code contributed to IPython has tests. Tests should
-be written as unittests, doctests or as entities that the `Nose`_ testing package will
-find. Regardless of how the tests are written, we will use `Nose`_ for discovering and
-running the tests. `Nose`_ will be required to run the IPython test suite, but will
-not be required to simply use IPython.
-
-.. _Nose: http://code.google.com/p/python-nose/
+It is extremely important that all code contributed to IPython has tests.
+Tests should be written as unittests, doctests or as entities that the Nose
+[Nose]_ testing package will find. Regardless of how the tests are written, we
+will use Nose for discovering and running the tests. Nose will be required to
+run the IPython test suite, but will not be required to simply use IPython.
 
-Tests of `Twisted`__ using code should be written by subclassing the ``TestCase`` class
-that comes with ``twisted.trial.unittest``. When this is done, `Nose`_ will be able to
-run the tests and the twisted reactor will be handled correctly.
+Tests of Twisted using code need to follow two additional guidelines:
 
-.. __: http://www.twistedmatrix.com
+1. Twisted using tests should be written by subclassing the :class:`TestCase`
+   class that comes with :mod:`twisted.trial.unittest`. 
 
-Each subpackage in IPython should have its own ``tests`` directory that contains all
-of the tests for that subpackage. This allows each subpackage to be self-contained. If
-a subpackage has any dependencies beyond the Python standard library, the tests for
-that subpackage should be skipped if the dependencies are not found. This is very
-important so users don't get tests failing simply because they don't have dependencies.
+2. All :class:`Deferred` instances that are created in the test must be
+   properly chained and the final one *must* be the return value of the test
+   method.
 
-We also need to look into use Noses ability to tag tests to allow a more modular
-approach of running tests.
-
-.. _devel_config:
+When these two things are done, Nose will be able to run the tests and the
+twisted reactor will be handled correctly.
 
-Configuration system
-====================
+Each subpackage in IPython should have its own :file:`tests` directory that
+contains all of the tests for that subpackage. This allows each subpackage to
+be self-contained. If a subpackage has any dependencies beyond the Python
+standard library, the tests for that subpackage should be skipped if the
+dependencies are not found. This is very important so users don't get tests
+failing simply because they don't have dependencies.
 
-IPython uses `.ini`_ files for configuration purposes. This represents a huge
-improvement over the configuration system used in IPython. IPython works with these
-files using the `ConfigObj`_ package, which IPython includes as
-``ipython1/external/configobj.py``.
+To run the IPython test suite, use the :command:`iptest` command that is installed with IPython::
 
-Currently, we are using raw `ConfigObj`_ objects themselves. Each subpackage of IPython
-should contain a ``config`` subdirectory that contains all of the configuration
-information for the subpackage. To see how configuration information is defined (along
-with defaults) see at the examples in ``ipython1/kernel/config`` and
-``ipython1/core/config``. Likewise, to see how the configuration information is used,
-see examples in ``ipython1/kernel/scripts/ipengine.py``.
-
-Eventually, we will add a new layer on top of the raw `ConfigObj`_ objects. We are
-calling this new layer, ``tconfig``, as it will use a `Traits`_-like validation model.
-We won't actually use `Traits`_, but will implement something similar in pure Python.
-But, even in this new system, we will still use `ConfigObj`_ and `.ini`_ files
-underneath the hood. Talk to Fernando if you are interested in working on this part of
-IPython. The current prototype of ``tconfig`` is located in the IPython sandbox.
-
-.. _.ini: http://docs.python.org/lib/module-ConfigParser.html
-.. _ConfigObj: http://www.voidspace.org.uk/python/configobj.html
-.. _Traits: http://code.enthought.com/traits/
+    $ iptest
 
+This command runs Nose with the proper options and extensions.
 
+.. _devel_config:
 
+Release checklist
+=================
 
+Most of the release process is automated by the :file:`release` script in the
+:file:`tools` directory.  This is just a handy reminder for the release manager.
 
+#. Run the release script, which makes the tar.gz, eggs and Win32 .exe
+   installer.  It posts them to the site and registers the release with PyPI.
 
+#. Updating the website with announcements and links to the updated
+   changes.txt in html form. Remember to put a short note both on the news
+   page of the site and on Launcphad.
 
+#. Drafting a short release announcement with i) highlights and ii) a link to
+   the html changes.txt.
 
+#. Make sure that the released version of the docs is live on the site.
 
+#. Celebrate!
 
+.. [Bazaar] Bazaar. http://bazaar-vcs.org/
+.. [Launchpad] Launchpad. http://www.launchpad.net/ipython
+.. [reStructuredText] reStructuredText.  http://docutils.sourceforge.net/rst.html
+.. [Sphinx] Sphinx. http://sphinx.pocoo.org/
+.. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/

docs/source/development/index.txt

@@ -7,3 +7,5 b' Development'
 
    development.txt
    roadmap.txt
+   notification_blueprint.txt
+   config_blueprint.txt

docs/source/development/notification_blueprint.txt

@@ -1,4 +1,4 b''
-.. Notification:
+.. _notification:
 
 ==========================================
 IPython.kernel.core.notification blueprint
@@ -6,42 +6,78 b' IPython.kernel.core.notification blueprint'
 
 Overview
 ========
-The :mod:`IPython.kernel.core.notification` module will provide a simple implementation of a notification center and support for the observer pattern within the :mod:`IPython.kernel.core`. The main intended use case is to provide notification of Interpreter events to an observing frontend during the execution of a single block of code.
+
+The :mod:`IPython.kernel.core.notification` module will provide a simple
+implementation of a notification center and support for the observer pattern
+within the :mod:`IPython.kernel.core`. The main intended use case is to
+provide notification of Interpreter events to an observing frontend during the
+execution of a single block of code.
 
 Functional Requirements
 =======================
+
 The notification center must:
-	* Provide synchronous notification of events to all registered observers.
-	* Provide typed or labeled notification types
-	* Allow observers to register callbacks for individual or all notification types
-	* Allow observers to register callbacks for events from individual or all notifying objects
-	* Notification to the observer consists of the notification type, notifying object and user-supplied extra information [implementation: as keyword parameters to the registered callback]
-	* Perform as O(1) in the case of no registered observers.
-	* Permit out-of-process or cross-network extension.
-	
+
+* Provide synchronous notification of events to all registered observers.
+
+* Provide typed or labeled notification types.
+
+* Allow observers to register callbacks for individual or all notification
+  types.
+
+* Allow observers to register callbacks for events from individual or all
+  notifying objects.
+
+* Notification to the observer consists of the notification type, notifying
+  object and user-supplied extra information [implementation: as keyword
+  parameters to the registered callback].
+
+* Perform as O(1) in the case of no registered observers.
+
+* Permit out-of-process or cross-network extension.
+    
 What's not included
-==============================================================
+===================
+
 As written, the :mod:`IPython.kernel.core.notificaiton` module does not:
- 	* Provide out-of-process or network notifications [these should be handled by a separate, Twisted aware module in :mod:`IPython.kernel`].
-	* Provide zope.interface-style interfaces for the notification system [these should also be provided by the :mod:`IPython.kernel` module]
-	
+
+* Provide out-of-process or network notifications (these should be handled by
+  a separate, Twisted aware module in :mod:`IPython.kernel`).
+
+* Provide zope.interface-style interfaces for the notification system (these
+  should also be provided by the :mod:`IPython.kernel` module).
+    
 Use Cases
 =========
+
 The following use cases describe the main intended uses of the notificaiton module and illustrate the main success scenario for each use case:
 
-	1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
-		from IPython.kernel.core.notification import NotificationCenter
-		center = NotificationCenter.sharedNotificationCenter
-		center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
-	
-	and elsewhere in his front end::
-		def stdout_notification(self, type, notifying_object, out_string=None):
-			self.writeStdOut(out_string)
-	
-	If everything works, the Interpreter will (according to its published API) fire a notification via the :data:`IPython.kernel.core.notification.sharedCenter` of type :const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up to the Intereter implementation to figure out when to do this]. The notificaiton center will then call the registered callbacks for that event type (in this case, Dwight's frontend's stdout_notification method). Again, according to its API, the Interpreter provides an additional keyword argument when firing the notificaiton of out_string, a copy of the string it will write to stdout.
-
-	Like magic, Dwight's frontend is able to provide output, even during long-running calculations. Now if Jim could just convince Dwight to use Twisted...
-
-	2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
-	
-	3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc.
  No newline at end of file
+    1. Dwight Schroot is writing a frontend for the IPython project. His frontend is stuck in the stone age and must communicate synchronously with an IPython.kernel.core.Interpreter instance. Because code is executed in blocks by the Interpreter, Dwight's UI freezes every time he executes a long block of code. To keep track of the progress of his long running block, Dwight adds the following code to his frontend's set-up code::
+
+        from IPython.kernel.core.notification import NotificationCenter
+        center = NotificationCenter.sharedNotificationCenter
+        center.registerObserver(self, type=IPython.kernel.core.Interpreter.STDOUT_NOTIFICATION_TYPE, notifying_object=self.interpreter, callback=self.stdout_notification)
+    
+    and elsewhere in his front end::
+
+        def stdout_notification(self, type, notifying_object, out_string=None):
+            self.writeStdOut(out_string)
+    
+If everything works, the Interpreter will (according to its published API)
+fire a notification via the
+:data:`IPython.kernel.core.notification.sharedCenter` of type
+:const:`STD_OUT_NOTIFICATION_TYPE` before writing anything to stdout [it's up
+to the Intereter implementation to figure out when to do this]. The
+notificaiton center will then call the registered callbacks for that event
+type (in this case, Dwight's frontend's stdout_notification method). Again,
+according to its API, the Interpreter provides an additional keyword argument
+when firing the notificaiton of out_string, a copy of the string it will write
+to stdout.
+
+Like magic, Dwight's frontend is able to provide output, even during
+long-running calculations. Now if Jim could just convince Dwight to use
+Twisted...
+
+    2. Boss Hog is writing a frontend for the IPython project. Because Boss Hog is stuck in the stone age, his frontend will be written in a new Fortran-like dialect of python and will run only from the command line. Because he doesn't need any fancy notification system and is used to worrying about every cycle on his rat-wheel powered mini, Boss Hog is adamant that the new notification system not produce any performance penalty. As they say in Hazard county, there's no such thing as a free lunch. If he wanted zero overhead, he should have kept using IPython 0.8. Instead, those tricky Duke boys slide in a suped-up bridge-out jumpin' awkwardly confederate-lovin' notification module that imparts only a constant (and small) performance penalty when the Interpreter (or any other object) fires an event for which there are no registered observers. Of course, the same notificaiton-enabled Interpreter can then be used in frontends that require notifications, thus saving the IPython project from a nasty civil war.
+    
+    3. Barry is wrting a frontend for the IPython project. Because Barry's front end is the *new hotness*, it uses an asynchronous event model to communicate with a Twisted :mod:`~IPython.kernel.engineservice` that communicates with the IPython :class:`~IPython.kernel.core.interpreter.Interpreter`. Using the :mod:`IPython.kernel.notification` module, an asynchronous wrapper on the :mod:`IPython.kernel.core.notification` module, Barry's frontend can register for notifications from the interpreter that are delivered asynchronously. Even if Barry's frontend is running on a separate process or even host from the Interpreter, the notifications are delivered, as if by dark and twisted magic. Just like Dwight's frontend, Barry's frontend can now recieve notifications of e.g. writing to stdout/stderr, opening/closing an external file, an exception in the executing code, etc.
  No newline at end of file

docs/source/development/roadmap.txt

@@ -4,93 +4,78 b''
 Development roadmap
 ===================
 
-.. contents::
-
 IPython is an ambitious project that is still under heavy development.  However, we want IPython to become useful to as many people as possible, as quickly as possible.  To help us accomplish this, we are laying out a roadmap of where we are headed and what needs to happen to get there.  Hopefully, this will help the IPython developers figure out the best things to work on for each upcoming release.
 
-Speaking of releases, we are going to begin releasing a new version of IPython every four weeks.  We are hoping that a regular release schedule, along with a clear roadmap of where we are headed will propel the project forward.
-
-Where are we headed
-===================
+Work targeted to particular releases
+====================================
 
-Our goal with IPython is simple: to provide a *powerful*, *robust* and *easy to use* framework for parallel computing.  While there are other secondary goals you will hear us talking about at various times, this is the primary goal of IPython that frames the roadmap.
+Release 0.10
+------------
 
-Steps along the way
-===================
+* Initial refactor of :command:`ipcluster`.
 
-Here we describe the various things that we need to work on to accomplish this goal.
+* Better TextMate integration.
 
-Setting up for regular release schedule
----------------------------------------
+* Merge in the daemon branch.
 
-We would like to begin to release IPython regularly (probably a 4 week release cycle).  To get ready for this, we need to revisit the development guidelines and put in information about releasing IPython.  
+Release 0.11
+------------
 
-Process startup and management
-------------------------------
+* Refactor the configuration system and command line options for
+  :command:`ipengine` and :command:`ipcontroller`. This will include the
+  creation of cluster directories that encapsulate all the configuration
+  files, log files and security related files for a particular cluster.
 
-IPython is implemented using a distributed set of processes that communicate using TCP/IP network channels.  Currently, users have to start each of the various processes separately using command line scripts.  This is both difficult and error prone.  Furthermore, there are a number of things that often need to be managed once the processes have been started, such as the sending of signals and the shutting down and cleaning up of processes.
+* Refactor :command:`ipcluster` to support the new configuration system.
 
-We need to build a system that makes it trivial for users to start and manage IPython processes.  This system should have the following properties:
+* Refactor the daemon stuff to support the new configuration system.
 
-	* It should possible to do everything through an extremely simple API that users
-	  can call from their own Python script.  No shell commands should be needed.
-	* This simple API should be configured using standard .ini files.
-	* The system should make it possible to start processes using a number of different
-	  approaches:  SSH, PBS/Torque, Xgrid, Windows Server, mpirun, etc.
-	* The controller and engine processes should each have a daemon for monitoring, 
-	  signaling and clean up.
-	* The system should be secure.
-	* The system should work under all the major operating systems, including
-	  Windows.
+* Merge back in the core of the notebook.
 
-Initial work has begun on the daemon infrastructure, and some of the needed logic is contained in the ipcluster script.
+Release 0.12
+------------
 
-Ease of use/high-level approaches to parallelism
-------------------------------------------------
+* Fully integrate process startup with the daemons for full process 
+  management.
 
-While our current API for clients is well designed, we can still do a lot better in designing a user-facing API that is super simple.  The main goal here is that it should take *almost no extra code* for users to get their code running in parallel.  For this to be possible, we need to tie into Python's standard idioms that enable efficient coding.  The biggest ones we are looking at are using context managers (i.e., Python 2.5's ``with`` statement) and decorators.  Initial work on this front has begun, but more work is needed.
+* Make the capabilites of :command:`ipcluster` available from simple Python 
+  classes.
 
-We also need to think about new models for expressing parallelism.  This is fun work as most of the foundation has already been established.
+Major areas of work
+===================
 
-Security
---------
+Refactoring the main IPython core
+---------------------------------
 
-Currently, IPython has no built in security or security model.  Because we would like IPython to be usable on public computer systems and over wide area networks, we need to come up with a robust solution for security.  Here are some of the specific things that need to be included:
+Process management for :mod:`IPython.kernel`
+--------------------------------------------
 
-	* User authentication between all processes (engines, controller and clients).
-	* Optional TSL/SSL based encryption of all communication channels.
-	* A good way of picking network ports so multiple users on the same system can
-	  run their own controller and engines without interfering with those of others.
-	* A clear model for security that enables users to evaluate the security risks
-	  associated with using IPython in various manners.
+Configuration system
+--------------------
 
-For the implementation of this, we plan on using Twisted's support for SSL and authentication.  One things that we really should look at is the `Foolscap`_ network protocol, which provides many of these things out of the box.
+Performance problems
+--------------------
 
-.. _Foolscap: http://foolscap.lothar.com/trac
+Currently, we have a number of performance issues that are waiting to bite users:
 
-The security work needs to be done in conjunction with other network protocol stuff.
+* The controller stores a large amount of state in Python dictionaries. Under
+  heavy usage, these dicts with get very large, causing memory usage problems.
+  We need to develop more scalable solutions to this problem, such as using a
+  sqlite database to store this state. This will also help the controller to
+  be more fault tolerant.
 
-Latent performance issues
--------------------------
+* We currently don't have a good way of handling large objects in the
+  controller. The biggest problem is that because we don't have any way of
+  streaming objects, we get lots of temporary copies in the low-level buffers.
+  We need to implement a better serialization approach and true streaming
+  support.
 
-Currently, we have a number of performance issues that are waiting to bite users:
+* The controller currently unpickles and repickles objects. We need to use the
+  [push|pull]_serialized methods instead.
 
-	* The controller store a large amount of state in Python dictionaries.  Under heavy
-	  usage, these dicts with get very large, causing memory usage problems.  We need to 
-	  develop more scalable solutions to this problem, such as using a sqlite database
-	  to store this state.  This will also help the controller to be more fault tolerant.
-	* Currently, the client to controller connections are done through XML-RPC using 
-	  HTTP 1.0.  This is very inefficient as XML-RPC is a very verbose protocol and 
-	  each request must be handled with a new connection.  We need to move these network
-	  connections over to PB or Foolscap.
-	* We currently don't have a good way of handling large objects in the controller.
-	  The biggest problem is that because we don't have any way of streaming objects,
-	  we get lots of temporary copies in the low-level buffers.  We need to implement
-	  a better serialization approach and true streaming support.
-	* The controller currently unpickles and repickles objects.  We need to use the 
-	  [push|pull]_serialized methods instead.
-	* Currently the controller is a bottleneck.  We need the ability to scale the 
-	  controller by aggregating multiple controllers into one effective controller.
+* Currently the controller is a bottleneck. The best approach for this is to
+  separate the controller itself into multiple processes, one for the core
+  controller and one each for the controller interfaces.
 
 
 

docs/source/faq.txt

@@ -16,10 +16,13 b' Will IPython speed my Python code up?'
 Yes and no. When converting a serial code to run in parallel, there often many
 difficulty questions that need to be answered, such as:
 
-	* How should data be decomposed onto the set of processors?
-	* What are the data movement patterns?
-	* Can the algorithm be structured to minimize data movement?
-	* Is dynamic load balancing important?
+* How should data be decomposed onto the set of processors?
+
+* What are the data movement patterns?
+
+* Can the algorithm be structured to minimize data movement?
+
+* Is dynamic load balancing important?
 
 We can't answer such questions for you. This is the hard (but fun) work of parallel
 computing. But, once you understand these things IPython will make it easier for you to
@@ -28,9 +31,7 b' resulting parallel code interactively.'
 
 With that said, if your problem is trivial to parallelize, IPython has a number of
 different interfaces that will enable you to parallelize things is almost no time at
-all.  A good place to start is the ``map`` method of our `multiengine interface`_.
-
-.. _multiengine interface: ./parallel_multiengine
+all.  A good place to start is the ``map`` method of our :class:`MultiEngineClient`.
 
 What is the best way to use MPI from Python?
 --------------------------------------------
@@ -40,26 +41,33 b' What about all the other parallel computing packages in Python?'
 
 Some of the unique characteristic of IPython are:
 
-	* IPython is the only architecture that abstracts out the notion of a 
-	  parallel computation in such a way that new models of parallel computing
-	  can be explored quickly and easily.  If you don't like the models we
-	  provide, you can simply create your own using the capabilities we provide.
-	* IPython is asynchronous from the ground up (we use `Twisted`_).
-	* IPython's architecture is designed to avoid subtle problems 
-	  that emerge because of Python's global interpreter lock (GIL).
-	* While IPython'1 architecture is designed to support a wide range
-	  of novel parallel computing models, it is fully interoperable with
-	  traditional MPI applications.
-	* IPython has been used and tested extensively on modern supercomputers.
-	* IPython's networking layers are completely modular.  Thus, is 
-	  straightforward to replace our existing network protocols with 
-	  high performance alternatives (ones based upon Myranet/Infiniband).
-	* IPython is designed from the ground up to support collaborative
-	  parallel computing.  This enables multiple users to actively develop
-	  and run the *same* parallel computation.
-	* Interactivity is a central goal for us.  While IPython does not have
-	  to be used interactivly, is can be.
-	  
+* IPython is the only architecture that abstracts out the notion of a 
+  parallel computation in such a way that new models of parallel computing
+  can be explored quickly and easily.  If you don't like the models we
+  provide, you can simply create your own using the capabilities we provide.
+  
+* IPython is asynchronous from the ground up (we use `Twisted`_).
+
+* IPython's architecture is designed to avoid subtle problems 
+  that emerge because of Python's global interpreter lock (GIL).
+  
+* While IPython's architecture is designed to support a wide range
+  of novel parallel computing models, it is fully interoperable with
+  traditional MPI applications.
+  
+* IPython has been used and tested extensively on modern supercomputers.
+
+* IPython's networking layers are completely modular.  Thus, is 
+  straightforward to replace our existing network protocols with 
+  high performance alternatives (ones based upon Myranet/Infiniband).
+  
+* IPython is designed from the ground up to support collaborative
+  parallel computing.  This enables multiple users to actively develop
+  and run the *same* parallel computation.
+  
+* Interactivity is a central goal for us.  While IPython does not have
+  to be used interactivly, it can be.
+      
 .. _Twisted: http://www.twistedmatrix.com
 
 Why The IPython controller a bottleneck in my parallel calculation?
@@ -71,13 +79,17 b' too much data is being pushed and pulled to and from the engines. If your algori'
 is structured in this way, you really should think about alternative ways of
 handling the data movement. Here are some ideas:
 
-	1. Have the engines write data to files on the locals disks of the engines.
-	2. Have the engines write data to files on a file system that is shared by
-	   the engines.
-	3. Have the engines write data to a database that is shared by the engines.
-	4. Simply keep data in the persistent memory of the engines and move the
-	   computation to the data (rather than the data to the computation).
-	5. See if you can pass data directly between engines using MPI.
+1. Have the engines write data to files on the locals disks of the engines.
+
+2. Have the engines write data to files on a file system that is shared by
+   the engines.
+
+3. Have the engines write data to a database that is shared by the engines.
+
+4. Simply keep data in the persistent memory of the engines and move the
+   computation to the data (rather than the data to the computation).
+
+5. See if you can pass data directly between engines using MPI.
 
 Isn't Python slow to be used for high-performance parallel computing?
 ---------------------------------------------------------------------

docs/source/history.txt

@@ -7,50 +7,32 b' History'
 Origins
 =======
 
-The current IPython system grew out of the following three projects:
-
-    * [ipython] by Fernando Pérez. I was working on adding
-      Mathematica-type prompts and a flexible configuration system
-      (something better than $PYTHONSTARTUP) to the standard Python
-      interactive interpreter.
-    * [IPP] by Janko Hauser. Very well organized, great usability. Had
-      an old help system. IPP was used as the 'container' code into
-      which I added the functionality from ipython and LazyPython.
-    * [LazyPython] by Nathan Gray. Simple but very powerful. The quick
-      syntax (auto parens, auto quotes) and verbose/colored tracebacks
-      were all taken from here.
-
-When I found out about IPP and LazyPython I tried to join all three
-into a unified system. I thought this could provide a very nice
-working environment, both for regular programming and scientific
-computing: shell-like features, IDL/Matlab numerics, Mathematica-type
-prompt history and great object introspection and help facilities. I
-think it worked reasonably well, though it was a lot more work than I
-had initially planned.
-
-
-Current status
-==============
-
-The above listed features work, and quite well for the most part. But
-until a major internal restructuring is done (see below), only bug
-fixing will be done, no other features will be added (unless very minor
-and well localized in the cleaner parts of the code).
-
-IPython consists of some 18000 lines of pure python code, of which
-roughly two thirds is reasonably clean. The rest is, messy code which
-needs a massive restructuring before any further major work is done.
-Even the messy code is fairly well documented though, and most of the
-problems in the (non-existent) class design are well pointed to by a
-PyChecker run. So the rewriting work isn't that bad, it will just be
-time-consuming.
-
-
-Future
-------
-
-See the separate new_design document for details. Ultimately, I would
-like to see IPython become part of the standard Python distribution as a
-'big brother with batteries' to the standard Python interactive
-interpreter. But that will never happen with the current state of the
-code, so all contributions are welcome.
  No newline at end of file
+IPython was starting in 2001 by Fernando Perez.  IPython as we know it
+today grew out of the following three projects:
+
+* ipython by Fernando Pérez. I was working on adding
+  Mathematica-type prompts and a flexible configuration system
+  (something better than $PYTHONSTARTUP) to the standard Python
+  interactive interpreter.
+* IPP by Janko Hauser. Very well organized, great usability. Had
+  an old help system. IPP was used as the 'container' code into
+  which I added the functionality from ipython and LazyPython.
+* LazyPython by Nathan Gray. Simple but very powerful. The quick
+  syntax (auto parens, auto quotes) and verbose/colored tracebacks
+  were all taken from here.
+
+Here is how Fernando describes it:
+
+    When I found out about IPP and LazyPython I tried to join all three
+    into a unified system. I thought this could provide a very nice
+    working environment, both for regular programming and scientific
+    computing: shell-like features, IDL/Matlab numerics, Mathematica-type
+    prompt history and great object introspection and help facilities. I
+    think it worked reasonably well, though it was a lot more work than I
+    had initially planned.
+
+Today and how we got here
+=========================
+
+This needs to be filled in.
+

docs/source/index.txt

@@ -2,11 +2,15 b''
 IPython Documentation
 =====================
 
-Contents
-========
+.. htmlonly::
+
+   :Release: |release|
+   :Date: |today|
+
+   Contents:
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    overview.txt
    install/index.txt
@@ -20,9 +24,7 b' Contents'
    license_and_copyright.txt
    credits.txt
 
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
  No newline at end of file
+.. htmlonly::
+   * :ref:`genindex`
+   * :ref:`modindex`
+   * :ref:`search`

docs/source/install/index.txt

@@ -7,5 +7,4 b' Installation'
 .. toctree::
    :maxdepth: 2
 
-   basic.txt
-   advanced.txt
+   install.txt

docs/source/interactive/index.txt

@@ -3,7 +3,7 b' Using IPython for interactive work'
 ==================================
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    tutorial.txt
    reference.txt

docs/source/interactive/reference.txt

@@ -1,14 +1,8 b''
-.. IPython documentation master file, created by sphinx-quickstart.py on Mon Mar 24 17:01:34 2008.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root 'toctree' directive.
-
 =================
 IPython reference
 =================
 
-.. contents::
-
-.. _Command line options:
+.. _command_line_options:
 
 Command-line usage
 ==================
@@ -288,12 +282,13 b' All options with a [no] prepended can be specified in negated form'
 	recursive inclusions.
 
     -prompt_in1, pi1 <string>
-	Specify the string used for input prompts. Note that if you
-	are using numbered prompts, the number is represented with a
-	'\#' in the string. Don't forget to quote strings with spaces
-	embedded in them. Default: 'In [\#]:'.  Sec. Prompts_
-	discusses in detail all the available escapes to customize
-	your prompts.
+
+        Specify the string used for input prompts. Note that if you are using
+	numbered prompts, the number is represented with a '\#' in the
+	string. Don't forget to quote strings with spaces embedded in
+	them. Default: 'In [\#]:'.  The :ref:`prompts section <prompts>`
+	discusses in detail all the available escapes to customize your
+	prompts.
 
     -prompt_in2, pi2 <string>
 	Similar to the previous option, but used for the continuation
@@ -2077,13 +2072,14 b' customizations.'
 Access to the standard Python help
 ----------------------------------
 
-As of Python 2.1, a help system is available with access to object
-docstrings and the Python manuals. Simply type 'help' (no quotes) to
-access it. You can also type help(object) to obtain information about a
-given object, and help('keyword') for information on a keyword. As noted
-in sec. `accessing help`_, you need to properly configure
-your environment variable PYTHONDOCS for this feature to work correctly.
+As of Python 2.1, a help system is available with access to object docstrings
+and the Python manuals. Simply type 'help' (no quotes) to access it. You can
+also type help(object) to obtain information about a given object, and
+help('keyword') for information on a keyword. As noted :ref:`here
+<accessing_help>`, you need to properly configure your environment variable
+PYTHONDOCS for this feature to work correctly.
 
+.. _dynamic_object_info:
 
 Dynamic object information
 --------------------------
@@ -2126,7 +2122,7 b' are not really defined as separate identifiers. Try for example typing'
 {}.get? or after doing import os, type os.path.abspath??.
 
 
-.. _Readline:
+.. _readline:
 
 Readline-based features
 -----------------------
@@ -2240,10 +2236,9 b' explanation in your ipythonrc file.'
 Session logging and restoring
 -----------------------------
 
-You can log all input from a session either by starting IPython with
-the command line switches -log or -logfile (see sec. `command line
-options`_) or by activating the logging at any moment with the magic
-function %logstart.
+You can log all input from a session either by starting IPython with the
+command line switches -log or -logfile (see :ref:`here <command_line_options>`)
+or by activating the logging at any moment with the magic function %logstart.
 
 Log files can later be reloaded with the -logplay option and IPython
 will attempt to 'replay' the log by executing all the lines in it, thus
@@ -2279,6 +2274,8 b' resume logging to a file which had previously been started with'
 %logstart. They will fail (with an explanation) if you try to use them
 before logging has been started.
 
+.. _system_shell_access:
+
 System shell access
 -------------------
 
@@ -2389,14 +2386,16 b" These features are basically a terminal version of Ka-Ping Yee's cgitb"
 module, now part of the standard Python library.
 
 
-.. _Input caching:
+.. _input_caching:
 
 Input caching system
 --------------------
 
-IPython offers numbered prompts (In/Out) with input and output caching.
-All input is saved and can be retrieved as variables (besides the usual
-arrow key recall).
+IPython offers numbered prompts (In/Out) with input and output caching
+(also referred to as 'input history'). All input is saved and can be 
+retrieved as variables (besides the usual arrow key recall), in 
+addition to the %rep magic command that brings a history entry
+up for editing on the next  command line.
 
 The following GLOBAL variables always exist (so don't overwrite them!):
 _i: stores previous input. _ii: next previous. _iii: next-next previous.
@@ -2429,7 +2428,43 b' sec. 6.2 <#sec:magic> for more details on the macro system.'
 A history function %hist allows you to see any part of your input
 history by printing a range of the _i variables.
 
-.. _Output caching:
+You can also search ('grep') through your history by typing 
+'%hist -g somestring'. This also searches through the so called *shadow history*,
+which remembers all the commands (apart from multiline code blocks)
+you have ever entered. Handy for searching for svn/bzr URL's, IP adrresses
+etc. You can bring shadow history entries listed by '%hist -g' up for editing 
+(or re-execution by just pressing ENTER) with %rep command. Shadow history 
+entries are not available as _iNUMBER variables, and they are identified by 
+the '0' prefix in %hist -g output. That is, history entry 12 is a normal 
+history entry, but 0231 is a shadow history entry.
+
+Shadow history was added because the readline history is inherently very
+unsafe - if you have multiple IPython sessions open, the last session 
+to close will overwrite the history of previountly closed session. Likewise,
+if a crash occurs, history is never saved, whereas shadow history entries
+are added after entering every command (so a command executed 
+in another IPython session is immediately available in other IPython 
+sessions that are open).
+
+To conserve space, a command can exist in shadow history only once - it doesn't 
+make sense to store a common line like "cd .." a thousand times. The idea is 
+mainly to provide a reliable place where valuable, hard-to-remember commands can 
+always be retrieved, as opposed to providing an exact sequence of commands
+you have entered in actual order.
+
+Because shadow history has all the commands you have ever executed,
+time taken by %hist -g will increase oven time. If it ever starts to take
+too long (or it ends up containing sensitive information like passwords), 
+clear the shadow history by `%clear shadow_nuke`.
+
+Time taken to add entries to shadow history should be negligible, but 
+in any case, if you start noticing performance degradation after using 
+IPython for a long time (or running a script that floods the shadow history!), 
+you can 'compress' the shadow history by executing 
+`%clear shadow_compress`. In practice, this should never be necessary
+in normal use.
+
+.. _output_caching:
 
 Output caching system
 ---------------------
@@ -2472,7 +2507,7 b' Directory history'
 
 Your history of visited directories is kept in the global list _dh, and
 the magic %cd command can be used to go to any entry in that list. The
-%dhist command allows you to view this history. do ``cd -<TAB`` to
+%dhist command allows you to view this history. Do ``cd -<TAB`` to
 conventiently view the directory history.
 
 
@@ -3034,7 +3069,7 b' which is being shared by the interactive IPython loop and your GUI'
 thread, you should really handle it with thread locking and
 syncrhonization properties. The Python documentation discusses these.
 
-.. _Interactive demos:
+.. _interactive_demos:
 
 Interactive demos with IPython
 ==============================
@@ -3143,21 +3178,17 b' toolkits, including Tk, GTK and WXPython. It also provides a number of'
 commands useful for scientific computing, all with a syntax compatible
 with that of the popular Matlab program.
 
-IPython accepts the special option -pylab (Sec. `Command line
-options`_). This configures it to support matplotlib, honoring the
-settings in the .matplotlibrc file. IPython will detect the user's
-choice of matplotlib GUI backend, and automatically select the proper
-threading model to prevent blocking. It also sets matplotlib in
-interactive mode and modifies %run slightly, so that any
-matplotlib-based script can be executed using %run and the final
-show() command does not block the interactive shell.
-
-The -pylab option must be given first in order for IPython to
-configure its threading mode. However, you can still issue other
-options afterwards. This allows you to have a matplotlib-based
-environment customized with additional modules using the standard
-IPython profile mechanism (Sec. Profiles_): ''ipython -pylab -p
-myprofile'' will load the profile defined in ipythonrc-myprofile after
-configuring matplotlib.
-
-
+IPython accepts the special option -pylab (see :ref:`here
+<command_line_options>`). This configures it to support matplotlib, honoring
+the settings in the .matplotlibrc file. IPython will detect the user's choice
+of matplotlib GUI backend, and automatically select the proper threading model
+to prevent blocking. It also sets matplotlib in interactive mode and modifies
+%run slightly, so that any matplotlib-based script can be executed using %run
+and the final show() command does not block the interactive shell.
+
+The -pylab option must be given first in order for IPython to configure its
+threading mode. However, you can still issue other options afterwards. This
+allows you to have a matplotlib-based environment customized with additional
+modules using the standard IPython profile mechanism (see :ref:`here
+<profiles>`): ``ipython -pylab -p myprofile`` will load the profile defined in
+ipythonrc-myprofile after configuring matplotlib.

docs/source/interactive/tutorial.txt

@@ -4,8 +4,6 b''
 Quick IPython tutorial
 ======================
 
-.. contents::
-
 IPython can be used as an improved replacement for the Python prompt,
 and for that you don't really need to read any more of this manual. But
 in this section we'll try to summarize a few tips on how to make the
@@ -24,11 +22,11 b' Tab completion'
 --------------
 
 TAB-completion, especially for attributes, is a convenient way to explore the
-structure of any object you're dealing with. Simply type object_name.<TAB>
-and a list of the object's attributes will be printed (see readline_ for
-more). Tab completion also works on file and directory names, which combined
-with IPython's alias system allows you to do from within IPython many of the
-things you normally would need the system shell for.
+structure of any object you're dealing with. Simply type object_name.<TAB> and
+a list of the object's attributes will be printed (see :ref:`the readline
+section <readline>` for more). Tab completion also works on file and directory
+names, which combined with IPython's alias system allows you to do from within
+IPython many of the things you normally would need the system shell for.
 
 Explore your objects
 --------------------
@@ -39,18 +37,18 b' constructor details for classes. The magic commands %pdoc, %pdef, %psource'
 and %pfile will respectively print the docstring, function definition line,
 full source code and the complete file for any object (when they can be
 found). If automagic is on (it is by default), you don't need to type the '%'
-explicitly. See sec. `dynamic object information`_ for more.
+explicitly. See :ref:`this section <dynamic_object_info>` for more.
 
 The `%run` magic command
 ------------------------
 
-The %run magic command allows you to run any python script and load all of
-its data directly into the interactive namespace. Since the file is re-read
-from disk each time, changes you make to it are reflected immediately (in
-contrast to the behavior of import). I rarely use import for code I am
-testing, relying on %run instead. See magic_ section for more on this and
-other magic commands, or type the name of any magic command and ? to get
-details on it. See also sec. dreload_ for a recursive reload command. %run
+The %run magic command allows you to run any python script and load all of its
+data directly into the interactive namespace. Since the file is re-read from
+disk each time, changes you make to it are reflected immediately (in contrast
+to the behavior of import). I rarely use import for code I am testing, relying
+on %run instead. See :ref:`this section <magic>` for more on this and other
+magic commands, or type the name of any magic command and ? to get details on
+it. See also :ref:`this section <dreload>` for a recursive reload command. %run
 also has special flags for timing the execution of your scripts (-t) and for
 executing them under the control of either Python's pdb debugger (-d) or
 profiler (-p). With all of these, %run can be used as the main tool for
@@ -60,21 +58,21 b' choice.'
 Debug a Python script
 ---------------------
 
-Use the Python debugger, pdb. The %pdb command allows you to toggle on and
-off the automatic invocation of an IPython-enhanced pdb debugger (with
-coloring, tab completion and more) at any uncaught exception. The advantage
-of this is that pdb starts inside the function where the exception occurred,
-with all data still available. You can print variables, see code, execute
-statements and even walk up and down the call stack to track down the true
-source of the problem (which often is many layers in the stack above where
-the exception gets triggered). Running programs with %run and pdb active can
-be an efficient to develop and debug code, in many cases eliminating the need
-for print statements or external debugging tools. I often simply put a 1/0 in
-a place where I want to take a look so that pdb gets called, quickly view
-whatever variables I need to or test various pieces of code and then remove
-the 1/0. Note also that '%run -d' activates pdb and automatically sets
-initial breakpoints for you to step through your code, watch variables, etc.
-See Sec. `Output caching`_ for details.
+Use the Python debugger, pdb. The %pdb command allows you to toggle on and off
+the automatic invocation of an IPython-enhanced pdb debugger (with coloring,
+tab completion and more) at any uncaught exception. The advantage of this is
+that pdb starts inside the function where the exception occurred, with all data
+still available. You can print variables, see code, execute statements and even
+walk up and down the call stack to track down the true source of the problem
+(which often is many layers in the stack above where the exception gets
+triggered). Running programs with %run and pdb active can be an efficient to
+develop and debug code, in many cases eliminating the need for print statements
+or external debugging tools. I often simply put a 1/0 in a place where I want
+to take a look so that pdb gets called, quickly view whatever variables I need
+to or test various pieces of code and then remove the 1/0. Note also that '%run
+-d' activates pdb and automatically sets initial breakpoints for you to step
+through your code, watch variables, etc.  The :ref:`output caching section
+<output_caching>` has more details.
 
 Use the output cache
 --------------------
@@ -84,7 +82,8 b' and variables named _1, _2, etc. alias them. For example, the result of input'
 line 4 is available either as Out[4] or as _4. Additionally, three variables
 named _, __ and ___ are always kept updated with the for the last three
 results. This allows you to recall any previous result and further use it for
-new calculations. See Sec. `Output caching`_ for more.
+new calculations. See :ref:`the output caching section <output_caching>` for
+more.
 
 Suppress output
 ---------------
@@ -102,7 +101,7 b' A similar system exists for caching input. All input is stored in a global'
 list called In , so you can re-execute lines 22 through 28 plus line 34 by
 typing 'exec In[22:29]+In[34]' (using Python slicing notation). If you need
 to execute the same set of lines often, you can assign them to a macro with
-the %macro function. See sec. `Input caching`_ for more.
+the %macro function. See :ref:`here <input_caching>` for more.
 
 Use your input history
 ----------------------
@@ -134,17 +133,18 b' into Python variables.'
 Use Python variables when calling the shell
 -------------------------------------------
 
-Expand python variables when calling the shell (either via '!' and '!!' or
-via aliases) by prepending a $ in front of them. You can also expand complete
-python expressions. See `System shell access`_ for more.
+Expand python variables when calling the shell (either via '!' and '!!' or via
+aliases) by prepending a $ in front of them. You can also expand complete
+python expressions. See :ref:`our shell section <system_shell_access>` for
+more details.
 
 Use profiles
 ------------
 
 Use profiles to maintain different configurations (modules to load, function
 definitions, option settings) for particular tasks. You can then have
-customized versions of IPython for specific purposes. See sec. profiles_ for
-more.
+customized versions of IPython for specific purposes. :ref:`This section
+<profiles>` has more details.
 
 
 Embed IPython in your programs
@@ -152,7 +152,7 b' Embed IPython in your programs'
 
 A few lines of code are enough to load a complete IPython inside your own
 programs, giving you the ability to work with your data interactively after
-automatic processing has been completed. See sec. embedding_ for more.
+automatic processing has been completed. See :ref:`here <embedding>` for more.
 
 Use the Python profiler
 -----------------------
@@ -166,8 +166,8 b' Use IPython to present interactive demos'
 ----------------------------------------
 
 Use the IPython.demo.Demo class to load any Python script as an interactive
-demo. With a minimal amount of simple markup, you can control the execution
-of the script, stopping as needed. See sec. `interactive demos`_ for more.
+demo. With a minimal amount of simple markup, you can control the execution of
+the script, stopping as needed. See :ref:`here <interactive_demos>` for more.
 
 Run doctests
 ------------

docs/source/license_and_copyright.txt

@@ -1,61 +1,91 b''
 .. _license:
 
-=============================
-License and Copyright 
-=============================
+=====================
+License and Copyright
+=====================
 
-This files needs to be updated to reflect what the new COPYING.txt files says about our license and copyright!
+License
+=======
 
-IPython is released under the terms of the BSD license, whose general
-form can be found at: http://www.opensource.org/licenses/bsd-license.php. The full text of the
-IPython license is reproduced below::
+IPython is licensed under the terms of the new or revised BSD license, as follows::
 
-    IPython is released under a BSD-type license.
-
-    Copyright (c) 2001, 2002, 2003, 2004 Fernando Perez
-    <[e-mail]>.
-
-    Copyright (c) 2001 Janko Hauser <[e-mail]> and  
-    Nathaniel Gray <[e-mail]>.
+    Copyright (c) 2008, IPython Development Team
 
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    a. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-
-    b. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
-    c. Neither the name of the copyright holders nor the names of any
-    contributors to this software may be used to endorse or promote
-    products derived from this software without specific prior written
-    permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE. 
-
-Individual authors are the holders of the copyright for their code and
-are listed in each file.
+    modification, are permitted provided that the following conditions are
+    met:
+
+    Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+    Neither the name of the IPython Development Team nor the names of its
+    contributors may be used to endorse or promote products derived from this
+    software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+    THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+    PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+About the IPython Development Team
+==================================
+
+Fernando Perez began IPython in 2001 based on code from Janko Hauser
+<[e-mail]> and Nathaniel Gray <[e-mail]>. Fernando is still
+the project lead.
+
+The IPython Development Team is the set of all contributors to the IPython
+project. This includes all of the IPython subprojects. Here is a list of the
+currently active contributors:
+
+	* Matthieu Brucher
+	* Ondrej Certik
+	* Laurent Dufrechou
+	* Robert Kern
+	* Brian E. Granger
+	* Fernando Perez (project leader)
+	* Benjamin Ragan-Kelley
+	* Ville M. Vainio
+	* Gael Varoququx
+	* Stefan van der Walt
+	* Tech-X Corporation
+	* Barry Wark
+
+If your name is missing, please add it.  
+
+Our Copyright Policy
+====================
+
+IPython uses a shared copyright model. Each contributor maintains copyright
+over their contributions to IPython. But, it is important to note that these
+contributions are typically only changes to the repositories. Thus, the
+IPython source code, in its entirety is not the copyright of any single person
+or institution. Instead, it is the collective copyright of the entire IPython
+Development Team. If individual contributors want to maintain a record of what
+changes/contributions they have specific copyright on, they should indicate
+their copyright in the commit message of the change, when they commit the
+change to one of the IPython repositories.
+
+Miscellaneous
+=============
 
 Some files (DPyGetOpt.py, for example) may be licensed under different
-conditions. Ultimately each file indicates clearly the conditions under
-which its author/authors have decided to publish the code.
+conditions. Ultimately each file indicates clearly the conditions under which
+its author/authors have decided to publish the code.
 
-Versions of IPython up to and including 0.6.3 were released under the
-GNU Lesser General Public License (LGPL), available at
+Versions of IPython up to and including 0.6.3 were released under the GNU
+Lesser General Public License (LGPL), available at
 http://www.gnu.org/copyleft/lesser.html.
  No newline at end of file

docs/source/overview.txt

@@ -14,136 +14,164 b' However, the interpreter supplied with the standard Python distribution'
 is somewhat limited for extended interactive use.
 
 The goal of IPython is to create a comprehensive environment for
-interactive and exploratory computing.  To support, this goal, IPython
+interactive and exploratory computing.  To support this goal, IPython
 has two main components:
 
-	* An enhanced interactive Python shell.
-	* An architecture for interactive parallel computing.
+* An enhanced interactive Python shell.
+* An architecture for interactive parallel computing.
 
 All of IPython is open source (released under the revised BSD license).
 
 Enhanced interactive Python shell
 =================================
 
-IPython's interactive shell (`ipython`), has the following goals:
-
-   1. Provide an interactive shell superior to Python's default. IPython
-      has many features for object introspection, system shell access,
-      and its own special command system for adding functionality when
-      working interactively. It tries to be a very efficient environment
-      both for Python code development and for exploration of problems
-      using Python objects (in situations like data analysis).
-   2. Serve as an embeddable, ready to use interpreter for your own
-      programs. IPython can be started with a single call from inside
-      another program, providing access to the current namespace. This
-      can be very useful both for debugging purposes and for situations
-      where a blend of batch-processing and interactive exploration are
-      needed.
-   3. Offer a flexible framework which can be used as the base
-      environment for other systems with Python as the underlying
-      language. Specifically scientific environments like Mathematica,
-      IDL and Matlab inspired its design, but similar ideas can be
-      useful in many fields.
-   4. Allow interactive testing of threaded graphical toolkits. IPython
-      has support for interactive, non-blocking control of GTK, Qt and
-      WX applications via special threading flags. The normal Python
-      shell can only do this for Tkinter applications.
+IPython's interactive shell (:command:`ipython`), has the following goals,
+amongst others:
+
+1. Provide an interactive shell superior to Python's default. IPython
+   has many features for object introspection, system shell access,
+   and its own special command system for adding functionality when
+   working interactively. It tries to be a very efficient environment
+   both for Python code development and for exploration of problems
+   using Python objects (in situations like data analysis).
+  
+2. Serve as an embeddable, ready to use interpreter for your own
+   programs. IPython can be started with a single call from inside
+   another program, providing access to the current namespace. This
+   can be very useful both for debugging purposes and for situations
+   where a blend of batch-processing and interactive exploration are
+   needed.  New in the 0.9 version of IPython is a reusable wxPython
+   based IPython widget.
+  
+3. Offer a flexible framework which can be used as the base
+   environment for other systems with Python as the underlying
+   language. Specifically scientific environments like Mathematica,
+   IDL and Matlab inspired its design, but similar ideas can be
+   useful in many fields.
+  
+4. Allow interactive testing of threaded graphical toolkits. IPython
+   has support for interactive, non-blocking control of GTK, Qt and
+   WX applications via special threading flags. The normal Python
+   shell can only do this for Tkinter applications.
 
 Main features of the interactive shell
 --------------------------------------
 
-	* Dynamic object introspection. One can access docstrings, function
-	  definition prototypes, source code, source files and other details
-	  of any object accessible to the interpreter with a single
-	  keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
-	* Searching through modules and namespaces with :samp:`*` wildcards, both
-	  when using the :samp:`?` system and via the :samp:`%psearch` command.
-	* Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
-	  This works for keywords, modules, methods, variables and files in the
-	  current directory. This is supported via the readline library, and
-	  full access to configuring readline's behavior is provided. 
-	  Custom completers can be implemented easily for different purposes
-	  (system commands, magic arguments etc.)
-	* Numbered input/output prompts with command history (persistent
-	  across sessions and tied to each profile), full searching in this
-	  history and caching of all input and output.
-	* User-extensible 'magic' commands. A set of commands prefixed with
-	  :samp:`%` is available for controlling IPython itself and provides
-	  directory control, namespace information and many aliases to
-	  common system shell commands.
-	* Alias facility for defining your own system aliases.
-	* Complete system shell access. Lines starting with :samp:`!` are passed
-	  directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
-	  captures shell output into python variables for further use.
-	* Background execution of Python commands in a separate thread.
-	  IPython has an internal job manager called jobs, and a
-	  conveninence backgrounding magic function called :samp:`%bg`.
-	* The ability to expand python variables when calling the system
-	  shell. In a shell command, any python variable prefixed with :samp:`$` is
-	  expanded. A double :samp:`$$` allows passing a literal :samp:`$` to the shell (for
-	  access to shell and environment variables like :envvar:`PATH`).
-	* Filesystem navigation, via a magic :samp:`%cd` command, along with a
-	  persistent bookmark system (using :samp:`%bookmark`) for fast access to
-	  frequently visited directories.
-	* A lightweight persistence framework via the :samp:`%store` command, which
-	  allows you to save arbitrary Python variables. These get restored
-	  automatically when your session restarts.
-	* Automatic indentation (optional) of code as you type (through the
-	  readline library).
-	* Macro system for quickly re-executing multiple lines of previous
-	  input with a single name. Macros can be stored persistently via
-	  :samp:`%store` and edited via :samp:`%edit`.
-	* Session logging (you can then later use these logs as code in your
-	  programs). Logs can optionally timestamp all input, and also store
-	  session output (marked as comments, so the log remains valid
-	  Python source code).
-	* Session restoring: logs can be replayed to restore a previous
-	  session to the state where you left it.
-	* Verbose and colored exception traceback printouts. Easier to parse
-	  visually, and in verbose mode they produce a lot of useful
-	  debugging information (basically a terminal version of the cgitb
-	  module).
-	* Auto-parentheses: callable objects can be executed without
-	  parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
-	* Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
-	  auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
-	  automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
-	  becomes :samp:`my_function("a b")`.
-	* Extensible input syntax. You can define filters that pre-process
-	  user input to simplify input in special situations. This allows
-	  for example pasting multi-line code fragments which start with
-	  :samp:`>>>` or :samp:`...` such as those from other python sessions or the
-	  standard Python documentation.
-	* Flexible configuration system. It uses a configuration file which
-	  allows permanent setting of all command-line options, module
-	  loading, code and file execution. The system allows recursive file
-	  inclusion, so you can have a base file with defaults and layers
-	  which load other customizations for particular projects.
-	* Embeddable. You can call IPython as a python shell inside your own
-	  python programs. This can be used both for debugging code or for
-	  providing interactive abilities to your programs with knowledge
-	  about the local namespaces (very useful in debugging and data
-	  analysis situations).
-	* Easy debugger access. You can set IPython to call up an enhanced
-	  version of the Python debugger (pdb) every time there is an
-	  uncaught exception. This drops you inside the code which triggered
-	  the exception with all the data live and it is possible to
-	  navigate the stack to rapidly isolate the source of a bug. The
-	  :samp:`%run` magic command (with the :samp:`-d` option) can run any script under
-	  pdb's control, automatically setting initial breakpoints for you.
-	  This version of pdb has IPython-specific improvements, including
-	  tab-completion and traceback coloring support. For even easier
-	  debugger access, try :samp:`%debug` after seeing an exception. winpdb is 
-	  also supported, see ipy_winpdb extension.
-	* Profiler support. You can run single statements (similar to
-	  :samp:`profile.run()`) or complete programs under the profiler's control.
-	  While this is possible with standard cProfile or profile modules,
-	  IPython wraps this functionality with magic commands (see :samp:`%prun`
-	  and :samp:`%run -p`) convenient for rapid interactive work.
-	* Doctest support. The special :samp:`%doctest_mode` command toggles a mode
-	  that allows you to paste existing doctests (with leading :samp:`>>>`
-	  prompts and whitespace) and uses doctest-compatible prompts and
-	  output, so you can use IPython sessions as doctest code.
+* Dynamic object introspection. One can access docstrings, function
+  definition prototypes, source code, source files and other details
+  of any object accessible to the interpreter with a single
+  keystroke (:samp:`?`, and using :samp:`??` provides additional detail).
+  
+* Searching through modules and namespaces with :samp:`*` wildcards, both
+  when using the :samp:`?` system and via the :samp:`%psearch` command.
+
+* Completion in the local namespace, by typing :kbd:`TAB` at the prompt.
+  This works for keywords, modules, methods, variables and files in the
+  current directory. This is supported via the readline library, and
+  full access to configuring readline's behavior is provided. 
+  Custom completers can be implemented easily for different purposes
+  (system commands, magic arguments etc.)
+
+* Numbered input/output prompts with command history (persistent
+  across sessions and tied to each profile), full searching in this
+  history and caching of all input and output.
+
+* User-extensible 'magic' commands. A set of commands prefixed with
+  :samp:`%` is available for controlling IPython itself and provides
+  directory control, namespace information and many aliases to
+  common system shell commands.
+
+* Alias facility for defining your own system aliases.
+
+* Complete system shell access. Lines starting with :samp:`!` are passed
+  directly to the system shell, and using :samp:`!!` or :samp:`var = !cmd` 
+  captures shell output into python variables for further use.
+
+* Background execution of Python commands in a separate thread.
+  IPython has an internal job manager called jobs, and a
+  convenience backgrounding magic function called :samp:`%bg`.
+
+* The ability to expand python variables when calling the system
+  shell. In a shell command, any python variable prefixed with :samp:`$` is
+  expanded. A double :samp:`$$` allows passing a literal :samp:`$` to the shell (for
+  access to shell and environment variables like :envvar:`PATH`).
+
+* Filesystem navigation, via a magic :samp:`%cd` command, along with a
+  persistent bookmark system (using :samp:`%bookmark`) for fast access to
+  frequently visited directories.
+
+* A lightweight persistence framework via the :samp:`%store` command, which
+  allows you to save arbitrary Python variables. These get restored
+  automatically when your session restarts.
+
+* Automatic indentation (optional) of code as you type (through the
+  readline library).
+
+* Macro system for quickly re-executing multiple lines of previous
+  input with a single name. Macros can be stored persistently via
+  :samp:`%store` and edited via :samp:`%edit`.
+
+* Session logging (you can then later use these logs as code in your
+  programs). Logs can optionally timestamp all input, and also store
+  session output (marked as comments, so the log remains valid
+  Python source code).
+
+* Session restoring: logs can be replayed to restore a previous
+  session to the state where you left it.
+
+* Verbose and colored exception traceback printouts. Easier to parse
+  visually, and in verbose mode they produce a lot of useful
+  debugging information (basically a terminal version of the cgitb
+  module).
+
+* Auto-parentheses: callable objects can be executed without
+  parentheses: :samp:`sin 3` is automatically converted to :samp:`sin(3)`.
+
+* Auto-quoting: using :samp:`,`, or :samp:`;` as the first character forces
+  auto-quoting of the rest of the line: :samp:`,my_function a b` becomes
+  automatically :samp:`my_function("a","b")`, while :samp:`;my_function a b`
+  becomes :samp:`my_function("a b")`.
+
+* Extensible input syntax. You can define filters that pre-process
+  user input to simplify input in special situations. This allows
+  for example pasting multi-line code fragments which start with
+  :samp:`>>>` or :samp:`...` such as those from other python sessions or the
+  standard Python documentation.
+
+* Flexible configuration system. It uses a configuration file which
+  allows permanent setting of all command-line options, module
+  loading, code and file execution. The system allows recursive file
+  inclusion, so you can have a base file with defaults and layers
+  which load other customizations for particular projects.
+
+* Embeddable. You can call IPython as a python shell inside your own
+  python programs. This can be used both for debugging code or for
+  providing interactive abilities to your programs with knowledge
+  about the local namespaces (very useful in debugging and data
+  analysis situations).
+
+* Easy debugger access. You can set IPython to call up an enhanced
+  version of the Python debugger (pdb) every time there is an
+  uncaught exception. This drops you inside the code which triggered
+  the exception with all the data live and it is possible to
+  navigate the stack to rapidly isolate the source of a bug. The
+  :samp:`%run` magic command (with the :samp:`-d` option) can run any script under
+  pdb's control, automatically setting initial breakpoints for you.
+  This version of pdb has IPython-specific improvements, including
+  tab-completion and traceback coloring support. For even easier
+  debugger access, try :samp:`%debug` after seeing an exception. winpdb is 
+  also supported, see ipy_winpdb extension.
+
+* Profiler support. You can run single statements (similar to
+  :samp:`profile.run()`) or complete programs under the profiler's control.
+  While this is possible with standard cProfile or profile modules,
+  IPython wraps this functionality with magic commands (see :samp:`%prun`
+  and :samp:`%run -p`) convenient for rapid interactive work.
+
+* Doctest support. The special :samp:`%doctest_mode` command toggles a mode
+  that allows you to paste existing doctests (with leading :samp:`>>>`
+  prompts and whitespace) and uses doctest-compatible prompts and
+  output, so you can use IPython sessions as doctest code.
 
 Interactive parallel computing
 ==============================
@@ -153,6 +181,37 b' architecture within IPython that allows such hardware to be used quickly and eas'
 from Python. Moreover, this architecture is designed to support interactive and
 collaborative parallel computing.
 
+The main features of this system are:
+
+* Quickly parallelize Python code from an interactive Python/IPython session.
+
+* A flexible and dynamic process model that be deployed on anything from 
+  multicore workstations to supercomputers.
+
+* An architecture that supports many different styles of parallelism, from
+  message passing to task farming.  And all of these styles can be handled
+  interactively.
+
+* Both blocking and fully asynchronous interfaces.
+
+* High level APIs that enable many things to be parallelized in a few lines
+  of code.
+
+* Write parallel code that will run unchanged on everything from multicore
+  workstations to supercomputers.
+
+* Full integration with Message Passing libraries (MPI).
+
+* Capabilities based security model with full encryption of network connections.
+
+* Share live parallel jobs with other users securely.  We call this collaborative
+  parallel computing.
+
+* Dynamically load balanced task farming system.
+
+* Robust error handling.  Python exceptions raised in parallel execution are
+  gathered and presented to the top-level code.
+
 For more information, see our :ref:`overview <parallel_index>` of using IPython for
 parallel computing.
 

docs/source/parallel/index.txt

@@ -1,17 +1,16 b''
 .. _parallel_index:
 
 ====================================
-Using IPython for Parallel computing
+Using IPython for parallel computing
 ====================================
 
-User Documentation
-==================
-
 .. toctree::
    :maxdepth: 2
 
    parallel_intro.txt
+   parallel_process.txt
    parallel_multiengine.txt
    parallel_task.txt
    parallel_mpi.txt
+   parallel_security.txt