merged from ~fdo.perez/ipython/trunk-dev

IPython/config/default/ipcluster_config.py

@@ -0,0 +1,184 b''
+import os
+
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Select which launchers to use
+#-----------------------------------------------------------------------------
+
+# This allows you to control what method is used to start the controller
+# and engines.  The following methods are currently supported:
+# - Start as a regular process on localhost.
+# - Start using mpiexec.
+# - Start using the Windows HPC Server 2008 scheduler
+# - Start using PBS
+# - Start using SSH (currently broken)
+
+
+# The selected launchers can be configured below.
+
+# Options are:
+# - LocalControllerLauncher
+# - MPIExecControllerLauncher
+# - PBSControllerLauncher
+# - WindowsHPCControllerLauncher
+# c.Global.controller_launcher = 'IPython.kernel.launcher.LocalControllerLauncher'
+
+# Options are:
+# - LocalEngineSetLauncher
+# - MPIExecEngineSetLauncher
+# - PBSEngineSetLauncher
+# - WindowsHPCEngineSetLauncher
+# c.Global.engine_launcher = 'IPython.kernel.launcher.LocalEngineSetLauncher'
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# The default number of engines that will be started. This is overridden by
+# the -n command line option: "ipcluster start -n 4"
+# c.Global.n = 2
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+
+#-----------------------------------------------------------------------------
+# Local process launchers
+#-----------------------------------------------------------------------------
+
+# The command line arguments to call the controller with.
+# c.LocalControllerLauncher.controller_args = \
+#    ['--log-to-file','--log-level', '40']
+
+# The working directory for the controller
+# c.LocalEngineSetLauncher.work_dir = u''
+
+# Command line argument passed to the engines.
+# c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
+
+#-----------------------------------------------------------------------------
+# MPIExec launchers
+#-----------------------------------------------------------------------------
+
+# The mpiexec/mpirun command to use in started the controller.
+# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
+
+# Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecControllerLauncher.mpi_args = []
+
+# The command line argument to call the controller with.
+# c.MPIExecControllerLauncher.controller_args = \
+#     ['--log-to-file','--log-level', '40']
+
+
+# The mpiexec/mpirun command to use in started the controller.
+# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
+
+# Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecEngineSetLauncher.mpi_args = []
+
+# Command line argument passed to the engines.
+# c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
+
+# The default number of engines to start if not given elsewhere.
+# c.MPIExecEngineSetLauncher.n = 1
+
+#-----------------------------------------------------------------------------
+# SSH launchers
+#-----------------------------------------------------------------------------
+
+# Todo
+
+
+#-----------------------------------------------------------------------------
+# Unix batch (PBS) schedulers launchers
+#-----------------------------------------------------------------------------
+
+# The command line program to use to submit a PBS job.
+# c.PBSControllerLauncher.submit_command = 'qsub'
+
+# The command line program to use to delete a PBS job.
+# c.PBSControllerLauncher.delete_command = 'qdel'
+
+# A regular expression that takes the output of qsub and find the job id.
+# c.PBSControllerLauncher.job_id_regexp = r'\d+'
+
+# The batch submission script used to start the controller. This is where
+# environment variables would be setup, etc. This string is interpolated using
+# the Itpl module in IPython.external. Basically, you can use ${n} for the
+# number of engine and ${cluster_dir} for the cluster_dir.
+# c.PBSControllerLauncher.batch_template = """"""
+
+# The name of the instantiated batch script that will actually be used to
+# submit the job. This will be written to the cluster directory.
+# c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller'
+
+
+# The command line program to use to submit a PBS job.
+# c.PBSEngineSetLauncher.submit_command = 'qsub'
+
+# The command line program to use to delete a PBS job.
+# c.PBSEngineSetLauncher.delete_command = 'qdel'
+
+# A regular expression that takes the output of qsub and find the job id.
+# c.PBSEngineSetLauncher.job_id_regexp = r'\d+'
+
+# The batch submission script used to start the engines. This is where
+# environment variables would be setup, etc. This string is interpolated using
+# the Itpl module in IPython.external. Basically, you can use ${n} for the
+# number of engine and ${cluster_dir} for the cluster_dir.
+# c.PBSEngineSetLauncher.batch_template = """"""
+
+# The name of the instantiated batch script that will actually be used to
+# submit the job. This will be written to the cluster directory.
+# c.PBSEngineSetLauncher.batch_file_name = u'pbs_batch_script_engines'
+
+#-----------------------------------------------------------------------------
+# Windows HPC Server 2008 launcher configuration
+#-----------------------------------------------------------------------------
+
+# c.IPControllerJob.job_name = 'IPController'
+# c.IPControllerJob.is_exclusive = False
+# c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
+# c.IPControllerJob.priority = 'Highest'
+# c.IPControllerJob.requested_nodes = ''
+# c.IPControllerJob.project = 'MyProject'
+
+# c.IPControllerTask.task_name = 'IPController'
+# c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
+# c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
+# c.IPControllerTask.environment_variables = {}
+
+# c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
+# c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
+
+
+# c.IPEngineSetJob.job_name = 'IPEngineSet'
+# c.IPEngineSetJob.is_exclusive = False
+# c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
+# c.IPEngineSetJob.priority = 'Highest'
+# c.IPEngineSetJob.requested_nodes = ''
+# c.IPEngineSetJob.project = 'MyProject'
+
+# c.IPEngineTask.task_name = 'IPEngine'
+# c.IPEngineTask.engine_cmd = [u'ipengine.exe']
+# c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
+# c.IPEngineTask.environment_variables = {}
+
+# c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
+# c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
+
+
+
+
+
+
+

IPython/config/default/ipcontroller_config.py

@@ -0,0 +1,136 b''
+from IPython.config.loader import Config
+
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# Basic Global config attributes
+
+# Start up messages are logged to stdout using the logging module.
+# These all happen before the twisted reactor is started and are
+# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
+# and smaller is more verbose.
+# c.Global.log_level = 20
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# A list of Python statements that will be run before starting the 
+# controller. This is provided because occasionally certain things need to 
+# be imported in the controller for pickling to work.
+# c.Global.import_statements = ['import math']
+
+# Reuse the controller's FURL files. If False, FURL files are regenerated
+# each time the controller is run. If True, they will be reused, *but*, you
+# also must set the network ports by hand. If set, this will override the
+# values set for the client and engine connections below.
+# c.Global.reuse_furls = True
+
+# Enable SSL encryption on all connections to the controller. If set, this
+# will override the values set for the client and engine connections below.
+# c.Global.secure = True
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+#-----------------------------------------------------------------------------
+# Configure the client services
+#-----------------------------------------------------------------------------
+
+# Basic client service config attributes
+
+# The network interface the controller will listen on for client connections.
+# This should be an IP address or hostname of the controller's host. The empty
+# string means listen on all interfaces.
+# c.FCClientServiceFactory.ip = ''
+
+# The TCP/IP port the controller will listen on for client connections. If 0
+# a random port will be used. If the controller's host has a firewall running
+# it must allow incoming traffic on this port.
+# c.FCClientServiceFactory.port = 0
+
+# The client learns how to connect to the controller by looking at the
+# location field embedded in the FURL. If this field is empty, all network
+# interfaces that the controller is listening on will be listed. To have the
+# client connect on a particular interface, list it here.
+# c.FCClientServiceFactory.location = ''
+
+# Use SSL encryption for the client connection.
+# c.FCClientServiceFactory.secure =  True
+
+# Reuse the client FURL each time the controller is started. If set, you must
+# also pick a specific network port above (FCClientServiceFactory.port).
+# c.FCClientServiceFactory.reuse_furls = False
+
+#-----------------------------------------------------------------------------
+# Configure the engine services
+#-----------------------------------------------------------------------------
+
+# Basic config attributes for the engine services.
+
+# The network interface the controller will listen on for engine connections.
+# This should be an IP address or hostname of the controller's host. The empty
+# string means listen on all interfaces.
+# c.FCEngineServiceFactory.ip = ''
+
+# The TCP/IP port the controller will listen on for engine connections. If 0
+# a random port will be used. If the controller's host has a firewall running
+# it must allow incoming traffic on this port.
+# c.FCEngineServiceFactory.port = 0
+
+# The engine learns how to connect to the controller by looking at the
+# location field embedded in the FURL. If this field is empty, all network
+# interfaces that the controller is listening on will be listed. To have the
+# client connect on a particular interface, list it here.
+# c.FCEngineServiceFactory.location = ''
+
+# Use SSL encryption for the engine connection.
+# c.FCEngineServiceFactory.secure = True
+
+# Reuse the client FURL each time the controller is started. If set, you must
+# also pick a specific network port above (FCClientServiceFactory.port).
+# c.FCEngineServiceFactory.reuse_furls = False
+
+#-----------------------------------------------------------------------------
+# Developer level configuration attributes
+#-----------------------------------------------------------------------------
+
+# You shouldn't have to modify anything in this section. These attributes
+# are more for developers who want to change the behavior of the controller
+# at a fundamental level.
+
+# c.FCClientServiceFactory.cert_file = u'ipcontroller-client.pem'
+
+# default_client_interfaces = Config()
+# default_client_interfaces.Task.interface_chain = [
+#     'IPython.kernel.task.ITaskController',
+#     'IPython.kernel.taskfc.IFCTaskController'
+# ]
+# 
+# default_client_interfaces.Task.furl_file = u'ipcontroller-tc.furl'
+# 
+# default_client_interfaces.MultiEngine.interface_chain = [
+#     'IPython.kernel.multiengine.IMultiEngine',
+#     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+# ]
+# 
+# default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_client_interfaces
+
+# c.FCEngineServiceFactory.cert_file = u'ipcontroller-engine.pem'
+
+# default_engine_interfaces = Config()
+# default_engine_interfaces.Default.interface_chain = [
+#     'IPython.kernel.enginefc.IFCControllerBase'
+# ]
+# 
+# default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_engine_interfaces

IPython/config/default/ipengine_config.py

@@ -0,0 +1,90 b''
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# Start up messages are logged to stdout using the logging module.
+# These all happen before the twisted reactor is started and are
+# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
+# and smaller is more verbose.
+# c.Global.log_level = 20
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# A list of strings that will be executed in the users namespace on the engine
+# before it connects to the controller.
+# c.Global.exec_lines = ['import numpy']
+
+# The engine will try to connect to the controller multiple times, to allow
+# the controller time to startup and write its FURL file. These parameters 
+# control the number of retries (connect_max_tries) and the initial delay
+# (connect_delay) between attemps. The actual delay between attempts gets
+# longer each time by a factor of 1.5 (delay[i] = 1.5*delay[i-1])
+# those attemps.
+# c.Global.connect_delay = 0.1
+# c.Global.connect_max_tries = 15
+
+# By default, the engine will look for the controller's FURL file in its own
+# cluster directory. Sometimes, the FURL file will be elsewhere and this 
+# attribute can be set to the full path of the FURL file.
+# c.Global.furl_file = u''
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+#-----------------------------------------------------------------------------
+# MPI configuration
+#-----------------------------------------------------------------------------
+
+# Upon starting the engine can be configured to call MPI_Init. This section
+# configures that.
+
+# Select which MPI section to execute to setup MPI. The value of this 
+# attribute must match the name of another attribute in the MPI config 
+# section (mpi4py, pytrilinos, etc.). This can also be set by the --mpi
+# command line option.
+# c.MPI.use = ''
+
+# Initialize MPI using mpi4py. To use this, set c.MPI.use = 'mpi4py' to use
+# --mpi=mpi4py at the command line.
+# c.MPI.mpi4py = """from mpi4py import MPI as mpi
+# mpi.size = mpi.COMM_WORLD.Get_size()
+# mpi.rank = mpi.COMM_WORLD.Get_rank()
+# """
+
+# Initialize MPI using pytrilinos. To use this, set c.MPI.use = 'pytrilinos' 
+# to use --mpi=pytrilinos at the command line.
+# c.MPI.pytrilinos = """from PyTrilinos import Epetra
+# class SimpleStruct:
+# pass
+# mpi = SimpleStruct()
+# mpi.rank = 0
+# mpi.size = 0
+# """
+
+#-----------------------------------------------------------------------------
+# Developer level configuration attributes
+#-----------------------------------------------------------------------------
+
+# You shouldn't have to modify anything in this section. These attributes
+# are more for developers who want to change the behavior of the controller
+# at a fundamental level.
+
+# You should not have to change these attributes.
+
+# c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+# c.Global.furl_file_name = u'ipcontroller-engine.furl'
+
+
+
+
+
+
+

IPython/config/profile/ipython_config_cluster.py

@@ -0,0 +1,24 b''
+c = get_config()
+
+# This can be used at any point in a config file to load a sub config
+# and merge it into the current one.
+load_subconfig('ipython_config.py')
+
+lines = """
+from IPython.kernel.client import *
+"""
+
+# You have to make sure that attributes that are containers already
+# exist before using them.  Simple assigning a new list will override
+# all previous values.
+if hasattr(c.Global, 'exec_lines'):
+    c.Global.exec_lines.append(lines)
+else:
+    c.Global.exec_lines = [lines]
+
+# Load the parallelmagic extension to enable %result, %px, %autopx magics.
+if hasattr(c.Global, 'extensions'):
+    c.Global.extensions.append('parallelmagic')
+else:
+    c.Global.extensions = ['parallelmagic']
+

IPython/core/pylabtools.py

@@ -0,0 +1,145 b''
+# -*- coding: utf-8 -*-
+"""Pylab (matplotlib) support utilities.
+
+Authors
+-------
+Fernando Perez.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from IPython.utils.genutils import flag_calls
+
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
+
+def pylab_activate(user_ns, gui=None, import_all=True):
+    """Activate pylab mode in the user's namespace.
+
+    Loads and initializes numpy, matplotlib and friends for interactive use.
+
+    Parameters
+    ----------
+    user_ns : dict
+      Namespace where the imports will occur.
+
+    gui : optional, string
+      A valid gui name following the conventions of the %gui magic.
+
+    import_all : optional, boolean
+      If true, an 'import *' is done from numpy and pylab.
+
+    Returns
+    -------
+    The actual gui used (if not given as input, it was obtained from matplotlib
+    itself, and will be needed next to configure IPython's gui integration.
+    """
+
+    # Initialize matplotlib to interactive mode always
+    import matplotlib
+
+    # If user specifies a GUI, that dictates the backend, otherwise we read the
+    # user's mpl default from the mpl rc structure
+    g2b = {'tk': 'TkAgg',
+           'gtk': 'GTKAgg',
+           'wx': 'WXAgg',
+           'qt': 'Qt4Agg', # qt3 not supported
+           'qt4': 'Qt4Agg' }
+
+    if gui:
+        # select backend based on requested gui
+        backend = g2b[gui]
+    else:
+        backend = matplotlib.rcParams['backend']
+        # In this case, we need to find what the appropriate gui selection call
+        # should be for IPython, so we can activate inputhook accordingly
+        b2g = dict(zip(g2b.values(),g2b.keys()))
+        gui = b2g[backend]
+
+    # We must set the desired backend before importing pylab
+    matplotlib.use(backend)
+    
+    # This must be imported last in the matplotlib series, after
+    # backend/interactivity choices have been made
+    import matplotlib.pylab as pylab
+
+    # XXX For now leave this commented out, but depending on discussions with
+    # mpl-dev, we may be able to allow interactive switching...
+    #import matplotlib.pyplot
+    #matplotlib.pyplot.switch_backend(backend)
+
+    pylab.show._needmain = False
+    # We need to detect at runtime whether show() is called by the user.
+    # For this, we wrap it into a decorator which adds a 'called' flag.
+    pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)
+
+    # Import numpy as np/pyplot as plt are conventions we're trying to
+    # somewhat standardize on.  Making them available to users by default
+    # will greatly help this. 
+    exec ("import numpy\n"
+          "import matplotlib\n"
+          "from matplotlib import pylab, mlab, pyplot\n"
+          "np = numpy\n"
+          "plt = pyplot\n"
+          ) in user_ns
+
+    if import_all:
+        exec("from matplotlib.pylab import *\n"
+             "from numpy import *\n") in user_ns
+
+    matplotlib.interactive(True)
+
+    print """
+Welcome to pylab, a matplotlib-based Python environment [backend: %s].
+For more information, type 'help(pylab)'.""" % backend
+    
+    return gui
+
+# We need a little factory function here to create the closure where
+# safe_execfile can live.
+def mpl_runner(safe_execfile):
+    """Factory to return a matplotlib-enabled runner for %run.
+
+    Parameters
+    ----------
+    safe_execfile : function
+      This must be a function with the same interface as the
+      :meth:`safe_execfile` method of IPython.
+
+    Returns
+    -------
+    A function suitable for use as the ``runner`` argument of the %run magic
+    function.
+    """
+    
+    def mpl_execfile(fname,*where,**kw):
+        """matplotlib-aware wrapper around safe_execfile.
+
+        Its interface is identical to that of the :func:`execfile` builtin.
+
+        This is ultimately a call to execfile(), but wrapped in safeties to
+        properly handle interactive rendering."""
+
+        import matplotlib
+        import matplotlib.pylab as pylab
+
+        #print '*** Matplotlib runner ***' # dbg
+        # turn off rendering until end of script
+        is_interactive = matplotlib.rcParams['interactive']
+        matplotlib.interactive(False)
+        safe_execfile(fname,*where,**kw)
+        matplotlib.interactive(is_interactive)
+        # make rendering call now, if the user tried to do it
+        if pylab.draw_if_interactive.called:
+            pylab.draw()
+            pylab.draw_if_interactive.called = False
+
+    return mpl_execfile

IPython/core/tests/simpleerr.py

@@ -0,0 +1,32 b''
+"""Error script. DO NOT EDIT FURTHER! It will break exception doctests!!!"""
+import sys
+
+def div0():
+    "foo"
+    x = 1
+    y = 0
+    x/y
+
+def sysexit(stat, mode):
+    raise SystemExit(stat, 'Mode = %s' % mode)
+
+def bar(mode):
+    "bar"
+    if mode=='div':
+        div0()
+    elif mode=='exit':
+        try:
+            stat = int(sys.argv[2])
+        except:
+            stat = 1
+        sysexit(stat, mode)
+    else:
+        raise ValueError('Unknown mode')
+
+if __name__ == '__main__':
+    try:
+        mode = sys.argv[1]
+    except IndexError:
+        mode = 'div'
+
+    bar(mode)

IPython/core/tests/test_completer.py

@@ -0,0 +1,35 b''
+"""Tests for the IPython tab-completion machinery.
+"""
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import sys
+
+# third party
+import nose.tools as nt
+
+# our own packages
+from IPython.core import completer
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+def test_protect_filename():
+    pairs = [ ('abc','abc'),
+              (' abc',r'\ abc'),
+              ('a bc',r'a\ bc'),
+              ('a  bc',r'a\ \ bc'),
+              ('  bc',r'\ \ bc'),
+              ]
+    # On posix, we also protect parens
+    if sys.platform != 'win32':
+        pairs.extend( [('a(bc',r'a\(bc'),
+                       ('a)bc',r'a\)bc'),
+                       ('a( )bc',r'a\(\ \)bc'),
+                       ] )
+    # run the actual tests
+    for s1, s2 in pairs:
+        s1p = completer.protect_filename(s1)
+        nt.assert_equals(s1p, s2)

IPython/core/tests/test_prefilter.py

@@ -0,0 +1,34 b''
+"""Tests for input manipulation machinery."""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import nose.tools as nt
+
+from IPython.testing import tools as tt, decorators as dec
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
[e-mail]
+def test_prefilter():
+    """Test user input conversions"""
+
+    # pairs of (raw, expected correct) input
+    pairs = [ ('2+2','2+2'),
+              ('>>> 2+2','2+2'),
+              ('>>> # This is a comment\n'
+               '... 2+2',
+               '# This is a comment\n'
+               '2+2'),
+              # Some IPython input
+              ('In [1]: 1', '1'),
+              ('In [2]: for i in range(5):\n'
+               '   ...:     print i,',
+               'for i in range(5):\n'
+               '    print i,'),
+             ]
+
+    ip = get_ipython()
+    for raw, correct in pairs:
+        yield nt.assert_equals(ip.prefilter(raw), correct)

IPython/core/tests/test_run.py

@@ -0,0 +1,174 b''
+"""Tests for code execution (%run and related), which is particularly tricky.
+
+Because of how %run manages namespaces, and the fact that we are trying here to
+verify subtle object deletion and reference counting issues, the %run tests
+will be kept in this separate file.  This makes it easier to aggregate in one
+place the tricks needed to handle it; most other magics are much easier to test
+and we do so in a common test_magic file.
+"""
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import os
+import sys
+import tempfile
+
+# third-party
+import nose.tools as nt
+
+# our own
+from IPython.utils.platutils import find_cmd
+from IPython.utils import genutils
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
+def doctest_refbug():
+    """Very nasty problem with references held by multiple runs of a script.
+    See: https://bugs.launchpad.net/ipython/+bug/269966
+
+    In [1]: _ip.clear_main_mod_cache()
+    # random
+    
+    In [2]: %run refbug
+
+    In [3]: call_f()
+    lowercased: hello
+
+    In [4]: %run refbug
+
+    In [5]: call_f()
+    lowercased: hello
+    lowercased: hello
+    """
+
+
+def doctest_run_builtins():
+    r"""Check that %run doesn't damage __builtins__.
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: fname = tempfile.mkstemp('.py')[1]
+
+    In [3]: f = open(fname,'w')
+
+    In [4]: f.write('pass\n')
+
+    In [5]: f.flush()
+
+    In [6]: t1 = type(__builtins__)
+
+    In [7]: %run "$fname"
+
+    In [7]: f.close()
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: t2 = type(__builtins__)
+
+    In [10]: t1 == t2
+    Out[10]: True
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+
+    In [12]: try:
+       ....:     os.unlink(fname)
+       ....: except:
+       ....:     pass
+       ....: 
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TestMagicRunPass(tt.TempFileMixin):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        self.mktmp('pass\n')
+
+    def run_tmpfile(self):
+        _ip = get_ipython()
+        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
+        # See below and ticket https://bugs.launchpad.net/bugs/366353
+        _ip.magic('run "%s"' % self.fname)
+
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+        _ip = get_ipython()
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        _ip = get_ipython()
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        _ip = get_ipython()
+        p2 = str(_ip.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+
+class TestMagicRunSimple(tt.TempFileMixin):
+
+    def test_simpledef(self):
+        """Test that simple class definitions work."""
+        src = ("class foo: pass\n"
+               "def f(): return foo()")
+        self.mktmp(src)
+        _ip.magic('run %s' % self.fname)
+        _ip.runlines('t = isinstance(f(), foo)')
+        nt.assert_true(_ip.user_ns['t'])
+
+    # We have to skip these in win32 because genutils.getoutputerr() crashes,
+    # due to the fact that subprocess does not support close_fds when
+    # redirecting stdout/err.  So unless someone who knows more tells us how to
+    # implement genutils.getoutputerr() in win32, we're stuck avoiding these.
+    @dec.skip_win32
+    def test_obj_del(self):
+        """Test that object's __del__ methods are called on exit."""
+        
+        # This test is known to fail on win32.
+        # See ticket https://bugs.launchpad.net/bugs/366334
+        src = ("class A(object):\n"
+               "    def __del__(self):\n"
+               "        print 'object A deleted'\n"
+               "a = A()\n")
+        self.mktmp(src)
+        tt.ipexec_validate(self.fname, 'object A deleted')
+
+    @dec.skip_win32
+    def test_tclass(self):
+        mydir = os.path.dirname(__file__)
+        tc = os.path.join(mydir, 'tclass')
+        src = ("%%run '%s' C-first\n"
+               "%%run '%s' C-second\n") % (tc, tc)
+        self.mktmp(src, '.ipy')
+        out = """\
+ARGV 1-: ['C-first']
+ARGV 1-: ['C-second']
+tclass.py: deleting object: C-first
+"""
+        tt.ipexec_validate(self.fname, out)

IPython/extensions/parallelmagic.py

@@ -0,0 +1,209 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""Magic command interface for interactive parallel work."""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import new
+
+from IPython.core.component import Component
+from IPython.utils.traitlets import Bool, Any
+from IPython.utils.autoattr import auto_attr
+from IPython.testing import decorators as testdec
+
+#-----------------------------------------------------------------------------
+# Definitions of magic functions for use with IPython
+#-----------------------------------------------------------------------------
+
+
+NO_ACTIVE_MULTIENGINE_CLIENT = """
+Use activate() on a MultiEngineClient object to activate it for magics.
+"""
+
+
+class ParalleMagicComponent(Component):
+    """A component to manage the %result, %px and %autopx magics."""
+
+    active_multiengine_client = Any()
+    verbose = Bool(False, config=True)
+
+    def __init__(self, parent, name=None, config=None):
+        super(ParalleMagicComponent, self).__init__(parent, name=name, config=config)
+        self._define_magics()
+        # A flag showing if autopx is activated or not
+        self.autopx = False
+
+    # Access other components like this rather than by a regular attribute.
+    # This won't lookup the InteractiveShell object until it is used and
+    # then it is cached.  This is both efficient and couples this class 
+    # more loosely to InteractiveShell.
+    @auto_attr
+    def shell(self):
+        return Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell')[0]
+
+    def _define_magics(self):
+        """Define the magic functions."""
+        self.shell.define_magic('result', self.magic_result)
+        self.shell.define_magic('px', self.magic_px)
+        self.shell.define_magic('autopx', self.magic_autopx)
+
+    @testdec.skip_doctest
+    def magic_result(self, ipself, parameter_s=''):
+        """Print the result of command i on all engines..
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method.
+
+        Then you can do the following::
+
+            In [23]: %result
+            Out[23]: 
+            <Results List>
+            [0] In [6]: a = 10
+            [1] In [6]: a = 10
+    
+            In [22]: %result 6
+            Out[22]: 
+            <Results List>
+            [0] In [6]: a = 10
+            [1] In [6]: a = 10
+        """
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+
+        try:
+            index = int(parameter_s)
+        except:
+            index = None
+        result = self.active_multiengine_client.get_result(index)
+        return result
+
+    @testdec.skip_doctest
+    def magic_px(self, ipself, parameter_s=''):
+        """Executes the given python command in parallel.
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method.
+        
+        Then you can do the following::
+
+            In [24]: %px a = 5
+            Parallel execution on engines: all
+            Out[24]: 
+            <Results List>
+            [0] In [7]: a = 5
+            [1] In [7]: a = 5
+        """
+
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+        print "Parallel execution on engines: %s" % self.active_multiengine_client.targets
+        result = self.active_multiengine_client.execute(parameter_s)
+        return result
+
+    @testdec.skip_doctest
+    def magic_autopx(self, ipself, parameter_s=''):
+        """Toggles auto parallel mode.
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method. Once this
+        is called, all commands typed at the command line are send to
+        the engines to be executed in parallel. To control which engine
+        are used, set the ``targets`` attributed of the multiengine client
+        before entering ``%autopx`` mode.
+
+        Then you can do the following::
+
+            In [25]: %autopx
+            %autopx to enabled
+
+            In [26]: a = 10
+            <Results List>
+            [0] In [8]: a = 10
+            [1] In [8]: a = 10
+
+
+            In [27]: %autopx
+            %autopx disabled
+        """
+        if self.autopx:
+            self._disable_autopx()
+        else:
+            self._enable_autopx()
+
+    def _enable_autopx(self):
+        """Enable %autopx mode by saving the original runsource and installing 
+        pxrunsource.
+        """
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+
+        self._original_runsource = self.shell.runsource
+        self.shell.runsource = new.instancemethod(
+            self.pxrunsource, self.shell, self.shell.__class__
+        )
+        self.autopx = True
+        print "%autopx enabled"
+    
+    def _disable_autopx(self):
+        """Disable %autopx by restoring the original InteractiveShell.runsource."""
+        if self.autopx:
+            self.shell.runsource = self._original_runsource
+            self.autopx = False
+            print "%autopx disabled"
+
+    def pxrunsource(self, ipself, source, filename="<input>", symbol="single"):
+        """A parallel replacement for InteractiveShell.runsource."""
+
+        try:
+            code = ipself.compile(source, filename, symbol)
+        except (OverflowError, SyntaxError, ValueError):
+            # Case 1
+            ipself.showsyntaxerror(filename)
+            return None
+
+        if code is None:
+            # Case 2
+            return True
+
+        # Case 3
+        # Because autopx is enabled, we now call executeAll or disable autopx if
+        # %autopx or autopx has been called
+        if 'get_ipython().magic("%autopx' in source or 'get_ipython().magic("autopx' in source:
+            self._disable_autopx()
+            return False
+        else:
+            try:
+                result = self.active_multiengine_client.execute(source)
+            except:
+                ipself.showtraceback()
+            else:
+                print result.__repr__()
+            return False
+
+
+_loaded = False
+
+
+def load_ipython_extension(ip):
+    """Load the extension in IPython."""
+    global _loaded
+    if not _loaded:
+        prd = ParalleMagicComponent(ip, name='parallel_magic')
+        _loaded = True
+

IPython/external/_numpy_testing_utils.py

@@ -0,0 +1,120 b''
+# IPython: modified copy of numpy.testing.utils, so numpy.testing.decorators
+# works without numpy being installed.
+"""
+Utility function to facilitate testing.
+"""
+
+import os
+import sys
+import re
+import operator
+import types
+import warnings
+
+# The following two classes are copied from python 2.6 warnings module (context
+# manager)
+class WarningMessage(object):
+
+    """
+    Holds the result of a single showwarning() call.
+
+    Notes
+    -----
+    `WarningMessage` is copied from the Python 2.6 warnings module,
+    so it can be used in NumPy with older Python versions.
+
+    """
+
+    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+                        "line")
+
+    def __init__(self, message, category, filename, lineno, file=None,
+                    line=None):
+        local_values = locals()
+        for attr in self._WARNING_DETAILS:
+            setattr(self, attr, local_values[attr])
+        if category:
+            self._category_name = category.__name__
+        else:
+            self._category_name = None
+
+    def __str__(self):
+        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
+                    "line : %r}" % (self.message, self._category_name,
+                                    self.filename, self.lineno, self.line))
+
+class WarningManager:
+    """
+    A context manager that copies and restores the warnings filter upon
+    exiting the context.
+
+    The 'record' argument specifies whether warnings should be captured by a
+    custom implementation of ``warnings.showwarning()`` and be appended to a
+    list returned by the context manager. Otherwise None is returned by the
+    context manager. The objects appended to the list are arguments whose
+    attributes mirror the arguments to ``showwarning()``.
+
+    The 'module' argument is to specify an alternative module to the module
+    named 'warnings' and imported under that name. This argument is only useful
+    when testing the warnings module itself.
+
+    Notes
+    -----
+    `WarningManager` is a copy of the ``catch_warnings`` context manager
+    from the Python 2.6 warnings module, with slight modifications.
+    It is copied so it can be used in NumPy with older Python versions.
+
+    """
+    def __init__(self, record=False, module=None):
+        self._record = record
+        if module is None:
+            self._module = sys.modules['warnings']
+        else:
+            self._module = module
+        self._entered = False
+
+    def __enter__(self):
+        if self._entered:
+            raise RuntimeError("Cannot enter %r twice" % self)
+        self._entered = True
+        self._filters = self._module.filters
+        self._module.filters = self._filters[:]
+        self._showwarning = self._module.showwarning
+        if self._record:
+            log = []
+            def showwarning(*args, **kwargs):
+                log.append(WarningMessage(*args, **kwargs))
+            self._module.showwarning = showwarning
+            return log
+        else:
+            return None
+
+    def __exit__(self):
+        if not self._entered:
+            raise RuntimeError("Cannot exit %r without entering first" % self)
+        self._module.filters = self._filters
+        self._module.showwarning = self._showwarning
+
+def assert_warns(warning_class, func, *args, **kw):
+    """Fail unless a warning of class warning_class is thrown by callable when
+    invoked with arguments args and keyword arguments kwargs.
+    
+    If a different type of warning is thrown, it will not be caught, and the
+    test case will be deemed to have suffered an error.
+    """
+
+    # XXX: once we may depend on python >= 2.6, this can be replaced by the
+    # warnings module context manager.
+    ctx = WarningManager(record=True)
+    l = ctx.__enter__()
+    warnings.simplefilter('always')
+    try:
+        func(*args, **kw)
+        if not len(l) > 0:
+            raise AssertionError("No warning raised when calling %s"
+                    % func.__name__)
+        if not l[0].category is warning_class:
+            raise AssertionError("First warning for %s is not a " \
+                    "%s( is %s)" % (func.__name__, warning_class, l[0]))
+    finally:
+        ctx.__exit__()

IPython/external/decorators.py

@@ -0,0 +1,284 b''
+"""
+Decorators for labeling and modifying behavior of test objects.
+
+Decorators that merely return a modified version of the original
+function object are straightforward. Decorators that return a new
+function object need to use
+::
+
+  nose.tools.make_decorator(original_function)(decorator)
+
+in returning the decorator, in order to preserve meta-data such as
+function name, setup and teardown functions and so on - see
+``nose.tools`` for more information.
+
+"""
+import warnings
+import sys
+
+# IPython changes: make this work if numpy not available
+# Original code:
+#from numpy.testing.utils import \
+#        WarningManager, WarningMessage
+# Our version:
+try:
+    from numpy.testing.utils import WarningManager, WarningMessage
+except ImportError:
+    from _numpy_testing_utils import WarningManager, WarningMessage
+    
+# End IPython changes
+
+def slow(t):
+    """
+    Label a test as 'slow'.
+
+    The exact definition of a slow test is obviously both subjective and
+    hardware-dependent, but in general any individual test that requires more
+    than a second or two should be labeled as slow (the whole suite consits of
+    thousands of tests, so even a second is significant).
+
+    Parameters
+    ----------
+    t : callable
+        The test to label as slow.
+
+    Returns
+    -------
+    t : callable
+        The decorated test `t`.
+
+    Examples
+    --------
+    The `numpy.testing` module includes ``import decorators as dec``.
+    A test can be decorated as slow like this::
+
+      from numpy.testing import *
+
+      @dec.slow
+      def test_big(self):
+          print 'Big, slow test'
+
+    """
+
+    t.slow = True
+    return t
+
+def setastest(tf=True):
+    """
+    Signals to nose that this function is or is not a test.
+
+    Parameters
+    ----------
+    tf : bool
+        If True, specifies that the decorated callable is a test.
+        If False, specifies that the decorated callable is not a test.
+        Default is True.
+
+    Notes
+    -----
+    This decorator can't use the nose namespace, because it can be
+    called from a non-test module. See also ``istest`` and ``nottest`` in
+    ``nose.tools``.
+
+    Examples
+    --------
+    `setastest` can be used in the following way::
+
+      from numpy.testing.decorators import setastest
+
+      @setastest(False)
+      def func_with_test_in_name(arg1, arg2):
+          pass
+
+    """
+    def set_test(t):
+        t.__test__ = tf
+        return t
+    return set_test
+
+def skipif(skip_condition, msg=None):
+    """
+    Make function raise SkipTest exception if a given condition is true.
+
+    If the condition is a callable, it is used at runtime to dynamically
+    make the decision. This is useful for tests that may require costly
+    imports, to delay the cost until the test suite is actually executed.
+
+    Parameters
+    ----------
+    skip_condition : bool or callable
+        Flag to determine whether to skip the decorated test.
+    msg : str, optional
+        Message to give on raising a SkipTest exception. Default is None.
+
+    Returns
+    -------
+    decorator : function
+        Decorator which, when applied to a function, causes SkipTest
+        to be raised when `skip_condition` is True, and the function
+        to be called normally otherwise.
+
+    Notes
+    -----
+    The decorator itself is decorated with the ``nose.tools.make_decorator``
+    function in order to transmit function name, and various other metadata.
+
+    """
+
+    def skip_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+
+        # Allow for both boolean or callable skip conditions.
+        if callable(skip_condition):
+            skip_val = lambda : skip_condition()
+        else:
+            skip_val = lambda : skip_condition
+
+        def get_msg(func,msg=None):
+            """Skip message with information about function being skipped."""
+            if msg is None: 
+                out = 'Test skipped due to test condition'
+            else: 
+                out = '\n'+msg
+
+            return "Skipping test: %s%s" % (func.__name__,out)
+
+        # We need to define *two* skippers because Python doesn't allow both
+        # return with value and yield inside the same function.
+        def skipper_func(*args, **kwargs):
+            """Skipper for normal test functions."""
+            if skip_val():
+                raise nose.SkipTest(get_msg(f,msg))
+            else:
+                return f(*args, **kwargs)
+
+        def skipper_gen(*args, **kwargs):
+            """Skipper for test generators."""
+            if skip_val():
+                raise nose.SkipTest(get_msg(f,msg))
+            else:
+                for x in f(*args, **kwargs):
+                    yield x
+
+        # Choose the right skipper to use when building the actual decorator.
+        if nose.util.isgenerator(f):
+            skipper = skipper_gen
+        else:
+            skipper = skipper_func
+            
+        return nose.tools.make_decorator(f)(skipper)
+
+    return skip_decorator
+
+
+def knownfailureif(fail_condition, msg=None):
+    """
+    Make function raise KnownFailureTest exception if given condition is true.
+
+    If the condition is a callable, it is used at runtime to dynamically
+    make the decision. This is useful for tests that may require costly
+    imports, to delay the cost until the test suite is actually executed.
+
+    Parameters
+    ----------
+    fail_condition : bool or callable
+        Flag to determine whether to mark the decorated test as a known
+        failure (if True) or not (if False).
+    msg : str, optional
+        Message to give on raising a KnownFailureTest exception.
+        Default is None.
+
+    Returns
+    -------
+    decorator : function
+        Decorator, which, when applied to a function, causes SkipTest
+        to be raised when `skip_condition` is True, and the function
+        to be called normally otherwise.
+
+    Notes
+    -----
+    The decorator itself is decorated with the ``nose.tools.make_decorator``
+    function in order to transmit function name, and various other metadata.
+
+    """
+    if msg is None:
+        msg = 'Test skipped due to known failure'
+
+    # Allow for both boolean or callable known failure conditions.
+    if callable(fail_condition):
+        fail_val = lambda : fail_condition()
+    else:
+        fail_val = lambda : fail_condition
+
+    def knownfail_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+        from noseclasses import KnownFailureTest
+        def knownfailer(*args, **kwargs):
+            if fail_val():
+                raise KnownFailureTest, msg
+            else:
+                return f(*args, **kwargs)
+        return nose.tools.make_decorator(f)(knownfailer)
+
+    return knownfail_decorator
+
+def deprecated(conditional=True):
+    """
+    Filter deprecation warnings while running the test suite.
+
+    This decorator can be used to filter DeprecationWarning's, to avoid
+    printing them during the test suite run, while checking that the test
+    actually raises a DeprecationWarning.
+
+    Parameters
+    ----------
+    conditional : bool or callable, optional
+        Flag to determine whether to mark test as deprecated or not. If the
+        condition is a callable, it is used at runtime to dynamically make the
+        decision. Default is True.
+
+    Returns
+    -------
+    decorator : function
+        The `deprecated` decorator itself.
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    """
+    def deprecate_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+        from noseclasses import KnownFailureTest
+
+        def _deprecated_imp(*args, **kwargs):
+            # Poor man's replacement for the with statement
+            ctx = WarningManager(record=True)
+            l = ctx.__enter__()
+            warnings.simplefilter('always')
+            try:
+                f(*args, **kwargs)
+                if not len(l) > 0:
+                    raise AssertionError("No warning raised when calling %s"
+                            % f.__name__)
+                if not l[0].category is DeprecationWarning:
+                    raise AssertionError("First warning for %s is not a " \
+                            "DeprecationWarning( is %s)" % (f.__name__, l[0]))
+            finally:
+                ctx.__exit__()
+
+        if callable(conditional):
+            cond = conditional()
+        else:
+            cond = conditional
+        if cond:
+            return nose.tools.make_decorator(f)(_deprecated_imp)
+        else:
+            return f
+    return deprecate_decorator

IPython/kernel/clusterdir.py

@@ -0,0 +1,450 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython cluster directory
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import shutil
+import sys
+
+from twisted.python import log
+
+from IPython.core import release
+from IPython.config.loader import PyFileConfigLoader
+from IPython.core.application import Application
+from IPython.core.component import Component
+from IPython.utils.traitlets import Unicode, Bool
+from IPython.utils import genutils
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+class ClusterDirError(Exception):
+    pass
+
+
+class PIDFileError(Exception):
+    pass
+
+
+class ClusterDir(Component):
+    """An object to manage the cluster directory and its resources.
+
+    The cluster directory is used by :command:`ipcontroller`, 
+    :command:`ipcontroller` and :command:`ipcontroller` to manage the
+    configuration, logging and security of these applications.
+
+    This object knows how to find, create and manage these directories. This
+    should be used by any code that want's to handle cluster directories.
+    """
+
+    security_dir_name = Unicode('security')
+    log_dir_name = Unicode('log')
+    pid_dir_name = Unicode('pid')
+    security_dir = Unicode(u'')
+    log_dir = Unicode(u'')
+    pid_dir = Unicode(u'')
+    location = Unicode(u'')
+
+    def __init__(self, location):
+        super(ClusterDir, self).__init__(None)
+        self.location = location
+
+    def _location_changed(self, name, old, new):
+        if not os.path.isdir(new):
+            os.makedirs(new)
+        self.security_dir = os.path.join(new, self.security_dir_name)
+        self.log_dir = os.path.join(new, self.log_dir_name)
+        self.pid_dir = os.path.join(new, self.pid_dir_name)
+        self.check_dirs()
+
+    def _log_dir_changed(self, name, old, new):
+        self.check_log_dir()
+
+    def check_log_dir(self):
+        if not os.path.isdir(self.log_dir):
+            os.mkdir(self.log_dir)
+
+    def _security_dir_changed(self, name, old, new):
+        self.check_security_dir()
+
+    def check_security_dir(self):
+        if not os.path.isdir(self.security_dir):
+            os.mkdir(self.security_dir, 0700)
+        os.chmod(self.security_dir, 0700)
+
+    def _pid_dir_changed(self, name, old, new):
+        self.check_pid_dir()
+
+    def check_pid_dir(self):
+        if not os.path.isdir(self.pid_dir):
+            os.mkdir(self.pid_dir, 0700)
+        os.chmod(self.pid_dir, 0700)
+
+    def check_dirs(self):
+        self.check_security_dir()
+        self.check_log_dir()
+        self.check_pid_dir()
+
+    def load_config_file(self, filename):
+        """Load a config file from the top level of the cluster dir.
+
+        Parameters
+        ----------
+        filename : unicode or str
+            The filename only of the config file that must be located in
+            the top-level of the cluster directory.
+        """
+        loader = PyFileConfigLoader(filename, self.location)
+        return loader.load_config()
+
+    def copy_config_file(self, config_file, path=None, overwrite=False):
+        """Copy a default config file into the active cluster directory.
+
+        Default configuration files are kept in :mod:`IPython.config.default`.
+        This function moves these from that location to the working cluster
+        directory.
+        """
+        if path is None:
+            import IPython.config.default
+            path = IPython.config.default.__file__.split(os.path.sep)[:-1]
+            path = os.path.sep.join(path)
+        src = os.path.join(path, config_file)
+        dst = os.path.join(self.location, config_file)
+        if not os.path.isfile(dst) or overwrite:
+            shutil.copy(src, dst)
+
+    def copy_all_config_files(self, path=None, overwrite=False):
+        """Copy all config files into the active cluster directory."""
+        for f in [u'ipcontroller_config.py', u'ipengine_config.py',
+                  u'ipcluster_config.py']:
+            self.copy_config_file(f, path=path, overwrite=overwrite)
+
+    @classmethod
+    def create_cluster_dir(csl, cluster_dir):
+        """Create a new cluster directory given a full path.
+
+        Parameters
+        ----------
+        cluster_dir : str
+            The full path to the cluster directory.  If it does exist, it will
+            be used.  If not, it will be created.
+        """
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def create_cluster_dir_by_profile(cls, path, profile=u'default'):
+        """Create a cluster dir by profile name and path.
+
+        Parameters
+        ----------
+        path : str
+            The path (directory) to put the cluster directory in.
+        profile : str
+            The name of the profile.  The name of the cluster directory will
+            be "cluster_<profile>".
+        """
+        if not os.path.isdir(path):
+            raise ClusterDirError('Directory not found: %s' % path)
+        cluster_dir = os.path.join(path, u'cluster_' + profile)
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
+        """Find an existing cluster dir by profile name, return its ClusterDir.
+
+        This searches through a sequence of paths for a cluster dir.  If it
+        is not found, a :class:`ClusterDirError` exception will be raised.
+
+        The search path algorithm is:
+        1. ``os.getcwd()``
+        2. ``ipython_dir``
+        3. The directories found in the ":" separated 
+           :env:`IPCLUSTER_DIR_PATH` environment variable.
+
+        Parameters
+        ----------
+        ipython_dir : unicode or str
+            The IPython directory to use.
+        profile : unicode or str
+            The name of the profile.  The name of the cluster directory
+            will be "cluster_<profile>".
+        """
+        dirname = u'cluster_' + profile
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
+        for p in paths:
+            cluster_dir = os.path.join(p, dirname)
+            if os.path.isdir(cluster_dir):
+                return ClusterDir(cluster_dir)
+        else:
+            raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
+
+    @classmethod
+    def find_cluster_dir(cls, cluster_dir):
+        """Find/create a cluster dir and return its ClusterDir.
+
+        This will create the cluster directory if it doesn't exist.
+
+        Parameters
+        ----------
+        cluster_dir : unicode or str
+            The path of the cluster directory.  This is expanded using
+            :func:`IPython.utils.genutils.expand_path`.
+        """
+        cluster_dir = genutils.expand_path(cluster_dir)
+        if not os.path.isdir(cluster_dir):
+            raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
+        return ClusterDir(cluster_dir)
+
+
+# Default command line options for IPython cluster applications.
+cl_args = (
+    (('--ipython-dir',), dict(
+        dest='Global.ipython_dir',type=unicode,
+        help='Set to override default location of Global.ipython_dir.',
+        metavar='Global.ipython_dir') ),
+    (('-p', '--profile',), dict(
+        dest='Global.profile',type=unicode,
+        help=
+        """The string name of the profile to be used. This determines the name
+        of the cluster dir as: cluster_<profile>. The default profile is named
+        'default'.  The cluster directory is resolve this way if the
+        --cluster-dir option is not used.""",
+        metavar='Global.profile') ),
+    (('--cluster-dir',), dict(
+        dest='Global.cluster_dir',type=unicode,
+        help="""Set the cluster dir. This overrides the logic used by the
+        --profile option.""",
+        metavar='Global.cluster_dir') ),
+    (('--work-dir',), dict(
+      dest='Global.work_dir',type=unicode,
+      help='Set the working dir for the process.',
+      metavar='Global.work_dir') ),
+    (('--clean-logs',), dict(
+      dest='Global.clean_logs', action='store_true',
+      help='Delete old log flies before starting.') ),
+    (('--no-clean-logs',), dict(
+      dest='Global.clean_logs', action='store_false',
+      help="Don't Delete old log flies before starting.") ),
+    )
+
+
+class ApplicationWithClusterDir(Application):
+    """An application that puts everything into a cluster directory.
+
+    Instead of looking for things in the ipython_dir, this type of application
+    will use its own private directory called the "cluster directory"
+    for things like config files, log files, etc.
+
+    The cluster directory is resolved as follows:
+
+    * If the ``--cluster-dir`` option is given, it is used.
+    * If ``--cluster-dir`` is not given, the application directory is 
+      resolve using the profile name as ``cluster_<profile>``. The search 
+      path for this directory is then i) cwd if it is found there
+      and ii) in ipython_dir otherwise.
+
+    The config file for the application is to be put in the cluster
+    dir and named the value of the ``config_file_name`` class attribute.
+    """
+
+    auto_create_cluster_dir = True
+
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(ApplicationWithClusterDir, self).create_default_config()
+        self.default_config.Global.profile = u'default'
+        self.default_config.Global.cluster_dir = u''
+        self.default_config.Global.work_dir = os.getcwd()
+        self.default_config.Global.log_to_file = False
+        self.default_config.Global.clean_logs = False
+
+    def find_resources(self):
+        """This resolves the cluster directory.
+
+        This tries to find the cluster directory and if successful, it will
+        have done:
+        * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
+          the application.
+        * Sets ``self.cluster_dir`` attribute of the application and config
+          objects.
+
+        The algorithm used for this is as follows:
+        1. Try ``Global.cluster_dir``.
+        2. Try using ``Global.profile``.
+        3. If both of these fail and ``self.auto_create_cluster_dir`` is
+           ``True``, then create the new cluster dir in the IPython directory.
+        4. If all fails, then raise :class:`ClusterDirError`.
+        """
+
+        try:
+            cluster_dir = self.command_line_config.Global.cluster_dir
+        except AttributeError:
+            cluster_dir = self.default_config.Global.cluster_dir
+        cluster_dir = genutils.expand_path(cluster_dir)
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        try:
+            self.profile = self.command_line_config.Global.profile
+        except AttributeError:
+            self.profile = self.default_config.Global.profile
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                self.ipython_dir, self.profile)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        if self.auto_create_cluster_dir:
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                self.ipython_dir, self.profile
+            )
+            self.log.info('Creating new cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+        else:
+            raise ClusterDirError('Could not find a valid cluster directory.')
+
+    def finish_cluster_dir(self):
+        # Set the cluster directory
+        self.cluster_dir = self.cluster_dir_obj.location
+        
+        # These have to be set because they could be different from the one
+        # that we just computed.  Because command line has the highest
+        # priority, this will always end up in the master_config.
+        self.default_config.Global.cluster_dir = self.cluster_dir
+        self.command_line_config.Global.cluster_dir = self.cluster_dir
+
+        # Set the search path to the cluster directory
+        self.config_file_paths = (self.cluster_dir,)
+
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        # For this type of Application it should be set as a class attribute.
+        if not hasattr(self, 'config_file_name'):
+            self.log.critical("No config filename found")
+
+    def find_config_file_paths(self):
+        # Set the search path to the cluster directory
+        self.config_file_paths = (self.cluster_dir,)
+
+    def pre_construct(self):
+        # The log and security dirs were set earlier, but here we put them
+        # into the config and log them.
+        config = self.master_config
+        sdir = self.cluster_dir_obj.security_dir
+        self.security_dir = config.Global.security_dir = sdir
+        ldir = self.cluster_dir_obj.log_dir
+        self.log_dir = config.Global.log_dir = ldir
+        pdir = self.cluster_dir_obj.pid_dir
+        self.pid_dir = config.Global.pid_dir = pdir
+        self.log.info("Cluster directory set to: %s" % self.cluster_dir)
+        config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
+        # Change to the working directory. We do this just before construct
+        # is called so all the components there have the right working dir.
+        self.to_work_dir()
+
+    def to_work_dir(self):
+        wd = self.master_config.Global.work_dir
+        if unicode(wd) != unicode(os.getcwd()):
+            os.chdir(wd)
+            self.log.info("Changing to working dir: %s" % wd)
+
+    def start_logging(self):
+        # Remove old log files
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if f.startswith(self.name + u'-') and f.endswith('.log'):
+                    os.remove(os.path.join(log_dir, f))
+        # Start logging to the new log file
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        else:
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
+
+    def write_pid_file(self, overwrite=False):
+        """Create a .pid file in the pid_dir with my pid.
+
+        This must be called after pre_construct, which sets `self.pid_dir`.
+        This raises :exc:`PIDFileError` if the pid file exists already.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            pid = self.get_pid_from_file()
+            if not overwrite:
+                raise PIDFileError(
+                    'The pid file [%s] already exists. \nThis could mean that this '
+                    'server is already running with [pid=%s].' % (pid_file, pid)
+                )
+        with open(pid_file, 'w') as f:
+            self.log.info("Creating pid file: %s" % pid_file)
+            f.write(repr(os.getpid())+'\n')
+
+    def remove_pid_file(self):
+        """Remove the pid file.
+
+        This should be called at shutdown by registering a callback with
+        :func:`reactor.addSystemEventTrigger`. This needs to return
+        ``None``.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            try:
+                self.log.info("Removing pid file: %s" % pid_file)
+                os.remove(pid_file)
+            except:
+                self.log.warn("Error removing the pid file: %s" % pid_file)
+
+    def get_pid_from_file(self):
+        """Get the pid from the pid file.
+
+        If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            with open(pid_file, 'r') as f:
+                pid = int(f.read().strip())
+                return pid
+        else:
+            raise PIDFileError('pid file not found: %s' % pid_file)

IPython/kernel/configobjfactory.py

@@ -0,0 +1,79 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A class for creating a Twisted service that is configured using IPython's
+configuration system.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import zope.interface as zi
+
+from IPython.core.component import Component
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IConfiguredObjectFactory(zi.Interface):
+    """I am a component that creates a configured object.
+
+    This class is useful if you want to configure a class that is not a 
+    subclass of :class:`IPython.core.component.Component`.
+    """
+
+    def __init__(config):
+        """Get ready to configure the object using config."""
+
+    def create():
+        """Return an instance of the configured object."""
+
+
+class ConfiguredObjectFactory(Component):
+
+    zi.implements(IConfiguredObjectFactory)
+
+    def __init__(self, config):
+        super(ConfiguredObjectFactory, self).__init__(None, config=config)
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
+
+
+class IAdaptedConfiguredObjectFactory(zi.Interface):
+    """I am a component that adapts and configures an object.
+
+    This class is useful if you have the adapt an instance and configure it.
+    """
+
+    def __init__(config, adaptee=None):
+        """Get ready to adapt adaptee and then configure it using config."""
+
+    def create():
+        """Return an instance of the adapted and configured object."""
+
+
+class AdaptedConfiguredObjectFactory(Component):
+
+    # zi.implements(IAdaptedConfiguredObjectFactory)
+
+    def __init__(self, config, adaptee):
+        # print
+        # print "config pre:", config
+        super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config)
+        # print
+        # print "config post:", config
+        self.adaptee = adaptee
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
  No newline at end of file

IPython/kernel/ipclusterapp.py

@@ -0,0 +1,460 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The ipcluster application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import logging
+import os
+import signal
+
+if os.name=='posix':
+    from twisted.scripts._twistd_unix import daemonize
+
+from IPython.core import release
+from IPython.external.argparse import ArgumentParser
+from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
+from IPython.utils.importstring import import_item
+
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir, ClusterDirError, PIDFileError
+)
+
+from twisted.internet import reactor, defer
+from twisted.python import log, failure
+
+
+#-----------------------------------------------------------------------------
+# The ipcluster application
+#-----------------------------------------------------------------------------
+
+
+# Exit codes for ipcluster
+
+# This will be the exit code if the ipcluster appears to be running because
+# a .pid file exists
+ALREADY_STARTED = 10
+
+# This will be the exit code if ipcluster stop is run, but there is not .pid
+# file to be found.
+ALREADY_STOPPED = 11
+
+
+class IPClusterCLLoader(ArgParseConfigLoader):
+
+    def _add_other_arguments(self):
+        # This has all the common options that all subcommands use
+        parent_parser1 = ArgumentParser(add_help=False,
+                                        argument_default=NoConfigDefault)
+        parent_parser1.add_argument('--ipython-dir', 
+            dest='Global.ipython_dir',type=unicode,
+            help='Set to override default location of Global.ipython_dir.',
+            metavar='Global.ipython_dir')
+        parent_parser1.add_argument('--log-level',
+            dest="Global.log_level",type=int,
+            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
+            metavar='Global.log_level')
+
+        # This has all the common options that other subcommands use
+        parent_parser2 = ArgumentParser(add_help=False,
+                                        argument_default=NoConfigDefault)
+        parent_parser2.add_argument('-p','--profile',
+            dest='Global.profile',type=unicode,
+            help='The string name of the profile to be used. This determines '
+            'the name of the cluster dir as: cluster_<profile>. The default profile '
+            'is named "default".  The cluster directory is resolve this way '
+            'if the --cluster-dir option is not used.',
+            metavar='Global.profile')
+        parent_parser2.add_argument('--cluster-dir',
+            dest='Global.cluster_dir',type=unicode,
+            help='Set the cluster dir. This overrides the logic used by the '
+            '--profile option.',
+            metavar='Global.cluster_dir'),
+        parent_parser2.add_argument('--work-dir',
+            dest='Global.work_dir',type=unicode,
+            help='Set the working dir for the process.',
+            metavar='Global.work_dir')
+        parent_parser2.add_argument('--log-to-file',
+            action='store_true', dest='Global.log_to_file', 
+            help='Log to a file in the log directory (default is stdout)'
+        )
+
+        subparsers = self.parser.add_subparsers(
+            dest='Global.subcommand',
+            title='ipcluster subcommands',
+            description='ipcluster has a variety of subcommands. '
+            'The general way of running ipcluster is "ipcluster <cmd> '
+            ' [options]""',
+            help='For more help, type "ipcluster <cmd> -h"')
+
+        parser_list = subparsers.add_parser(
+            'list',
+            help='List all clusters in cwd and ipython_dir.',
+            parents=[parent_parser1]
+        )
+
+        parser_create = subparsers.add_parser(
+            'create',
+            help='Create a new cluster directory.',
+            parents=[parent_parser1, parent_parser2] 
+        )
+        parser_create.add_argument(
+            '--reset-config',
+            dest='Global.reset_config', action='store_true',
+            default=NoConfigDefault,
+            help='Recopy the default config files to the cluster directory. '
+            'You will loose any modifications you have made to these files.'
+        )
+
+        parser_start = subparsers.add_parser(
+            'start',
+            help='Start a cluster.',
+            parents=[parent_parser1, parent_parser2]
+        )
+        parser_start.add_argument(
+            '-n', '--number',
+            type=int, dest='Global.n',
+            help='The number of engines to start.',
+            metavar='Global.n'
+        )
+        parser_start.add_argument('--clean-logs',
+            dest='Global.clean_logs', action='store_true',
+            help='Delete old log flies before starting.',
+        )
+        parser_start.add_argument('--no-clean-logs',
+            dest='Global.clean_logs', action='store_false',
+            help="Don't delete old log flies before starting.",
+        )
+        parser_start.add_argument('--daemon',
+            dest='Global.daemonize', action='store_true',
+            help='Daemonize the ipcluster program. This implies --log-to-file',
+        )
+        parser_start.add_argument('--no-daemon',
+            dest='Global.daemonize', action='store_false',
+            help="Dont't daemonize the ipcluster program.",
+        )
+
+        parser_start = subparsers.add_parser(
+            'stop',
+            help='Stop a cluster.',
+            parents=[parent_parser1, parent_parser2]
+        )
+        parser_start.add_argument('--signal',
+            dest='Global.signal', type=int,
+            help="The signal number to use in stopping the cluster (default=2).",
+            metavar="Global.signal",
+        )
+
+
+default_config_file_name = u'ipcluster_config.py'
+
+
+_description = """Start an IPython cluster for parallel computing.\n\n
+
+An IPython cluster consists of 1 controller and 1 or more engines.
+This command automates the startup of these processes using a wide
+range of startup methods (SSH, local processes, PBS, mpiexec,
+Windows HPC Server 2008). To start a cluster with 4 engines on your
+local host simply do "ipcluster start -n 4". For more complex usage 
+you will typically do "ipcluster create -p mycluster", then edit
+configuration files, followed by "ipcluster start -p mycluster -n 4".
+"""
+
+
+class IPClusterApp(ApplicationWithClusterDir):
+
+    name = u'ipcluster'
+    description = _description
+    config_file_name = default_config_file_name
+    default_log_level = logging.INFO
+    auto_create_cluster_dir = False
+
+    def create_default_config(self):
+        super(IPClusterApp, self).create_default_config()
+        self.default_config.Global.controller_launcher = \
+            'IPython.kernel.launcher.LocalControllerLauncher'
+        self.default_config.Global.engine_launcher = \
+            'IPython.kernel.launcher.LocalEngineSetLauncher'
+        self.default_config.Global.n = 2
+        self.default_config.Global.reset_config = False
+        self.default_config.Global.clean_logs = True
+        self.default_config.Global.signal = 2
+        self.default_config.Global.daemonize = False
+
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+        return IPClusterCLLoader(
+            description=self.description, 
+            version=release.version
+        )
+
+    def find_resources(self):
+        subcommand = self.command_line_config.Global.subcommand
+        if subcommand=='list':
+            self.list_cluster_dirs()
+            # Exit immediately because there is nothing left to do.
+            self.exit()
+        elif subcommand=='create':
+            self.auto_create_cluster_dir = True
+            super(IPClusterApp, self).find_resources()
+        elif subcommand=='start' or subcommand=='stop':
+            self.auto_create_cluster_dir = True
+            try:
+                super(IPClusterApp, self).find_resources()
+            except ClusterDirError:
+                raise ClusterDirError(
+                    "Could not find a cluster directory. A cluster dir must "
+                    "be created before running 'ipcluster start'.  Do "
+                    "'ipcluster create -h' or 'ipcluster list -h' for more "
+                    "information about creating and listing cluster dirs."
+                )
+
+    def list_cluster_dirs(self):
+        # Find the search paths
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        try:
+            ipython_dir = self.command_line_config.Global.ipython_dir
+        except AttributeError:
+            ipython_dir = self.default_config.Global.ipython_dir
+        paths = [os.getcwd(), ipython_dir] + \
+            cluster_dir_paths
+        paths = list(set(paths))
+
+        self.log.info('Searching for cluster dirs in paths: %r' % paths)
+        for path in paths:
+            files = os.listdir(path)
+            for f in files:
+                full_path = os.path.join(path, f)
+                if os.path.isdir(full_path) and f.startswith('cluster_'):
+                    profile = full_path.split('_')[-1]
+                    start_cmd = 'ipcluster start -p %s -n 4' % profile
+                    print start_cmd + " ==> " + full_path
+
+    def pre_construct(self):
+        # IPClusterApp.pre_construct() is where we cd to the working directory.
+        super(IPClusterApp, self).pre_construct()
+        config = self.master_config
+        try:
+            daemon = config.Global.daemonize
+            if daemon:
+                config.Global.log_to_file = True
+        except AttributeError:
+            pass
+
+    def construct(self):
+        config = self.master_config
+        subcmd = config.Global.subcommand
+        reset = config.Global.reset_config
+        if subcmd == 'list':
+            return
+        if subcmd == 'create':
+            self.log.info('Copying default config files to cluster directory '
+            '[overwrite=%r]' % (reset,))
+            self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
+        if subcmd =='start':
+            self.cluster_dir_obj.copy_all_config_files(overwrite=False)
+            self.start_logging()
+            reactor.callWhenRunning(self.start_launchers)
+
+    def start_launchers(self):
+        config = self.master_config
+
+        # Create the launchers. In both bases, we set the work_dir of
+        # the launcher to the cluster_dir. This is where the launcher's
+        # subprocesses will be launched. It is not where the controller
+        # and engine will be launched.
+        el_class = import_item(config.Global.engine_launcher)
+        self.engine_launcher = el_class(
+            work_dir=self.cluster_dir, config=config
+        )
+        cl_class = import_item(config.Global.controller_launcher)
+        self.controller_launcher = cl_class(
+            work_dir=self.cluster_dir, config=config
+        )
+
+        # Setup signals
+        signal.signal(signal.SIGINT, self.sigint_handler)
+
+        # Setup the observing of stopping. If the controller dies, shut
+        # everything down as that will be completely fatal for the engines.
+        d1 = self.controller_launcher.observe_stop()
+        d1.addCallback(self.stop_launchers)
+        # But, we don't monitor the stopping of engines. An engine dying
+        # is just fine and in principle a user could start a new engine.
+        # Also, if we did monitor engine stopping, it is difficult to
+        # know what to do when only some engines die. Currently, the
+        # observing of engine stopping is inconsistent. Some launchers
+        # might trigger on a single engine stopping, other wait until
+        # all stop.  TODO: think more about how to handle this.
+        
+        # Start the controller and engines
+        self._stopping = False  # Make sure stop_launchers is not called 2x.
+        d = self.start_controller()
+        d.addCallback(self.start_engines)
+        d.addCallback(self.startup_message)
+        # If the controller or engines fail to start, stop everything
+        d.addErrback(self.stop_launchers)
+        return d
+
+    def startup_message(self, r=None):
+        log.msg("IPython cluster: started")
+        return r
+        
+    def start_controller(self, r=None):
+        # log.msg("In start_controller")
+        config = self.master_config
+        d = self.controller_launcher.start(
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+            
+    def start_engines(self, r=None):
+        # log.msg("In start_engines")
+        config = self.master_config
+        d = self.engine_launcher.start(
+            config.Global.n,
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+
+    def stop_controller(self, r=None):
+        # log.msg("In stop_controller")
+        if self.controller_launcher.running:
+            d = self.controller_launcher.stop()
+            d.addErrback(self.log_err)
+            return d
+        else:
+            return defer.succeed(None)
+
+    def stop_engines(self, r=None):
+        # log.msg("In stop_engines")
+        if self.engine_launcher.running:
+            d = self.engine_launcher.stop()
+            d.addErrback(self.log_err)
+            return d
+        else:
+            return defer.succeed(None)
+
+    def log_err(self, f):
+        log.msg(f.getTraceback())
+        return None
+        
+    def stop_launchers(self, r=None):
+        if not self._stopping:
+            self._stopping = True
+            if isinstance(r, failure.Failure):
+                log.msg('Unexpected error in ipcluster:')
+                log.msg(r.getTraceback())
+            log.msg("IPython cluster: stopping")
+            self.stop_engines()
+            self.stop_controller()
+            # Wait a few seconds to let things shut down.
+            reactor.callLater(4.0, reactor.stop)
+
+    def sigint_handler(self, signum, frame):
+        self.stop_launchers()
+        
+    def start_logging(self):
+        # Remove old log files of the controller and engine
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if f.startswith('ipengine' + '-'):
+                    if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
+                        os.remove(os.path.join(log_dir, f))
+                if f.startswith('ipcontroller' + '-'):
+                    if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
+                        os.remove(os.path.join(log_dir, f))
+        # This will remote old log files for ipcluster itself
+        super(IPClusterApp, self).start_logging()
+
+    def start_app(self):
+        """Start the application, depending on what subcommand is used."""
+        subcmd = self.master_config.Global.subcommand
+        if subcmd=='create' or subcmd=='list':
+            return
+        elif subcmd=='start':
+            self.start_app_start()
+        elif subcmd=='stop':
+            self.start_app_stop()
+
+    def start_app_start(self):
+        """Start the app for the start subcommand."""
+        config = self.master_config
+        # First see if the cluster is already running
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            pass
+        else:
+            self.log.critical(
+                'Cluster is already running with [pid=%s]. '
+                'use "ipcluster stop" to stop the cluster.' % pid
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STARTED)
+
+        # Now log and daemonize
+        self.log.info(
+            'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
+        )
+        # TODO: Get daemonize working on Windows or as a Windows Server.
+        if config.Global.daemonize:
+            if os.name=='posix':
+                daemonize()
+
+        # Now write the new pid file AFTER our new forked pid is active.
+        self.write_pid_file()
+        reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
+        reactor.run()
+
+    def start_app_stop(self):
+        """Start the app for the stop subcommand."""
+        config = self.master_config
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            self.log.critical(
+                'Problem reading pid file, cluster is probably not running.'
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STOPPED)
+        else:
+            if os.name=='posix':
+                sig = config.Global.signal
+                self.log.info(
+                    "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
+                )
+                os.kill(pid, sig)
+            elif os.name=='nt':
+                # As of right now, we don't support daemonize on Windows, so
+                # stop will not do anything. Minimally, it should clean up the
+                # old .pid files.
+                self.remove_pid_file()
+
+def launch_new_instance():
+    """Create and run the IPython cluster."""
+    app = IPClusterApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/kernel/ipcontrollerapp.py

@@ -0,0 +1,255 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import copy
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.config.loader import Config, NoConfigDefault
+from IPython.core import release
+from IPython.core.application import Application
+from IPython.kernel import controllerservice
+from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.fcutil import FCServiceFactory
+from IPython.utils.traitlets import Str, Instance, Unicode
+
+#-----------------------------------------------------------------------------
+# Default interfaces
+#-----------------------------------------------------------------------------
+
+# The default client interfaces for FCClientServiceFactory.interfaces
+default_client_interfaces = Config()
+default_client_interfaces.Task.interface_chain = [
+    'IPython.kernel.task.ITaskController',
+    'IPython.kernel.taskfc.IFCTaskController'
+]
+
+default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+
+default_client_interfaces.MultiEngine.interface_chain = [
+    'IPython.kernel.multiengine.IMultiEngine',
+    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+]
+
+default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
+
+
+
+# The default engine interfaces for FCEngineServiceFactory.interfaces
+default_engine_interfaces = Config()
+default_engine_interfaces.Default.interface_chain = [
+    'IPython.kernel.enginefc.IFCControllerBase'
+]
+
+default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
+
+
+#-----------------------------------------------------------------------------
+# Service factories
+#-----------------------------------------------------------------------------
+
+
+class FCClientServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the client services."""
+
+    cert_file = Unicode(u'ipcontroller-client.pem', config=True)
+    interfaces = Instance(klass=Config, kw=default_client_interfaces,
+                          allow_none=False, config=True)
+
+
+class FCEngineServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the engine services."""
+
+    cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
+    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+                          allow_none=False, config=True)
+
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+cl_args = (
+    # Client config
+    (('--client-ip',), dict(
+        type=str, dest='FCClientServiceFactory.ip', 
+        help='The IP address or hostname the controller will listen on for '
+        'client connections.',
+        metavar='FCClientServiceFactory.ip')
+    ),
+    (('--client-port',), dict(
+        type=int, dest='FCClientServiceFactory.port', 
+        help='The port the controller will listen on for client connections. '
+        'The default is to use 0, which will autoselect an open port.',
+        metavar='FCClientServiceFactory.port')
+    ),
+    (('--client-location',), dict(
+        type=str, dest='FCClientServiceFactory.location',
+        help='The hostname or IP that clients should connect to. This does '
+        'not control which interface the controller listens on. Instead, this '
+        'determines the hostname/IP that is listed in the FURL, which is how '
+        'clients know where to connect. Useful if the controller is listening '
+        'on multiple interfaces.',
+        metavar='FCClientServiceFactory.location')
+    ),
+    # Engine config
+    (('--engine-ip',), dict(
+        type=str, dest='FCEngineServiceFactory.ip', 
+        help='The IP address or hostname the controller will listen on for '
+        'engine connections.',
+        metavar='FCEngineServiceFactory.ip')
+    ),
+    (('--engine-port',), dict(
+        type=int, dest='FCEngineServiceFactory.port',
+        help='The port the controller will listen on for engine connections. '
+        'The default is to use 0, which will autoselect an open port.',
+        metavar='FCEngineServiceFactory.port')
+    ),
+    (('--engine-location',), dict(
+        type=str, dest='FCEngineServiceFactory.location',
+        help='The hostname or IP that engines should connect to. This does '
+        'not control which interface the controller listens on. Instead, this '
+        'determines the hostname/IP that is listed in the FURL, which is how '
+        'engines know where to connect. Useful if the controller is listening '
+        'on multiple interfaces.',
+        metavar='FCEngineServiceFactory.location')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file',
+        help='Log to a file in the log directory (default is stdout)')
+    ),
+    (('-r','--reuse-furls'), dict(
+        action='store_true', dest='Global.reuse_furls',
+        help='Try to reuse all FURL files. If this is not set all FURL files '
+        'are deleted before the controller starts. This must be set if '
+        'specific ports are specified by --engine-port or --client-port.')
+    ),
+    (('--no-secure',), dict(
+        action='store_false', dest='Global.secure',
+        help='Turn off SSL encryption for all connections.')
+    ),
+    (('--secure',), dict(
+        action='store_true', dest='Global.secure',
+        help='Turn off SSL encryption for all connections.')
+    )
+)
+
+
+_description = """Start the IPython controller for parallel computing.
+
+The IPython controller provides a gateway between the IPython engines and
+clients. The controller needs to be started before the engines and can be
+configured using command line options or using a cluster directory. Cluster
+directories contain config, log and security files and are usually located in
+your .ipython directory and named as "cluster_<profile>". See the --profile
+and --cluster-dir options for details.
+"""
+
+default_config_file_name = u'ipcontroller_config.py'
+
+
+class IPControllerApp(ApplicationWithClusterDir):
+
+    name = u'ipcontroller'
+    description = _description
+    config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(IPControllerApp, self).create_default_config()
+        self.default_config.Global.reuse_furls = False
+        self.default_config.Global.secure = True
+        self.default_config.Global.import_statements = []
+        self.default_config.Global.clean_logs = True
+
+    def post_load_command_line_config(self):
+        # Now setup reuse_furls
+        c = self.command_line_config
+        if hasattr(c.Global, 'reuse_furls'):
+            c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
+            c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
+            del c.Global.reuse_furls
+        if hasattr(c.Global, 'secure'):
+            c.FCClientServiceFactory.secure = c.Global.secure
+            c.FCEngineServiceFactory.secure = c.Global.secure
+            del c.Global.secure
+
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+        self.start_logging()
+        self.import_statements()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        # The controller service
+        controller_service = controllerservice.ControllerService()
+        controller_service.setServiceParent(self.main_service)
+        # The client tub and all its refereceables
+        csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        client_service = csfactory.create()
+        client_service.setServiceParent(self.main_service)
+        # The engine tub
+        esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        engine_service = esfactory.create()
+        engine_service.setServiceParent(self.main_service)
+
+    def import_statements(self):
+        statements = self.master_config.Global.import_statements
+        for s in statements:
+            try:
+                log.msg("Executing statement: '%s'" % s)
+                exec s in globals(), locals()
+            except:
+                log.msg("Error running statement: %s" % s)
+
+    def start_app(self):
+        # Start the controller service.
+        self.main_service.startService()
+        # Write the .pid file overwriting old ones. This allow multiple
+        # controllers to clober each other. But Windows is not cleaning
+        # these up properly.
+        self.write_pid_file(overwrite=True)
+        # Add a trigger to delete the .pid file upon shutting down.
+        reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPControllerApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()

IPython/kernel/ipengineapp.py

@@ -0,0 +1,229 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.core.application import Application
+from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.engineconnector import EngineConnector
+from IPython.kernel.engineservice import EngineService
+from IPython.kernel.fcutil import Tub
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+cl_args = (
+    # Controller config
+    (('--furl-file',), dict(
+        type=unicode, dest='Global.furl_file',
+        help='The full location of the file containing the FURL of the '
+        'controller. If this is not given, the FURL file must be in the '
+        'security directory of the cluster directory.  This location is '
+        'resolved using the --profile and --app-dir options.',
+        metavar='Global.furl_file')
+    ),
+    # MPI
+    (('--mpi',), dict(
+        type=str, dest='MPI.use',
+        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+        metavar='MPI.use')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file',
+        help='Log to a file in the log directory (default is stdout)')
+    )
+)
+
+
+mpi4py_init = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+pytrilinos_init = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+
+default_config_file_name = u'ipengine_config.py'
+
+
+_description = """Start an IPython engine for parallel computing.\n\n
+
+IPython engines run in parallel and perform computations on behalf of a client
+and controller. A controller needs to be started before the engines. The
+engine can be configured using command line options or using a cluster
+directory. Cluster directories contain config, log and security files and are
+usually located in your .ipython directory and named as "cluster_<profile>".
+See the --profile and --cluster-dir options for details.
+"""
+
+
+class IPEngineApp(ApplicationWithClusterDir):
+
+    name = u'ipengine'
+    description = _description
+    config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(IPEngineApp, self).create_default_config()
+
+        # The engine should not clean logs as we don't want to remove the
+        # active log files of other running engines.
+        self.default_config.Global.clean_logs = False
+
+        # Global config attributes
+        self.default_config.Global.exec_lines = []
+        self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+        # Configuration related to the controller
+        # This must match the filename (path not included) that the controller
+        # used for the FURL file.
+        self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
+        # If given, this is the actual location of the controller's FURL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        self.default_config.Global.furl_file = u''
+
+        # The max number of connection attemps and the initial delay between
+        # those attemps.
+        self.default_config.Global.connect_delay = 0.1
+        self.default_config.Global.connect_max_tries = 15
+
+        # MPI related config attributes
+        self.default_config.MPI.use = ''
+        self.default_config.MPI.mpi4py = mpi4py_init
+        self.default_config.MPI.pytrilinos = pytrilinos_init
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        super(IPEngineApp, self).pre_construct()
+        self.find_cont_furl_file()
+
+    def find_cont_furl_file(self):
+        """Set the furl file.
+
+        Here we don't try to actually see if it exists for is valid as that
+        is hadled by the connection logic.
+        """
+        config = self.master_config
+        # Find the actual controller FURL file
+        if not config.Global.furl_file:
+            try_this = os.path.join(
+                config.Global.cluster_dir, 
+                config.Global.security_dir,
+                config.Global.furl_file_name
+            )
+            config.Global.furl_file = try_this
+
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+        self.start_mpi()
+        self.start_logging()
+
+        # Create the underlying shell class and EngineService
+        shell_class = import_item(self.master_config.Global.shell_class)
+        self.engine_service = EngineService(shell_class, mpi=mpi)
+
+        self.exec_lines()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        self.engine_service.setServiceParent(self.main_service)
+        self.tub_service = Tub()
+        self.tub_service.setServiceParent(self.main_service)
+        # This needs to be called before the connection is initiated
+        self.main_service.startService()
+
+        # This initiates the connection to the controller and calls
+        # register_engine to tell the controller we are ready to do work
+        self.engine_connector = EngineConnector(self.tub_service)
+
+        log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
+
+        reactor.callWhenRunning(self.call_connect)
+
+    def call_connect(self):
+        d = self.engine_connector.connect_to_controller(
+            self.engine_service, 
+            self.master_config.Global.furl_file,
+            self.master_config.Global.connect_delay,
+            self.master_config.Global.connect_max_tries
+        )
+
+        def handle_error(f):
+            log.msg('Error connecting to controller. This usually means that '
+            'i) the controller was not started, ii) a firewall was blocking '
+            'the engine from connecting to the controller or iii) the engine '
+            ' was not pointed at the right FURL file:')
+            log.msg(f.getErrorMessage())
+            reactor.callLater(0.1, reactor.stop)
+
+        d.addErrback(handle_error)
+
+    def start_mpi(self):
+        global mpi
+        mpikey = self.master_config.MPI.use
+        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
+        if mpi_import_statement is not None:
+            try:
+                self.log.info("Initializing MPI:")
+                self.log.info(mpi_import_statement)
+                exec mpi_import_statement in globals()
+            except:
+                mpi = None
+        else:
+            mpi = None
+
+    def exec_lines(self):
+        for line in self.master_config.Global.exec_lines:
+            try:
+                log.msg("Executing statement: '%s'" % line)
+                self.engine_service.execute(line)
+            except:
+                log.msg("Error executing statement: %s" % line)
+
+    def start_app(self):
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPEngineApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/kernel/launcher.py

@@ -0,0 +1,869 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Facilities for launching IPython processes asynchronously.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import re
+import sys
+
+from IPython.core.component import Component
+from IPython.external import Itpl
+from IPython.utils.traitlets import Str, Int, List, Unicode, Enum
+from IPython.utils.platutils import find_cmd
+from IPython.kernel.twistedutil import gatherBoth, make_deferred, sleep_deferred
+from IPython.kernel.winhpcjob import (
+    WinHPCJob, WinHPCTask,
+    IPControllerTask, IPEngineTask,
+    IPControllerJob, IPEngineSetJob
+)
+
+from twisted.internet import reactor, defer
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.protocol import ProcessProtocol
+from twisted.internet.utils import getProcessOutput
+from twisted.internet.error import ProcessDone, ProcessTerminated
+from twisted.python import log
+from twisted.python.failure import Failure
+
+#-----------------------------------------------------------------------------
+# Utilities
+#-----------------------------------------------------------------------------
+
+
+def find_controller_cmd():
+    """Find the command line ipcontroller program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipcontroller script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipcontrollerapp
+        script_location = ipcontrollerapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipcontroller']
+    return cmd
+
+
+def find_engine_cmd():
+    """Find the command line ipengine program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipengine script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipengineapp
+        script_location = ipengineapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipengine']
+    return cmd
+
+
+#-----------------------------------------------------------------------------
+# Base launchers and errors
+#-----------------------------------------------------------------------------
+
+
+class LauncherError(Exception):
+    pass
+
+
+class ProcessStateError(LauncherError):
+    pass
+
+
+class UnknownStatus(LauncherError):
+    pass
+
+
+class BaseLauncher(Component):
+    """An asbtraction for starting, stopping and signaling a process."""
+
+    # In all of the launchers, the work_dir is where child processes will be
+    # run. This will usually be the cluster_dir, but may not be. any work_dir
+    # passed into the __init__ method will override the config value.
+    # This should not be used to set the work_dir for the actual engine
+    # and controller. Instead, use their own config files or the
+    # controller_args, engine_args attributes of the launchers to add
+    # the --work-dir option.
+    work_dir = Unicode(u'')
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(BaseLauncher, self).__init__(parent, name, config)
+        self.work_dir = work_dir
+        self.state = 'before' # can be before, running, after
+        self.stop_deferreds = []
+        self.start_data = None
+        self.stop_data = None
+
+    @property
+    def args(self):
+        """A list of cmd and args that will be used to start the process.
+
+        This is what is passed to :func:`spawnProcess` and the first element
+        will be the process name.
+        """
+        return self.find_args()
+
+    def find_args(self):
+        """The ``.args`` property calls this to find the args list.
+
+        Subcommand should implement this to construct the cmd and args.
+        """
+        raise NotImplementedError('find_args must be implemented in a subclass')
+
+    @property
+    def arg_str(self):
+        """The string form of the program arguments."""
+        return ' '.join(self.args)
+
+    @property
+    def running(self):
+        """Am I running."""
+        if self.state == 'running':
+            return True
+        else:
+            return False
+
+    def start(self):
+        """Start the process.
+
+        This must return a deferred that fires with information about the
+        process starting (like a pid, job id, etc.).
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'start must be implemented in a subclass')
+            )
+        )
+
+    def stop(self):
+        """Stop the process and notify observers of stopping.
+
+        This must return a deferred that fires with information about the
+        processing stopping, like errors that occur while the process is
+        attempting to be shut down. This deferred won't fire when the process
+        actually stops. To observe the actual process stopping, see
+        :func:`observe_stop`.
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'stop must be implemented in a subclass')
+            )
+        )
+
+    def observe_stop(self):
+        """Get a deferred that will fire when the process stops.
+
+        The deferred will fire with data that contains information about
+        the exit status of the process.
+        """
+        if self.state=='after':
+            return defer.succeed(self.stop_data)
+        else:
+            d = defer.Deferred()
+            self.stop_deferreds.append(d)
+            return d
+
+    def notify_start(self, data):
+        """Call this to trigger startup actions.
+
+        This logs the process startup and sets the state to 'running'.  It is
+        a pass-through so it can be used as a callback.
+        """
+
+        log.msg('Process %r started: %r' % (self.args[0], data))
+        self.start_data = data
+        self.state = 'running'
+        return data
+
+    def notify_stop(self, data):
+        """Call this to trigger process stop actions.
+
+        This logs the process stopping and sets the state to 'after'. Call
+        this to trigger all the deferreds from :func:`observe_stop`."""
+
+        log.msg('Process %r stopped: %r' % (self.args[0], data))
+        self.stop_data = data
+        self.state = 'after'
+        for i in range(len(self.stop_deferreds)):
+            d = self.stop_deferreds.pop()
+            d.callback(data)
+        return data
+
+    def signal(self, sig):
+        """Signal the process.
+
+        Return a semi-meaningless deferred after signaling the process.
+
+        Parameters
+        ----------
+        sig : str or int
+            'KILL', 'INT', etc., or any signal number
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'signal must be implemented in a subclass')
+            )
+        )
+
+
+#-----------------------------------------------------------------------------
+# Local process launchers
+#-----------------------------------------------------------------------------
+
+
+class LocalProcessLauncherProtocol(ProcessProtocol):
+    """A ProcessProtocol to go with the LocalProcessLauncher."""
+
+    def __init__(self, process_launcher):
+        self.process_launcher = process_launcher
+        self.pid = None
+
+    def connectionMade(self):
+        self.pid = self.transport.pid
+        self.process_launcher.notify_start(self.transport.pid)
+
+    def processEnded(self, status):
+        value = status.value
+        if isinstance(value, ProcessDone):
+            self.process_launcher.notify_stop(
+                {'exit_code':0,
+                 'signal':None,
+                 'status':None,
+                 'pid':self.pid
+                }
+            )
+        elif isinstance(value, ProcessTerminated):
+            self.process_launcher.notify_stop(
+                {'exit_code':value.exitCode,
+                 'signal':value.signal,
+                 'status':value.status,
+                 'pid':self.pid
+                }
+            )
+        else:
+            raise UnknownStatus("Unknown exit status, this is probably a "
+                                "bug in Twisted")
+
+    def outReceived(self, data):
+        log.msg(data)
+
+    def errReceived(self, data):
+        log.err(data)
+
+
+class LocalProcessLauncher(BaseLauncher):
+    """Start and stop an external process in an asynchronous manner.
+
+    This will launch the external process with a working directory of
+    ``self.work_dir``.
+    """
+
+    # This is used to to construct self.args, which is passed to 
+    # spawnProcess.
+    cmd_and_args = List([])
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(LocalProcessLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.process_protocol = None
+        self.start_deferred = None
+
+    def find_args(self):
+        return self.cmd_and_args
+
+    def start(self):
+        if self.state == 'before':
+            self.process_protocol = LocalProcessLauncherProtocol(self)
+            self.start_deferred = defer.Deferred()
+            self.process_transport = reactor.spawnProcess(
+                self.process_protocol,
+                str(self.args[0]),  # twisted expects these to be str, not unicode
+                [str(a) for a in self.args],  # str expected, not unicode
+                env=os.environ,
+                path=self.work_dir  # start in the work_dir
+            )
+            return self.start_deferred
+        else:
+            s = 'The process was already started and has state: %r' % self.state
+            return defer.fail(ProcessStateError(s))
+
+    def notify_start(self, data):
+        super(LocalProcessLauncher, self).notify_start(data)
+        self.start_deferred.callback(data)
+
+    def stop(self):
+        return self.interrupt_then_kill()
+
+    @make_deferred
+    def signal(self, sig):
+        if self.state == 'running':
+            self.process_transport.signalProcess(sig)
+
+    @inlineCallbacks
+    def interrupt_then_kill(self, delay=2.0):
+        """Send INT, wait a delay and then send KILL."""
+        yield self.signal('INT')
+        yield sleep_deferred(delay)
+        yield self.signal('KILL')
+
+
+class LocalControllerLauncher(LocalProcessLauncher):
+    """Launch a controller as a regular external process."""
+
+    controller_cmd = List(find_controller_cmd(), config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', '40'], config=True)
+
+    def find_args(self):
+        return self.controller_cmd + self.controller_args
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting LocalControllerLauncher: %r" % self.args)
+        return super(LocalControllerLauncher, self).start()
+
+
+class LocalEngineLauncher(LocalProcessLauncher):
+    """Launch a single engine as a regular externall process."""
+
+    engine_cmd = List(find_engine_cmd(), config=True)
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+
+    def find_args(self):
+        return self.engine_cmd + self.engine_args
+
+    def start(self, cluster_dir):
+        """Start the engine by cluster_dir."""
+        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        return super(LocalEngineLauncher, self).start()
+
+
+class LocalEngineSetLauncher(BaseLauncher):
+    """Launch a set of engines as regular external processes."""
+
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(LocalEngineSetLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.launchers = []
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.cluster_dir = unicode(cluster_dir)
+        dlist = []
+        for i in range(n):
+            el = LocalEngineLauncher(self.work_dir, self)
+            # Copy the engine args over to each engine launcher.
+            import copy
+            el.engine_args = copy.deepcopy(self.engine_args)
+            d = el.start(cluster_dir)
+            if i==0:
+                log.msg("Starting LocalEngineSetLauncher: %r" % el.args)
+            self.launchers.append(el)
+            dlist.append(d)
+        # The consumeErrors here could be dangerous
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self.notify_start)
+        return dfinal
+
+    def find_args(self):
+        return ['engine set']
+
+    def signal(self, sig):
+        dlist = []
+        for el in self.launchers:
+            d = el.signal(sig)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dfinal
+
+    def interrupt_then_kill(self, delay=1.0):
+        dlist = []
+        for el in self.launchers:
+            d = el.interrupt_then_kill(delay)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dfinal
+
+    def stop(self):
+        return self.interrupt_then_kill()
+
+    def observe_stop(self):
+        dlist = [el.observe_stop() for el in self.launchers]
+        dfinal = gatherBoth(dlist, consumeErrors=False)
+        dfinal.addCallback(self.notify_stop)
+        return dfinal
+
+
+#-----------------------------------------------------------------------------
+# MPIExec launchers
+#-----------------------------------------------------------------------------
+
+
+class MPIExecLauncher(LocalProcessLauncher):
+    """Launch an external process using mpiexec."""
+
+    # The mpiexec command to use in starting the process.
+    mpi_cmd = List(['mpiexec'], config=True)
+    # The command line arguments to pass to mpiexec.
+    mpi_args = List([], config=True)
+    # The program to start using mpiexec.
+    program = List(['date'], config=True)
+    # The command line argument to the program.
+    program_args = List([], config=True)
+    # The number of instances of the program to start.
+    n = Int(1, config=True)
+
+    def find_args(self):
+        """Build self.args using all the fields."""
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.program + self.program_args
+
+    def start(self, n):
+        """Start n instances of the program using mpiexec."""
+        self.n = n
+        return super(MPIExecLauncher, self).start()
+
+
+class MPIExecControllerLauncher(MPIExecLauncher):
+    """Launch a controller using mpiexec."""
+
+    controller_cmd = List(find_controller_cmd(), config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', '40'], config=True)
+    n = Int(1, config=False)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting MPIExecControllerLauncher: %r" % self.args)
+        return super(MPIExecControllerLauncher, self).start(1)
+
+    def find_args(self):
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.controller_cmd + self.controller_args
+
+
+class MPIExecEngineSetLauncher(MPIExecLauncher):
+
+    engine_cmd = List(find_engine_cmd(), config=True)
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+    n = Int(1, config=True)
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        self.n = n
+        log.msg('Starting MPIExecEngineSetLauncher: %r' % self.args)
+        return super(MPIExecEngineSetLauncher, self).start(n)
+
+    def find_args(self):
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.engine_cmd + self.engine_args
+
+
+#-----------------------------------------------------------------------------
+# SSH launchers
+#-----------------------------------------------------------------------------
+
+# TODO: Get SSH Launcher working again.
+
+class SSHLauncher(BaseLauncher):
+    """A minimal launcher for ssh.
+
+    To be useful this will probably have to be extended to use the ``sshx``
+    idea for environment variables.  There could be other things this needs
+    as well.
+    """
+
+    ssh_cmd = List(['ssh'], config=True)
+    ssh_args = List([], config=True)
+    program = List(['date'], config=True)
+    program_args = List([], config=True)
+    hostname = Str('', config=True)
+    user = Str('', config=True)
+    location = Str('')
+
+    def _hostname_changed(self, name, old, new):
+        self.location = '%s@%s' % (self.user, new)
+
+    def _user_changed(self, name, old, new):
+        self.location = '%s@%s' % (new, self.hostname)
+
+    def find_args(self):
+        return self.ssh_cmd + self.ssh_args + [self.location] + \
+               self.program + self.program_args
+
+    def start(self, n, hostname=None, user=None):
+        if hostname is not None:
+            self.hostname = hostname
+        if user is not None:
+            self.user = user
+        return super(SSHLauncher, self).start()
+
+
+class SSHControllerLauncher(SSHLauncher):
+    pass
+
+
+class SSHEngineSetLauncher(BaseLauncher):
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Windows HPC Server 2008 scheduler launchers
+#-----------------------------------------------------------------------------
+
+
+# This is only used on Windows.
+def find_job_cmd():
+    if os.name=='nt':
+        return find_cmd('job')
+    else:
+        return 'job'
+
+
+class WindowsHPCLauncher(BaseLauncher):
+
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = Str(r'\d+', config=True)
+    # The filename of the instantiated job script.
+    job_file_name = Unicode(u'ipython_job.xml', config=True)
+    # The full path to the instantiated job script. This gets made dynamically
+    # by combining the work_dir with the job_file_name.
+    job_file = Unicode(u'')
+    # The hostname of the scheduler to submit the job to
+    scheduler = Str('', config=True)
+    job_cmd = Str(find_job_cmd(), config=True)
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(WindowsHPCLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+
+    @property
+    def job_file(self):
+        return os.path.join(self.work_dir, self.job_file_name)
+
+    def write_job_file(self, n):
+        raise NotImplementedError("Implement write_job_file in a subclass.")
+
+    def find_args(self):
+        return ['job.exe']
+        
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.search(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+
+    @inlineCallbacks
+    def start(self, n):
+        """Start n copies of the process using the Win HPC job scheduler."""
+        self.write_job_file(n)
+        args = [
+            'submit',
+            '/jobfile:%s' % self.job_file,
+            '/scheduler:%s' % self.scheduler
+        ]
+        log.msg("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+        output = yield getProcessOutput(str(self.job_cmd),
+            [str(a) for a in args],
+            env=dict((str(k),str(v)) for k,v in os.environ.items()),
+            path=self.work_dir
+        )
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        defer.returnValue(job_id)
+
+    @inlineCallbacks
+    def stop(self):
+        args = [
+            'cancel',
+            self.job_id,
+            '/scheduler:%s' % self.scheduler
+        ]
+        log.msg("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        try:
+            # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+            output = yield getProcessOutput(str(self.job_cmd),
+                [str(a) for a in args],
+                env=dict((str(k),str(v)) for k,v in os.environ.items()),
+                path=self.work_dir
+            )
+        except:
+            output = 'The job already appears to be stoppped: %r' % self.job_id
+        self.notify_stop(output)  # Pass the output of the kill cmd
+        defer.returnValue(output)
+
+
+class WindowsHPCControllerLauncher(WindowsHPCLauncher):
+
+    job_file_name = Unicode(u'ipcontroller_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPControllerJob(self)
+
+        t = IPControllerTask(self)
+        # The tasks work directory is *not* the actual work directory of 
+        # the controller. It is used as the base path for the stdout/stderr
+        # files that the scheduler redirects to.
+        t.work_directory = self.cluster_dir
+        # Add the --cluster-dir and from self.start().
+        t.controller_args.extend(self.extra_args)
+        job.add_task(t)
+
+        log.msg("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCControllerLauncher, self).start(1)
+
+
+class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
+
+    job_file_name = Unicode(u'ipengineset_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPEngineSetJob(self)
+
+        for i in range(n):
+            t = IPEngineTask(self)
+            # The tasks work directory is *not* the actual work directory of 
+            # the engine. It is used as the base path for the stdout/stderr
+            # files that the scheduler redirects to.
+            t.work_directory = self.cluster_dir
+            # Add the --cluster-dir and from self.start().
+            t.engine_args.extend(self.extra_args)
+            job.add_task(t)
+
+        log.msg("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, n, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCEngineSetLauncher, self).start(n)
+
+
+#-----------------------------------------------------------------------------
+# Batch (PBS) system launchers
+#-----------------------------------------------------------------------------
+
+# TODO: Get PBS launcher working again.
+
+class BatchSystemLauncher(BaseLauncher):
+    """Launch an external process using a batch system.
+
+    This class is designed to work with UNIX batch systems like PBS, LSF,
+    GridEngine, etc.  The overall model is that there are different commands
+    like qsub, qdel, etc. that handle the starting and stopping of the process.
+
+    This class also has the notion of a batch script. The ``batch_template``
+    attribute can be set to a string that is a template for the batch script.
+    This template is instantiated using Itpl. Thus the template can use
+    ${n} fot the number of instances. Subclasses can add additional variables
+    to the template dict.
+    """
+
+    # Subclasses must fill these in.  See PBSEngineSet
+    # The name of the command line program used to submit jobs.
+    submit_command = Str('', config=True)
+    # The name of the command line program used to delete jobs.
+    delete_command = Str('', config=True)
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = Str('', config=True)
+    # The string that is the batch script template itself.
+    batch_template = Str('', config=True)
+    # The filename of the instantiated batch script.
+    batch_file_name = Unicode(u'batch_script', config=True)
+    # The full path to the instantiated batch script.
+    batch_file = Unicode(u'')
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(BatchSystemLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
+        self.context = {}
+
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.match(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+
+    def write_batch_script(self, n):
+        """Instantiate and write the batch script to the work_dir."""
+        self.context['n'] = n
+        script_as_string = Itpl.itplns(self.batch_template, self.context)
+        log.msg('Writing instantiated batch script: %s' % self.batch_file)
+        f = open(self.batch_file, 'w')
+        f.write(script_as_string)
+        f.close()
+
+    @inlineCallbacks
+    def start(self, n):
+        """Start n copies of the process using a batch system."""
+        self.write_batch_script(n)
+        output = yield getProcessOutput(self.submit_command,
+            [self.batch_file], env=os.environ)
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        defer.returnValue(job_id)
+
+    @inlineCallbacks
+    def stop(self):
+        output = yield getProcessOutput(self.delete_command,
+            [self.job_id], env=os.environ
+        )
+        self.notify_stop(output)  # Pass the output of the kill cmd
+        defer.returnValue(output)
+
+
+class PBSLauncher(BatchSystemLauncher):
+    """A BatchSystemLauncher subclass for PBS."""
+
+    submit_command = Str('qsub', config=True)
+    delete_command = Str('qdel', config=True)
+    job_id_regexp = Str(r'\d+', config=True)
+    batch_template = Str('', config=True)
+    batch_file_name = Unicode(u'pbs_batch_script', config=True)
+    batch_file = Unicode(u'')
+
+
+class PBSControllerLauncher(PBSLauncher):
+    """Launch a controller using PBS."""
+
+    batch_file_name = Unicode(u'pbs_batch_script_controller', config=True)
+
+    def start(self, cluster_dir):
+        """Start the controller by profile or cluster_dir."""
+        # Here we save profile and cluster_dir in the context so they
+        # can be used in the batch script template as ${profile} and
+        # ${cluster_dir}
+        self.context['cluster_dir'] = cluster_dir
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting PBSControllerLauncher: %r" % self.args)
+        return super(PBSControllerLauncher, self).start(1)
+
+
+class PBSEngineSetLauncher(PBSLauncher):
+
+    batch_file_name = Unicode(u'pbs_batch_script_engines', config=True)
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.program_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg('Starting PBSEngineSetLauncher: %r' % self.args)
+        return super(PBSEngineSetLauncher, self).start(n)
+
+
+#-----------------------------------------------------------------------------
+# A launcher for ipcluster itself!
+#-----------------------------------------------------------------------------
+
+
+def find_ipcluster_cmd():
+    """Find the command line ipcluster program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipcluster script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipclusterapp
+        script_location = ipclusterapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipcluster']
+    return cmd
+
+
+class IPClusterLauncher(LocalProcessLauncher):
+    """Launch the ipcluster program in an external process."""
+
+    ipcluster_cmd = List(find_ipcluster_cmd(), config=True)
+    # Command line arguments to pass to ipcluster.
+    ipcluster_args = List(
+        ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True)
+    ipcluster_subcommand = Str('start')
+    ipcluster_n = Int(2)
+
+    def find_args(self):
+        return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
+            ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
+
+    def start(self):
+        log.msg("Starting ipcluster: %r" % self.args)
+        return super(IPClusterLauncher, self).start()
+

IPython/kernel/winhpcjob.py

@@ -0,0 +1,318 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Job and task components for writing .xml files that the Windows HPC Server 
+2008 can use to start jobs.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import re
+import uuid
+
+from xml.etree import ElementTree as ET
+from xml.dom import minidom
+
+from IPython.core.component import Component
+from IPython.external import Itpl
+from IPython.utils.traitlets import (
+    Str, Int, List, Unicode, Instance,
+    Enum, Bool, CStr
+)
+
+#-----------------------------------------------------------------------------
+# Job and Task Component
+#-----------------------------------------------------------------------------
+
+
+def as_str(value):
+    if isinstance(value, str):
+        return value
+    elif isinstance(value, bool):
+        if value:
+            return 'true'
+        else:
+            return 'false'
+    elif isinstance(value, (int, float)):
+        return repr(value)
+    else:
+        return value
+
+
+def indent(elem, level=0):
+    i = "\n" + level*"  "
+    if len(elem):
+        if not elem.text or not elem.text.strip():
+            elem.text = i + "  "
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+        for elem in elem:
+            indent(elem, level+1)
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+    else:
+        if level and (not elem.tail or not elem.tail.strip()):
+            elem.tail = i
+
+
+def find_username():
+    domain = os.environ.get('USERDOMAIN')
+    username = os.environ.get('USERNAME','')
+    if domain is None:
+        return username
+    else:
+        return '%s\\%s' % (domain, username)
+
+
+class WinHPCJob(Component):
+
+    job_id = Str('')
+    job_name = Str('MyJob', config=True)
+    min_cores = Int(1, config=True)
+    max_cores = Int(1, config=True)
+    min_sockets = Int(1, config=True)
+    max_sockets = Int(1, config=True)
+    min_nodes = Int(1, config=True)
+    max_nodes = Int(1, config=True)
+    unit_type = Str("Core", config=True)
+    auto_calculate_min = Bool(True, config=True)
+    auto_calculate_max = Bool(True, config=True)
+    run_until_canceled = Bool(False, config=True)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    job_type = Str('Batch', config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+    xmlns = Str('http://schemas.microsoft.com/HPCS2008/scheduler/')
+    version = Str("2.000")
+    tasks = List([])
+
+    @property
+    def owner(self):
+        return self.username
+
+    def _write_attr(self, root, attr, key):
+        s = as_str(getattr(self, attr, ''))
+        if s:
+            root.set(key, s)
+
+    def as_element(self):
+        # We have to add _A_ type things to get the right order than
+        # the MSFT XML parser expects.
+        root = ET.Element('Job')
+        self._write_attr(root, 'version', '_A_Version')
+        self._write_attr(root, 'job_name', '_B_Name')
+        self._write_attr(root, 'unit_type', '_C_UnitType')
+        self._write_attr(root, 'min_cores', '_D_MinCores')
+        self._write_attr(root, 'max_cores', '_E_MaxCores')
+        self._write_attr(root, 'min_sockets', '_F_MinSockets')
+        self._write_attr(root, 'max_sockets', '_G_MaxSockets')
+        self._write_attr(root, 'min_nodes', '_H_MinNodes')
+        self._write_attr(root, 'max_nodes', '_I_MaxNodes')
+        self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
+        self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
+        self._write_attr(root, 'username', '_L_UserName')
+        self._write_attr(root, 'job_type', '_M_JobType')
+        self._write_attr(root, 'priority', '_N_Priority')
+        self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
+        self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
+        self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
+        self._write_attr(root, 'project', '_R_Project')
+        self._write_attr(root, 'owner', '_S_Owner')
+        self._write_attr(root, 'xmlns', '_T_xmlns')
+        dependencies = ET.SubElement(root, "Dependencies")
+        etasks = ET.SubElement(root, "Tasks")
+        for t in self.tasks:
+            etasks.append(t.as_element())
+        return root
+
+    def tostring(self):
+        """Return the string representation of the job description XML."""
+        root = self.as_element()
+        indent(root)
+        txt = ET.tostring(root, encoding="utf-8")
+        # Now remove the tokens used to order the attributes.
+        txt = re.sub(r'_[A-Z]_','',txt)
+        txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
+        return txt
+
+    def write(self, filename):
+        """Write the XML job description to a file."""
+        txt = self.tostring()
+        with open(filename, 'w') as f:
+            f.write(txt)
+
+    def add_task(self, task):
+        """Add a task to the job.
+
+        Parameters
+        ----------
+        task : :class:`WinHPCTask`
+            The task object to add.
+        """
+        self.tasks.append(task)
+
+
+class WinHPCTask(Component):
+
+    task_id = Str('')
+    task_name = Str('')
+    version = Str("2.000")
+    min_cores = Int(1, config=True)
+    max_cores = Int(1, config=True)
+    min_sockets = Int(1, config=True)
+    max_sockets = Int(1, config=True)
+    min_nodes = Int(1, config=True)
+    max_nodes = Int(1, config=True)
+    unit_type = Str("Core", config=True)
+    command_line = CStr('', config=True)
+    work_directory = CStr('', config=True)
+    is_rerunnaable = Bool(True, config=True)
+    std_out_file_path = CStr('', config=True)
+    std_err_file_path = CStr('', config=True)
+    is_parametric = Bool(False, config=True)
+    environment_variables = Instance(dict, args=(), config=True)
+
+    def _write_attr(self, root, attr, key):
+        s = as_str(getattr(self, attr, ''))
+        if s:
+            root.set(key, s)
+
+    def as_element(self):
+        root = ET.Element('Task')
+        self._write_attr(root, 'version', '_A_Version')
+        self._write_attr(root, 'task_name', '_B_Name')
+        self._write_attr(root, 'min_cores', '_C_MinCores')
+        self._write_attr(root, 'max_cores', '_D_MaxCores')
+        self._write_attr(root, 'min_sockets', '_E_MinSockets')
+        self._write_attr(root, 'max_sockets', '_F_MaxSockets')
+        self._write_attr(root, 'min_nodes', '_G_MinNodes')
+        self._write_attr(root, 'max_nodes', '_H_MaxNodes')
+        self._write_attr(root, 'command_line', '_I_CommandLine')
+        self._write_attr(root, 'work_directory', '_J_WorkDirectory')
+        self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
+        self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
+        self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
+        self._write_attr(root, 'is_parametric', '_N_IsParametric')
+        self._write_attr(root, 'unit_type', '_O_UnitType')
+        root.append(self.get_env_vars())
+        return root
+
+    def get_env_vars(self):
+        env_vars = ET.Element('EnvironmentVariables')
+        for k, v in self.environment_variables.items():
+            variable = ET.SubElement(env_vars, "Variable")
+            name = ET.SubElement(variable, "Name")
+            name.text = k
+            value = ET.SubElement(variable, "Value")
+            value.text = v
+        return env_vars
+
+
+
+# By declaring these, we can configure the controller and engine separately!
+
+class IPControllerJob(WinHPCJob):
+    job_name = Str('IPController', config=False)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+
+
+class IPEngineSetJob(WinHPCJob):
+    job_name = Str('IPEngineSet', config=False)
+    is_exclusive = Bool(False, config=True)
+    username = Str(find_username(), config=True)
+    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
+        default_value='Highest', config=True)
+    requested_nodes = Str('', config=True)
+    project = Str('IPython', config=True)
+
+
+class IPControllerTask(WinHPCTask):
+
+    task_name = Str('IPController', config=True)
+    controller_cmd = List(['ipcontroller.exe'], config=True)
+    controller_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    # I don't want these to be configurable
+    std_out_file_path = CStr('', config=False)
+    std_err_file_path = CStr('', config=False)
+    min_cores = Int(1, config=False)
+    max_cores = Int(1, config=False)
+    min_sockets = Int(1, config=False)
+    max_sockets = Int(1, config=False)
+    min_nodes = Int(1, config=False)
+    max_nodes = Int(1, config=False)
+    unit_type = Str("Core", config=False)
+    work_directory = CStr('', config=False)
+
+    def __init__(self, parent, name=None, config=None):
+        super(IPControllerTask, self).__init__(parent, name, config)
+        the_uuid = uuid.uuid1()
+        self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
+        self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)
+
+    @property
+    def command_line(self):
+        return ' '.join(self.controller_cmd + self.controller_args)
+
+
+class IPEngineTask(WinHPCTask):
+
+    task_name = Str('IPEngine', config=True)
+    engine_cmd = List(['ipengine.exe'], config=True)
+    engine_args = List(['--log-to-file', '--log-level', '40'], config=True)
+    # I don't want these to be configurable
+    std_out_file_path = CStr('', config=False)
+    std_err_file_path = CStr('', config=False)
+    min_cores = Int(1, config=False)
+    max_cores = Int(1, config=False)
+    min_sockets = Int(1, config=False)
+    max_sockets = Int(1, config=False)
+    min_nodes = Int(1, config=False)
+    max_nodes = Int(1, config=False)
+    unit_type = Str("Core", config=False)
+    work_directory = CStr('', config=False)
+
+    def __init__(self, parent, name=None, config=None):
+        super(IPEngineTask,self).__init__(parent, name, config)
+        the_uuid = uuid.uuid1()
+        self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
+        self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)
+    
+    @property
+    def command_line(self):
+        return ' '.join(self.engine_cmd + self.engine_args)
+
+
+# j = WinHPCJob(None)
+# j.job_name = 'IPCluster'
+# j.username = 'GNET\\bgranger'
+# j.requested_nodes = 'GREEN'
+# 
+# t = WinHPCTask(None)
+# t.task_name = 'Controller'
+# t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
+# t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
+# t.std_out_file_path = 'controller-out.txt'
+# t.std_err_file_path = 'controller-err.txt'
+# t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
+# j.add_task(t)
+

IPython/testing/_doctest26.py

@@ -0,0 +1,110 b''
+"""Code taken from the Python2.6 standard library for backwards compatibility.
+
+This is just so we can use 2.6 features when running in 2.5, the code below is
+copied verbatim from the stdlib's collections and doctest modules.
+"""
+
+from keyword import iskeyword as _iskeyword
+from operator import itemgetter as _itemgetter
+import sys as _sys
+
+def namedtuple(typename, field_names, verbose=False):
+    """Returns a new subclass of tuple with named fields.
+
+    >>> Point = namedtuple('Point', 'x y')
+    >>> Point.__doc__                   # docstring for the new class
+    'Point(x, y)'
+    >>> p = Point(11, y=22)             # instantiate with positional args or keywords
+    >>> p[0] + p[1]                     # indexable like a plain tuple
+    33
+    >>> x, y = p                        # unpack like a regular tuple
+    >>> x, y
+    (11, 22)
+    >>> p.x + p.y                       # fields also accessable by name
+    33
+    >>> d = p._asdict()                 # convert to a dictionary
+    >>> d['x']
+    11
+    >>> Point(**d)                      # convert from a dictionary
+    Point(x=11, y=22)
+    >>> p._replace(x=100)               # _replace() is like str.replace() but targets named fields
+    Point(x=100, y=22)
+
+    """
+
+    # Parse and validate the field names.  Validation serves two purposes,
+    # generating informative error messages and preventing template injection attacks.
+    if isinstance(field_names, basestring):
+        field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas
+    field_names = tuple(map(str, field_names))
+    for name in (typename,) + field_names:
+        if not all(c.isalnum() or c=='_' for c in name):
+            raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name)
+        if _iskeyword(name):
+            raise ValueError('Type names and field names cannot be a keyword: %r' % name)
+        if name[0].isdigit():
+            raise ValueError('Type names and field names cannot start with a number: %r' % name)
+    seen_names = set()
+    for name in field_names:
+        if name.startswith('_'):
+            raise ValueError('Field names cannot start with an underscore: %r' % name)
+        if name in seen_names:
+            raise ValueError('Encountered duplicate field name: %r' % name)
+        seen_names.add(name)
+
+    # Create and fill-in the class template
+    numfields = len(field_names)
+    argtxt = repr(field_names).replace("'", "")[1:-1]   # tuple repr without parens or quotes
+    reprtxt = ', '.join('%s=%%r' % name for name in field_names)
+    dicttxt = ', '.join('%r: t[%d]' % (name, pos) for pos, name in enumerate(field_names))
+    template = '''class %(typename)s(tuple):
+        '%(typename)s(%(argtxt)s)' \n
+        __slots__ = () \n
+        _fields = %(field_names)r \n
+        def __new__(_cls, %(argtxt)s):
+            return _tuple.__new__(_cls, (%(argtxt)s)) \n
+        @classmethod
+        def _make(cls, iterable, new=tuple.__new__, len=len):
+            'Make a new %(typename)s object from a sequence or iterable'
+            result = new(cls, iterable)
+            if len(result) != %(numfields)d:
+                raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result))
+            return result \n
+        def __repr__(self):
+            return '%(typename)s(%(reprtxt)s)' %% self \n
+        def _asdict(t):
+            'Return a new dict which maps field names to their values'
+            return {%(dicttxt)s} \n
+        def _replace(_self, **kwds):
+            'Return a new %(typename)s object replacing specified fields with new values'
+            result = _self._make(map(kwds.pop, %(field_names)r, _self))
+            if kwds:
+                raise ValueError('Got unexpected field names: %%r' %% kwds.keys())
+            return result \n
+        def __getnewargs__(self):
+            return tuple(self) \n\n''' % locals()
+    for i, name in enumerate(field_names):
+        template += '        %s = _property(_itemgetter(%d))\n' % (name, i)
+    if verbose:
+        print template
+
+    # Execute the template string in a temporary namespace and
+    # support tracing utilities by setting a value for frame.f_globals['__name__']
+    namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename,
+                     _property=property, _tuple=tuple)
+    try:
+        exec template in namespace
+    except SyntaxError, e:
+        raise SyntaxError(e.message + ':\n' + template)
+    result = namespace[typename]
+
+    # For pickling to work, the __module__ variable needs to be set to the frame
+    # where the named tuple is created.  Bypass this step in enviroments where
+    # sys._getframe is not defined (Jython for example).
+    if hasattr(_sys, '_getframe'):
+        result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__')
+
+    return result
+
+
+TestResults = namedtuple('TestResults', 'failed attempted')

IPython/testing/_paramtestpy2.py

@@ -0,0 +1,89 b''
+"""Implementation of the parametric test support for Python 2.x
+"""
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import unittest
+from compiler.consts import CO_GENERATOR
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+def isgenerator(func):
+    try:
+        return func.func_code.co_flags & CO_GENERATOR != 0
+    except AttributeError:
+        return False
+
+class ParametricTestCase(unittest.TestCase):
+    """Write parametric tests in normal unittest testcase form.
+
+    Limitations: the last iteration misses printing out a newline when running
+    in verbose mode.
+    """
+    def run_parametric(self, result, testMethod):
+        # But if we have a test generator, we iterate it ourselves
+        testgen = testMethod()
+        while True:
+            try:
+                # Initialize test
+                result.startTest(self)
+
+                # SetUp
+                try:
+                    self.setUp()
+                except KeyboardInterrupt:
+                    raise
+                except:
+                    result.addError(self, self._exc_info())
+                    return
+                # Test execution
+                ok = False
+                try:
+                    testgen.next()
+                    ok = True
+                except StopIteration:
+                    # We stop the loop
+                    break
+                except self.failureException:
+                    result.addFailure(self, self._exc_info())
+                except KeyboardInterrupt:
+                    raise
+                except:
+                    result.addError(self, self._exc_info())
+                # TearDown
+                try:
+                    self.tearDown()
+                except KeyboardInterrupt:
+                    raise
+                except:
+                    result.addError(self, self._exc_info())
+                    ok = False
+                if ok: result.addSuccess(self)
+
+            finally:
+                result.stopTest(self)
+
+    def run(self, result=None):
+        if result is None:
+            result = self.defaultTestResult()
+        testMethod = getattr(self, self._testMethodName)
+        # For normal tests, we just call the base class and return that
+        if isgenerator(testMethod):
+            return self.run_parametric(result, testMethod)
+        else:
+            return super(ParametricTestCase, self).run(result)
+
+
+def parametric(func):
+    """Decorator to make a simple function into a normal test via unittest."""
+
+    class Tester(ParametricTestCase):
+        test = staticmethod(func)
+
+    Tester.__name__ = func.__name__
+
+    return Tester

IPython/testing/_paramtestpy3.py

@@ -0,0 +1,62 b''
+"""Implementation of the parametric test support for Python 3.x.
+
+Thanks for the py3 version to Robert Collins, from the Testing in Python
+mailing list.
+"""
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import unittest
+from unittest import TestSuite
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+
+def isgenerator(func):
+    return hasattr(func,'_generator')
+
+
+class IterCallableSuite(TestSuite):
+   def __init__(self, iterator, adapter):
+       self._iter = iterator
+       self._adapter = adapter
+   def __iter__(self):
+       yield self._adapter(self._iter.__next__)
+
+class ParametricTestCase(unittest.TestCase):
+   """Write parametric tests in normal unittest testcase form.
+
+   Limitations: the last iteration misses printing out a newline when
+   running in verbose mode.
+   """
+
+   def run(self, result=None):
+       testMethod = getattr(self, self._testMethodName)
+       # For normal tests, we just call the base class and return that
+       if isgenerator(testMethod):
+           def adapter(next_test):
+               return unittest.FunctionTestCase(next_test,
+                                                self.setUp,
+                                                self.tearDown)
+
+           return IterCallableSuite(testMethod(),adapter).run(result)
+       else:
+           return super(ParametricTestCase, self).run(result)
+
+
+def parametric(func):
+   """Decorator to make a simple function into a normal test via
+unittest."""
+   # Hack, until I figure out how to write isgenerator() for python3!!
+   func._generator = True
+
+   class Tester(ParametricTestCase):
+       test = staticmethod(func)
+
+   Tester.__name__ = func.__name__
+
+   return Tester

IPython/testing/globalipapp.py

@@ -0,0 +1,168 b''
+"""Global IPython app to support test running.
+
+We must start our own ipython object and heavily muck with it so that all the
+modifications IPython makes to system behavior don't send the doctest machinery
+into a fit.  This code should be considered a gross hack, but it gets the job
+done.
+"""
+
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
+
+# From the standard library
+import __builtin__
+import commands
+import new
+import os
+import sys
+
+from . import tools
+from IPython.utils.genutils import Term
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
+# Hack to modify the %run command so we can sync the user's namespace with the
+# test globals.  Once we move over to a clean magic system, this will be done
+# with much less ugliness.
+
+class py_file_finder(object):
+    def __init__(self,test_filename):
+        self.test_filename = test_filename
+        
+    def __call__(self,name):
+        from IPython.utils.genutils import get_py_filename
+        try:
+            return get_py_filename(name)
+        except IOError:
+            test_dir = os.path.dirname(self.test_filename)
+            new_path = os.path.join(test_dir,name)
+            return get_py_filename(new_path)
+    
+
+def _run_ns_sync(self,arg_s,runner=None):
+    """Modified version of %run that syncs testing namespaces.
+
+    This is strictly needed for running doctests that call %run.
+    """
+    #print >> sys.stderr,  'in run_ns_sync', arg_s  # dbg
+    
+    _ip = get_ipython()
+    finder = py_file_finder(arg_s)
+    out = _ip.magic_run_ori(arg_s,runner,finder)
+    return out
+
+
+class ipnsdict(dict):
+    """A special subclass of dict for use as an IPython namespace in doctests.
+
+    This subclass adds a simple checkpointing capability so that when testing
+    machinery clears it (we use it as the test execution context), it doesn't
+    get completely destroyed.
+    """
+    
+    def __init__(self,*a):
+        dict.__init__(self,*a)
+        self._savedict = {}
+        
+    def clear(self):
+        dict.clear(self)
+        self.update(self._savedict)
+        
+    def _checkpoint(self):
+        self._savedict.clear()
+        self._savedict.update(self)
+
+    def update(self,other):
+        self._checkpoint()
+        dict.update(self,other)
+
+        # If '_' is in the namespace, python won't set it when executing code,
+        # and we have examples that test it.  So we ensure that the namespace
+        # is always 'clean' of it before it's used for test code execution.
+        self.pop('_',None)
+
+        # The builtins namespace must *always* be the real __builtin__ module,
+        # else weird stuff happens.  The main ipython code does have provisions
+        # to ensure this after %run, but since in this class we do some
+        # aggressive low-level cleaning of the execution namespace, we need to
+        # correct for that ourselves, to ensure consitency with the 'real'
+        # ipython.
+        self['__builtins__'] = __builtin__
+        
+
+def get_ipython():
+    # This will get replaced by the real thing once we start IPython below
+    return start_ipython()
+
+def start_ipython():
+    """Start a global IPython shell, which we need for IPython-specific syntax.
+    """
+    global get_ipython
+
+    # This function should only ever run once!
+    if hasattr(start_ipython,'already_called'):
+        return
+    start_ipython.already_called = True
+
+    # Ok,  first time we're called, go ahead
+    from IPython.core import ipapp, iplib
+    
+    def xsys(cmd):
+        """Execute a command and print its output.
+
+        This is just a convenience function to replace the IPython system call
+        with one that is more doctest-friendly.
+        """
+        cmd = _ip.var_expand(cmd,depth=1)
+        sys.stdout.write(commands.getoutput(cmd))
+        sys.stdout.flush()
+
+    # Store certain global objects that IPython modifies
+    _displayhook = sys.displayhook
+    _excepthook = sys.excepthook
+    _main = sys.modules.get('__main__')
+
+    # Create custom argv and namespaces for our IPython to be test-friendly
+    argv = tools.default_argv()
+    user_ns, global_ns = iplib.make_user_namespaces(ipnsdict(), {})
+    
+    # Create and initialize our test-friendly IPython instance.
+    ip = ipapp.IPythonApp(argv, user_ns=user_ns, user_global_ns=global_ns)
+    ip.initialize()
+
+    # A few more tweaks needed for playing nicely with doctests...
+    
+    # These traps are normally only active for interactive use, set them
+    # permanently since we'll be mocking interactive sessions.
+    ip.shell.builtin_trap.set()
+
+    # Set error printing to stdout so nose can doctest exceptions
+    ip.shell.InteractiveTB.out_stream = 'stdout'
+    
+    # Modify the IPython system call with one that uses getoutput, so that we
+    # can capture subcommands and print them to Python's stdout, otherwise the
+    # doctest machinery would miss them.
+    ip.shell.system = xsys
+
+    # IPython is ready, now clean up some global state...
+    
+    # Deactivate the various python system hooks added by ipython for
+    # interactive convenience so we don't confuse the doctest system
+    sys.modules['__main__'] = _main
+    sys.displayhook = _displayhook
+    sys.excepthook = _excepthook
+
+    # So that ipython magics and aliases can be doctested (they work by making
+    # a call into a global _ip object).  Also make the top-level get_ipython
+    # now return this without recursively calling here again.
+    _ip = ip.shell
+    get_ipython = _ip.get_ipython
+    __builtin__._ip = _ip
+    __builtin__.get_ipython = get_ipython
+
+    return _ip

IPython/testing/ipunittest.py

@@ -0,0 +1,189 b''
+"""Experimental code for cleaner support of IPython syntax with unittest.
+
+In IPython up until 0.10, we've used very hacked up nose machinery for running
+tests with IPython special syntax, and this has proved to be extremely slow.
+This module provides decorators to try a different approach, stemming from a
+conversation Brian and I (FP) had about this problem Sept/09.
+
+The goal is to be able to easily write simple functions that can be seen by
+unittest as tests, and ultimately for these to support doctests with full
+IPython syntax.  Nose already offers this based on naming conventions and our
+hackish plugins, but we are seeking to move away from nose dependencies if
+possible.
+
+This module follows a different approach, based on decorators.
+
+- A decorator called @ipdoctest can mark any function as having a docstring
+  that should be viewed as a doctest, but after syntax conversion.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import re
+import sys
+import unittest
+from doctest import DocTestFinder, DocTestRunner
+try:
+    from doctest import TestResults
+except:
+    from ._doctest26 import TestResults
+
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from ._paramtestpy2 import ParametricTestCase
+else:
+    from ._paramtestpy3 import ParametricTestCase
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+def count_failures(runner):
+    """Count number of failures in a doctest runner.
+
+    Code modeled after the summarize() method in doctest.
+    """
+    return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0 ]
+
+
+class IPython2PythonConverter(object):
+    """Convert IPython 'syntax' to valid Python.
+
+    Eventually this code may grow to be the full IPython syntax conversion
+    implementation, but for now it only does prompt convertion."""
+    
+    def __init__(self):
+        self.rps1 = re.compile(r'In\ \[\d+\]: ')
+        self.rps2 = re.compile(r'\ \ \ \.\.\.+: ')
+        self.rout = re.compile(r'Out\[\d+\]: \s*?\n?')
+        self.pyps1 = '>>> '
+        self.pyps2 = '... '
+        self.rpyps1 = re.compile ('(\s*%s)(.*)$' % self.pyps1)
+        self.rpyps2 = re.compile ('(\s*%s)(.*)$' % self.pyps2)
+
+    def __call__(self, ds):
+        """Convert IPython prompts to python ones in a string."""
+        from . import globalipapp
+
+        pyps1 = '>>> '
+        pyps2 = '... '
+        pyout = ''
+
+        dnew = ds
+        dnew = self.rps1.sub(pyps1, dnew)
+        dnew = self.rps2.sub(pyps2, dnew)
+        dnew = self.rout.sub(pyout, dnew)
+        ip = globalipapp.get_ipython()
+
+        # Convert input IPython source into valid Python.
+        out = []
+        newline = out.append
+        for line in dnew.splitlines():
+
+            mps1 = self.rpyps1.match(line)
+            if mps1 is not None:
+                prompt, text = mps1.groups()
+                newline(prompt+ip.prefilter(text, False))
+                continue
+
+            mps2 = self.rpyps2.match(line)
+            if mps2 is not None:
+                prompt, text = mps2.groups()
+                newline(prompt+ip.prefilter(text, True))
+                continue
+            
+            newline(line)
+        newline('')  # ensure a closing newline, needed by doctest
+        #print "PYSRC:", '\n'.join(out)  # dbg
+        return '\n'.join(out)
+
+    #return dnew
+
+
+class Doc2UnitTester(object):
+    """Class whose instances act as a decorator for docstring testing.
+
+    In practice we're only likely to need one instance ever, made below (though
+    no attempt is made at turning it into a singleton, there is no need for
+    that).
+    """
+    def __init__(self, verbose=False):
+        """New decorator.
+
+        Parameters
+        ----------
+
+        verbose : boolean, optional (False)
+          Passed to the doctest finder and runner to control verbosity.
+        """
+        self.verbose = verbose
+        # We can reuse the same finder for all instances
+        self.finder = DocTestFinder(verbose=verbose, recurse=False)
+
+    def __call__(self, func):
+        """Use as a decorator: doctest a function's docstring as a unittest.
+        
+        This version runs normal doctests, but the idea is to make it later run
+        ipython syntax instead."""
+
+        # Capture the enclosing instance with a different name, so the new
+        # class below can see it without confusion regarding its own 'self'
+        # that will point to the test instance at runtime
+        d2u = self
+
+        # Rewrite the function's docstring to have python syntax
+        if func.__doc__ is not None:
+            func.__doc__ = ip2py(func.__doc__)
+
+        # Now, create a tester object that is a real unittest instance, so
+        # normal unittest machinery (or Nose, or Trial) can find it.
+        class Tester(unittest.TestCase):
+            def test(self):
+                # Make a new runner per function to be tested
+                runner = DocTestRunner(verbose=d2u.verbose)
+                map(runner.run, d2u.finder.find(func, func.__name__))
+                failed = count_failures(runner)
+                if failed:
+                    # Since we only looked at a single function's docstring,
+                    # failed should contain at most one item.  More than that
+                    # is a case we can't handle and should error out on
+                    if len(failed) > 1:
+                        err = "Invalid number of test results:" % failed
+                        raise ValueError(err)
+                    # Report a normal failure.
+                    self.fail('failed doctests: %s' % str(failed[0]))
+                    
+        # Rename it so test reports have the original signature.
+        Tester.__name__ = func.__name__
+        return Tester
+
+
+def ipdocstring(func):
+    """Change the function docstring via ip2py.
+    """
+    if func.__doc__ is not None:
+        func.__doc__ = ip2py(func.__doc__)
+    return func
+
+        
+# Make an instance of the classes for public use
+ipdoctest = Doc2UnitTester()
+ip2py = IPython2PythonConverter()

IPython/testing/nosepatch.py

@@ -0,0 +1,53 b''
+"""Monkeypatch nose to accept any callable as a method.
+
+By default, nose's ismethod() fails for static methods.
+Once this is fixed in upstream nose we can disable it.
+
+Note: merely importing this module causes the monkeypatch to be applied."""
+
+import unittest
+import nose.loader
+from inspect import ismethod, isfunction
+
+def getTestCaseNames(self, testCaseClass):
+    """Override to select with selector, unless
+    config.getTestCaseNamesCompat is True
+    """
+    if self.config.getTestCaseNamesCompat:
+        return unittest.TestLoader.getTestCaseNames(self, testCaseClass)
+
+    def wanted(attr, cls=testCaseClass, sel=self.selector):
+        item = getattr(cls, attr, None)
+        # MONKEYPATCH: replace this:
+        #if not ismethod(item):
+        #    return False
+        # return sel.wantMethod(item)
+        # With:
+        if ismethod(item):
+            return sel.wantMethod(item)
+        # static method or something. If this is a static method, we
+        # can't get the class information, and we have to treat it
+        # as a function.  Thus, we will miss things like class
+        # attributes for test selection
+        if isfunction(item):
+            return sel.wantFunction(item)
+        return False
+        # END MONKEYPATCH
+    
+    cases = filter(wanted, dir(testCaseClass))
+    for base in testCaseClass.__bases__:
+        for case in self.getTestCaseNames(base):
+            if case not in cases:
+                cases.append(case)
+    # add runTest if nothing else picked
+    if not cases and hasattr(testCaseClass, 'runTest'):
+        cases = ['runTest']
+    if self.sortTestMethodsUsing:
+        cases.sort(self.sortTestMethodsUsing)
+    return cases
+
+
+##########################################################################
+# Apply monkeypatch here
+nose.loader.TestLoader.getTestCaseNames = getTestCaseNames
+##########################################################################

IPython/testing/tests/test_ipunittest.py

@@ -0,0 +1,122 b''
+"""Tests for IPyhton's test support utilities.
+
+These are decorators that allow standalone functions and docstrings to be seen
+as tests by unittest, replicating some of nose's functionality.  Additionally,
+IPython-syntax docstrings can be auto-converted to '>>>' so that ipython
+sessions can be copy-pasted as tests.
+
+This file can be run as a script, and it will call unittest.main().  We must
+check that it works with unittest as well as with nose...
+
+
+Notes:
+
+- Using nosetests --with-doctest --doctest-tests testfile.py
+  will find docstrings as tests wherever they are, even in methods.  But
+  if we use ipython syntax in the docstrings, they must be decorated with
+  @ipdocstring.  This is OK for test-only code, but not for user-facing
+  docstrings where we want to keep the ipython syntax.
+
+- Using nosetests --with-doctest file.py
+  also finds doctests if the file name doesn't have 'test' in it, because it is
+  treated like a normal module.  But if nose treats the file like a test file,
+  then for normal classes to be doctested the extra --doctest-tests is
+  necessary.
+
+- running this script with python (it has a __main__ section at the end) misses
+  one docstring test, the one embedded in the Foo object method.  Since our
+  approach relies on using decorators that create standalone TestCase
+  instances, it can only be used for functions, not for methods of objects.  
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython.testing.ipunittest import ipdoctest, ipdocstring
+
+#-----------------------------------------------------------------------------
+# Test classes and functions
+#-----------------------------------------------------------------------------
+@ipdoctest
+def simple_dt():
+    """
+    >>> print 1+1
+    2
+    """
+
+
+@ipdoctest
+def ipdt_flush():
+    """
+In [20]: print 1
+1
+
+In [26]: for i in range(10):
+   ....:     print i,
+   ....:     
+   ....:     
+0 1 2 3 4 5 6 7 8 9
+
+In [27]: 3+4
+Out[27]: 7
+"""
+
+
+@ipdoctest
+def ipdt_indented_test():
+    """
+    In [20]: print 1
+    1
+
+    In [26]: for i in range(10):
+       ....:     print i,
+       ....:     
+       ....:     
+    0 1 2 3 4 5 6 7 8 9
+
+    In [27]: 3+4
+    Out[27]: 7
+    """
+
+
+class Foo(object):
+    """For methods, the normal decorator doesn't work.
+
+    But rewriting the docstring with ip2py does, *but only if using nose
+    --with-doctest*.  Do we want to have that as a dependency?
+    """
+
+    @ipdocstring
+    def ipdt_method(self):
+        """
+        In [20]: print 1
+        1
+
+        In [26]: for i in range(10):
+           ....:     print i,
+           ....:     
+           ....:     
+        0 1 2 3 4 5 6 7 8 9
+
+        In [27]: 3+4
+        Out[27]: 7
+        """
+
+    def normaldt_method(self):
+        """
+        >>> print 1+1
+        2
+        """

docs/examples/core/new-embed.py

@@ -0,0 +1,17 b''
+# This shows how to use the new top-level embed function.  It is a simpler
+# API that manages the creation of the embedded shell.
+
+from IPython import embed
+
+a = 10
+b = 20
+
+embed('First time')
+
+c = 30
+d = 40
+
+try:
+    raise Exception('adsfasdf')
+except:
+    embed('The second time')

docs/examples/kernel/fractal.py

@@ -0,0 +1,90 b''
+from numpy import *
+
+def mandel(n, m, itermax, xmin, xmax, ymin, ymax):
+    '''
+    Fast mandelbrot computation using numpy.
+
+    (n, m) are the output image dimensions
+    itermax is the maximum number of iterations to do
+    xmin, xmax, ymin, ymax specify the region of the
+    set to compute.
+    '''
+    # The point of ix and iy is that they are 2D arrays
+    # giving the x-coord and y-coord at each point in
+    # the array. The reason for doing this will become
+    # clear below...
+    ix, iy = mgrid[0:n, 0:m]
+    # Now x and y are the x-values and y-values at each
+    # point in the array, linspace(start, end, n)
+    # is an array of n linearly spaced points between
+    # start and end, and we then index this array using
+    # numpy fancy indexing. If A is an array and I is
+    # an array of indices, then A[I] has the same shape
+    # as I and at each place i in I has the value A[i].
+    x = linspace(xmin, xmax, n)[ix]
+    y = linspace(ymin, ymax, m)[iy]
+    # c is the complex number with the given x, y coords
+    c = x+complex(0,1)*y
+    del x, y # save a bit of memory, we only need z
+    # the output image coloured according to the number
+    # of iterations it takes to get to the boundary
+    # abs(z)>2
+    img = zeros(c.shape, dtype=int)
+    # Here is where the improvement over the standard
+    # algorithm for drawing fractals in numpy comes in.
+    # We flatten all the arrays ix, iy and c. This
+    # flattening doesn't use any more memory because
+    # we are just changing the shape of the array, the
+    # data in memory stays the same. It also affects
+    # each array in the same way, so that index i in
+    # array c has x, y coords ix[i], iy[i]. The way the
+    # algorithm works is that whenever abs(z)>2 we
+    # remove the corresponding index from each of the
+    # arrays ix, iy and c. Since we do the same thing
+    # to each array, the correspondence between c and
+    # the x, y coords stored in ix and iy is kept.
+    ix.shape = n*m
+    iy.shape = n*m
+    c.shape = n*m
+    # we iterate z->z^2+c with z starting at 0, but the
+    # first iteration makes z=c so we just start there.
+    # We need to copy c because otherwise the operation
+    # z->z^2 will send c->c^2.
+    z = copy(c)
+    for i in xrange(itermax):
+        if not len(z): break # all points have escaped
+        # equivalent to z = z*z+c but quicker and uses
+        # less memory
+        multiply(z, z, z)
+        add(z, c, z)
+        # these are the points that have escaped
+        rem = abs(z)>2.0
+        # colour them with the iteration number, we
+        # add one so that points which haven't
+        # escaped have 0 as their iteration number,
+        # this is why we keep the arrays ix and iy
+        # because we need to know which point in img
+        # to colour
+        img[ix[rem], iy[rem]] = i+1
+        # -rem is the array of points which haven't
+        # escaped, in numpy -A for a boolean array A
+        # is the NOT operation.
+        rem = -rem
+        # So we select out the points in
+        # z, ix, iy and c which are still to be
+        # iterated on in the next step
+        z = z[rem]
+        ix, iy = ix[rem], iy[rem]
+        c = c[rem]
+    return img
+
+if __name__=='__main__':
+    from pylab import *
+    import time
+    start = time.time()
+    I = mandel(400, 400, 100, -2, .5, -1.25, 1.25)
+    print 'Time taken:', time.time()-start
+    I[I==0] = 101
+    img = imshow(I.T, origin='lower left')
+    img.write_png('mandel.png', noscale=True)
+    show()

docs/examples/kernel/parallelpi.py

@@ -0,0 +1,54 b''
+"""Calculate statistics on the digits of pi in parallel.
+
+This program uses the functions in :file:`pidigits.py` to calculate
+the frequencies of 2 digit sequences in the digits of pi. The
+results are plotted using matplotlib.
+
+To run, text files from http://www.super-computing.org/
+must be installed in the working directory of the IPython engines.
+The actual filenames to be used can be set with the ``filestring``
+variable below.
+
+The dataset we have been using for this is the 200 million digit one here:
+ftp://pi.super-computing.org/.2/pi200m/
+""" 
+
+from IPython.kernel import client
+from matplotlib import pyplot as plt
+import numpy as np
+from pidigits import *
+from timeit import default_timer as clock
+
+
+# Files with digits of pi (10m digits each)
+filestring = 'pi200m-ascii-%(i)02dof20.txt'
+files = [filestring % {'i':i} for i in range(1,16)]
+
+
+# Connect to the IPython cluster
+mec = client.MultiEngineClient(profile='mycluster')
+mec.run('pidigits.py')
+
+
+# Run 10m digits on 1 engine
+mapper = mec.mapper(targets=0)
+t1 = clock()
+freqs10m = mapper.map(compute_two_digit_freqs, files[:1])[0]
+t2 = clock()
+digits_per_second1 = 10.0e6/(t2-t1)
+print "Digits per second (1 core, 10m digits):   ", digits_per_second1
+
+
+# Run 150m digits on 15 engines (8 cores)
+t1 = clock()
+freqs_all = mec.map(compute_two_digit_freqs, files[:len(mec)])
+freqs150m = reduce_freqs(freqs_all)
+t2 = clock()
+digits_per_second8 = 150.0e6/(t2-t1)
+print "Digits per second (8 cores, 150m digits): ", digits_per_second8
+
+print "Speedup: ", digits_per_second8/digits_per_second1
+
+plot_two_digit_freqs(freqs150m)
+plt.title("2 digit sequences in 150m digits of pi")
+

docs/examples/kernel/pidigits.py

@@ -0,0 +1,144 b''
+"""Compute statistics on the digits of pi.
+
+This uses precomputed digits of pi from the website
+of Professor Yasumasa Kanada at the University of 
+Tokoyo: http://www.super-computing.org/
+
+Currently, there are only functions to read the
+.txt (non-compressed, non-binary) files, but adding
+support for compression and binary files would be
+straightforward.
+
+This focuses on computing the number of times that
+all 1, 2, n digits sequences occur in the digits of pi.
+If the digits of pi are truly random, these frequencies
+should be equal.
+"""
+
+# Import statements
+
+from __future__ import division, with_statement
+import numpy as np
+from matplotlib import pyplot as plt
+
+# Top-level functions
+
+def compute_one_digit_freqs(filename):
+    """
+    Read digits of pi from a file and compute the 1 digit frequencies.
+    """
+    d = txt_file_to_digits(filename)
+    freqs = one_digit_freqs(d)
+    return freqs
+
+def compute_two_digit_freqs(filename):
+    """
+    Read digits of pi from a file and compute the 2 digit frequencies.
+    """
+    d = txt_file_to_digits(filename)
+    freqs = two_digit_freqs(d)
+    return freqs
+
+def reduce_freqs(freqlist):
+    """
+    Add up a list of freq counts to get the total counts.
+    """
+    allfreqs = np.zeros_like(freqlist[0])
+    for f in freqlist:
+        allfreqs += f
+    return allfreqs
+
+def compute_n_digit_freqs(filename, n):
+    """
+    Read digits of pi from a file and compute the n digit frequencies.
+    """
+    d = txt_file_to_digits(filename)
+    freqs = n_digit_freqs(d, n)
+    return freqs
+
+# Read digits from a txt file
+    
+def txt_file_to_digits(filename, the_type=str):
+    """
+    Yield the digits of pi read from a .txt file.
+    """
+    with open(filename, 'r') as f:
+        for line in f.readlines():
+            for c in line:
+                if c != '\n' and c!= ' ':
+                    yield the_type(c)
+
+# Actual counting functions                    
+
+def one_digit_freqs(digits, normalize=False):
+    """
+    Consume digits of pi and compute 1 digit freq. counts.
+    """
+    freqs = np.zeros(10, dtype='i4')
+    for d in digits:
+        freqs[int(d)] += 1
+    if normalize:
+        freqs = freqs/freqs.sum()
+    return freqs
+    
+def two_digit_freqs(digits, normalize=False):
+    """
+    Consume digits of pi and compute 2 digits freq. counts.
+    """
+    freqs = np.zeros(100, dtype='i4')
+    last = digits.next()
+    this = digits.next()
+    for d in digits:
+        index = int(last + this)
+        freqs[index] += 1
+        last = this
+        this = d
+    if normalize:
+        freqs = freqs/freqs.sum()
+    return freqs
+
+def n_digit_freqs(digits, n, normalize=False):
+    """
+    Consume digits of pi and compute n digits freq. counts.
+
+    This should only be used for 1-6 digits.
+    """
+    freqs = np.zeros(pow(10,n), dtype='i4')
+    current = np.zeros(n, dtype=int)
+    for i in range(n):
+        current[i] = digits.next()
+    for d in digits:
+        index = int(''.join(map(str, current)))
+        freqs[index] += 1
+        current[0:-1] = current[1:]
+        current[-1] = d
+    if normalize:
+        freqs = freqs/freqs.sum()
+    return freqs
+
+# Plotting functions
+
+def plot_two_digit_freqs(f2):
+    """
+    Plot two digits frequency counts using matplotlib.
+    """
+    f2_copy = f2.copy()
+    f2_copy.shape = (10,10)
+    ax = plt.matshow(f2_copy)
+    plt.colorbar()
+    for i in range(10):
+        for j in range(10):
+            plt.text(i-0.2, j+0.2, str(j)+str(i))
+    plt.ylabel('First digit')
+    plt.xlabel('Second digit')
+    return ax
+
+def plot_one_digit_freqs(f1):
+    """
+    Plot one digit frequency counts using matplotlib.
+    """
+    ax = plt.plot(f1,'bo-')
+    plt.title('Single digit counts in pi')
+    plt.xlabel('Digit')
+    plt.ylabel('Count')
+    return ax

docs/source/development/ipgraph.txt

@@ -0,0 +1,59 b''
+====================================================
+Notes on code execution in :class:`InteractiveShell`
+====================================================
+
+Overview
+========
+
+This section contains information and notes about the code execution 
+system in :class:`InteractiveShell`.  This system needs to be refactored
+and we are keeping notes about this process here.
+
+Current design
+==============
+
+Here is a script that shows the relationships between the various
+methods in :class:`InteractiveShell` that manage code execution::
+
+    import networkx as nx
+    import matplotlib.pyplot as plt
+
+    exec_init_cmd = 'exec_init_cmd'
+    interact = 'interact'
+    runlines = 'runlines'
+    runsource = 'runsource'
+    runcode = 'runcode'
+    push_line = 'push_line'
+    mainloop = 'mainloop'
+    embed_mainloop = 'embed_mainloop'
+    ri = 'raw_input'
+    prefilter = 'prefilter'
+
+    g = nx.DiGraph()
+
+    g.add_node(exec_init_cmd)
+    g.add_node(interact)
+    g.add_node(runlines)
+    g.add_node(runsource)
+    g.add_node(push_line)
+    g.add_node(mainloop)
+    g.add_node(embed_mainloop)
+    g.add_node(ri)
+    g.add_node(prefilter)
+
+    g.add_edge(exec_init_cmd, push_line)
+    g.add_edge(exec_init_cmd, prefilter)
+    g.add_edge(mainloop, exec_init_cmd)
+    g.add_edge(mainloop, interact)
+    g.add_edge(embed_mainloop, interact)
+    g.add_edge(interact, ri)
+    g.add_edge(interact, push_line)
+    g.add_edge(push_line, runsource)
+    g.add_edge(runlines, push_line)
+    g.add_edge(runlines, prefilter)
+    g.add_edge(runsource, runcode)
+    g.add_edge(ri, prefilter)
+
+    nx.draw_spectral(g, node_size=100, alpha=0.6, node_color='r',
+                     font_size=10, node_shape='o')
+    plt.show()

docs/source/development/magic_blueprint.txt

@@ -0,0 +1,103 b''
+==============================
+ The magic commands subsystem
+==============================
+
+.. warning::
+
+   These are *preliminary* notes and thoughts on the magic system, kept here
+   for reference so we can come up with a good design now that the major core
+   refactoring has made so much progress.  Do not consider yet any part of this
+   document final.
+
+Two entry points:
+
+- m.line_eval(self,parameter_s): like today
+- m.block_eval(self,code_block): for whole-block evaluation.
+
+This would allow us to have magics that take input, and whose single line form
+can even take input and call block_eval later (like %cpaste does, but with a
+generalized interface).
+
+Constructor
+===========
+
+Suggested syntax::
+
+    class MyMagic(BaseMagic):
+        requires_shell = True/False
+        def __init__(self,shell=None):
+
+
+Registering magics
+==================
+
+Today, ipapi provides an *expose_magic()* function for making simple magics.
+We will probably extend this (in a backwards-compatible manner if possible) to
+allow the simplest cases to work as today, while letting users register more
+complex ones.
+
+Use cases::
+
+    def func(arg): pass  # note signature, no 'self'
+    ip.expose_magic('name',func)
+    
+    def func_line(arg): pass
+    def func_block(arg):pass
+    ip.expose_magic('name',func_line,func_block)
+
+    class mymagic(BaseMagic):
+        """Magic docstring, used in help messages.
+        """
+        def line_eval(self,arg): pass
+        def block_eval(self,arg): pass
+
+    ip.register_magic(mymagic)
+
+
+The BaseMagic class will offer common functionality to all, including things
+like options handling (via argparse).
+
+
+Call forms: line and block
+==========================
+
+Block-oriented environments will call line_eval() for the first line of input
+(the call line starting with '%') and will then feed the rest of the block to
+block_eval() if the magic in question has a block mode.
+
+In line environments, by default %foo -> foo.line_eval(), but no block call is
+made.  Specific implementations of line_eval can decide to then call block_eval
+if they want to provide for whole-block input in line-oriented environments.
+
+The api might be adapted for this decision to be made automatically by the
+frontend...
+
+
+Precompiled magics for rapid loading
+====================================
+
+For IPython itself, we'll have a module of 'core' magic functions that do not
+require run-time registration.  These will be the ones contained today in
+Magic.py, plus any others we deem worthy of being available by default.  This
+is a trick to enable faster startup, since once we move to a model where each
+magic can in principle be registered at runtime, creating a lot of them can
+easily swamp startup time.
+
+The trick is to make a module with a top-level class object that contains
+explicit references to all the 'core' magics in its dict.  This way, the magic
+table can be quickly updated at interpreter startup with a single call, by
+doing something along the lines of::
+
+    self.magic_table.update(static_magics.__dict__)
+
+The point will be to be able to bypass the explicit calling of whatever
+register_magic() API we end up making for users to declare their own magics.
+So ultimately one should be able to do either::
+
+    ip.register_magic(mymagic) # for one function
+
+or::
+
+    ip.load_magics(static_magics)  # for a bunch of them
+
+I still need to clarify exactly how this should work though.

docs/source/parallel/parallel_demos.txt

@@ -0,0 +1,282 b''
+=================
+Parallel examples
+=================
+
+In this section we describe two more involved examples of using an IPython
+cluster to perform a parallel computation. In these examples, we will be using
+IPython's "pylab" mode, which enables interactive plotting using the
+Matplotlib package. IPython can be started in this mode by typing::
+
+    ipython -p pylab
+
+at the system command line. If this prints an error message, you will 
+need to install the default profiles from within IPython by doing,
+
+.. sourcecode:: ipython
+
+    In [1]: %install_profiles
+
+and then restarting IPython.
+
+150 million digits of pi
+========================
+
+In this example we would like to study the distribution of digits in the
+number pi (in base 10). While it is not known if pi is a normal number (a
+number is normal in base 10 if 0-9 occur with equal likelihood) numerical
+investigations suggest that it is. We will begin with a serial calculation on
+10,000 digits of pi and then perform a parallel calculation involving 150
+million digits.
+
+In both the serial and parallel calculation we will be using functions defined
+in the :file:`pidigits.py` file, which is available in the
+:file:`docs/examples/kernel` directory of the IPython source distribution.
+These functions provide basic facilities for working with the digits of pi and
+can be loaded into IPython by putting :file:`pidigits.py` in your current
+working directory and then doing:
+
+.. sourcecode:: ipython
+
+    In [1]: run pidigits.py
+
+Serial calculation
+------------------
+
+For the serial calculation, we will use SymPy (http://www.sympy.org) to
+calculate 10,000 digits of pi and then look at the frequencies of the digits
+0-9. Out of 10,000 digits, we expect each digit to occur 1,000 times. While
+SymPy is capable of calculating many more digits of pi, our purpose here is to
+set the stage for the much larger parallel calculation.
+
+In this example, we use two functions from :file:`pidigits.py`: 
+:func:`one_digit_freqs` (which calculates how many times each digit occurs)
+and :func:`plot_one_digit_freqs` (which uses Matplotlib to plot the result).
+Here is an interactive IPython session that uses these functions with
+SymPy:
+
+.. sourcecode:: ipython
+
+    In [7]: import sympy
+
+    In [8]: pi = sympy.pi.evalf(40)
+
+    In [9]: pi
+    Out[9]: 3.141592653589793238462643383279502884197
+
+    In [10]: pi = sympy.pi.evalf(10000)
+
+    In [11]: digits = (d for d in str(pi)[2:])  # create a sequence of digits
+
+    In [12]: run pidigits.py  # load one_digit_freqs/plot_one_digit_freqs
+
+    In [13]: freqs = one_digit_freqs(digits)
+
+    In [14]: plot_one_digit_freqs(freqs)
+    Out[14]: [<matplotlib.lines.Line2D object at 0x18a55290>]
+
+The resulting plot of the single digit counts shows that each digit occurs
+approximately 1,000 times, but that with only 10,000 digits the
+statistical fluctuations are still rather large:
+
+.. image:: single_digits.*
+
+It is clear that to reduce the relative fluctuations in the counts, we need
+to look at many more digits of pi. That brings us to the parallel calculation.
+
+Parallel calculation
+--------------------
+
+Calculating many digits of pi is a challenging computational problem in itself. 
+Because we want to focus on the distribution of digits in this example, we
+will use pre-computed digit of pi from the website of Professor Yasumasa
+Kanada at the University of Tokoyo (http://www.super-computing.org). These
+digits come in a set of text files (ftp://pi.super-computing.org/.2/pi200m/)
+that each have 10 million digits of pi.
+
+For the parallel calculation, we have copied these files to the local hard
+drives of the compute nodes. A total of 15 of these files will be used, for a
+total of 150 million digits of pi. To make things a little more interesting we
+will calculate the frequencies of all 2 digits sequences (00-99) and then plot
+the result using a 2D matrix in Matplotlib.
+
+The overall idea of the calculation is simple: each IPython engine will
+compute the two digit counts for the digits in a single file. Then in a final
+step the counts from each engine will be added up. To perform this
+calculation, we will need two top-level functions from :file:`pidigits.py`:
+
+.. literalinclude:: ../../examples/kernel/pidigits.py
+   :language: python
+   :lines: 34-49
+
+We will also use the :func:`plot_two_digit_freqs` function to plot the
+results. The code to run this calculation in parallel is contained in
+:file:`docs/examples/kernel/parallelpi.py`. This code can be run in parallel
+using IPython by following these steps:
+
+1. Copy the text files with the digits of pi 
+   (ftp://pi.super-computing.org/.2/pi200m/) to the working directory of the
+   engines on the compute nodes.
+2. Use :command:`ipcluster` to start 15 engines. We used an 8 core (2 quad 
+   core CPUs) cluster with hyperthreading enabled which makes the 8 cores 
+   looks like 16 (1 controller + 15 engines) in the OS. However, the maximum
+   speedup we can observe is still only 8x.
+3. With the file :file:`parallelpi.py` in your current working directory, open
+   up IPython in pylab mode and type ``run parallelpi.py``.
+
+When run on our 8 core cluster, we observe a speedup of 7.7x. This is slightly
+less than linear scaling (8x) because the controller is also running on one of
+the cores.
+
+To emphasize the interactive nature of IPython, we now show how the
+calculation can also be run by simply typing the commands from
+:file:`parallelpi.py` interactively into IPython:
+
+.. sourcecode:: ipython
+
+    In [1]: from IPython.kernel import client
+    2009-11-19 11:32:38-0800 [-] Log opened.
+
+    # The MultiEngineClient allows us to use the engines interactively.
+    # We simply pass MultiEngineClient the name of the cluster profile we
+    # are using.
+    In [2]: mec = client.MultiEngineClient(profile='mycluster')
+    2009-11-19 11:32:44-0800 [-] Connecting [0]
+    2009-11-19 11:32:44-0800 [Negotiation,client] Connected: ./ipcontroller-mec.furl
+
+    In [3]: mec.get_ids()
+    Out[3]: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    In [4]: run pidigits.py
+
+    In [5]: filestring = 'pi200m-ascii-%(i)02dof20.txt'
+
+    # Create the list of files to process.
+    In [6]: files = [filestring % {'i':i} for i in range(1,16)]
+
+    In [7]: files
+    Out[7]: 
+    ['pi200m-ascii-01of20.txt',
+     'pi200m-ascii-02of20.txt',
+     'pi200m-ascii-03of20.txt',
+     'pi200m-ascii-04of20.txt',
+     'pi200m-ascii-05of20.txt',
+     'pi200m-ascii-06of20.txt',
+     'pi200m-ascii-07of20.txt',
+     'pi200m-ascii-08of20.txt',
+     'pi200m-ascii-09of20.txt',
+     'pi200m-ascii-10of20.txt',
+     'pi200m-ascii-11of20.txt',
+     'pi200m-ascii-12of20.txt',
+     'pi200m-ascii-13of20.txt',
+     'pi200m-ascii-14of20.txt',
+     'pi200m-ascii-15of20.txt']
+
+    # This is the parallel calculation using the MultiEngineClient.map method 
+    # which applies compute_two_digit_freqs to each file in files in parallel.
+    In [8]: freqs_all = mec.map(compute_two_digit_freqs, files)
+
+    # Add up the frequencies from each engine.
+    In [8]: freqs = reduce_freqs(freqs_all)
+
+    In [9]: plot_two_digit_freqs(freqs)
+    Out[9]: <matplotlib.image.AxesImage object at 0x18beb110>
+
+    In [10]: plt.title('2 digit counts of 150m digits of pi')
+    Out[10]: <matplotlib.text.Text object at 0x18d1f9b0>
+
+The resulting plot generated by Matplotlib is shown below. The colors indicate
+which two digit sequences are more (red) or less (blue) likely to occur in the
+first 150 million digits of pi. We clearly see that the sequence "41" is 
+most likely and that "06" and "07" are least likely. Further analysis would
+show that the relative size of the statistical fluctuations have decreased
+compared to the 10,000 digit calculation.
+
+.. image:: two_digit_counts.*
+
+
+Parallel options pricing
+========================
+
+An option is a financial contract that gives the buyer of the contract the
+right to buy (a "call") or sell (a "put") a secondary asset (a stock for
+example) at a particular date in the future (the expiration date) for a
+pre-agreed upon price (the strike price). For this right, the buyer pays the
+seller a premium (the option price). There are a wide variety of flavors of
+options (American, European, Asian, etc.) that are useful for different
+purposes: hedging against risk, speculation, etc.
+
+Much of modern finance is driven by the need to price these contracts
+accurately based on what is known about the properties (such as volatility) of
+the underlying asset. One method of pricing options is to use a Monte Carlo
+simulation of the underlying asset price. In this example we use this approach
+to price both European and Asian (path dependent) options for various strike
+prices and volatilities.
+
+The code for this example can be found in the :file:`docs/examples/kernel`
+directory of the IPython source. The function :func:`price_options` in
+:file:`mcpricer.py` implements the basic Monte Carlo pricing algorithm using
+the NumPy package and is shown here:
+
+.. literalinclude:: ../../examples/kernel/mcpricer.py
+   :language: python
+
+To run this code in parallel, we will use IPython's :class:`TaskClient` class,
+which distributes work to the engines using dynamic load balancing. This
+client can be used along side the :class:`MultiEngineClient` class shown in
+the previous example. The parallel calculation using :class:`TaskClient` can
+be found in the file :file:`mcpricer.py`. The code in this file creates a
+:class:`TaskClient` instance and then submits a set of tasks using
+:meth:`TaskClient.run` that calculate the option prices for different
+volatilities and strike prices. The results are then plotted as a 2D contour
+plot using Matplotlib.
+
+.. literalinclude:: ../../examples/kernel/mcdriver.py
+   :language: python
+
+To use this code, start an IPython cluster using :command:`ipcluster`, open
+IPython in the pylab mode with the file :file:`mcdriver.py` in your current
+working directory and then type:
+
+.. sourcecode:: ipython
+
+    In [7]: run mcdriver.py
+    Submitted tasks:  [0, 1, 2, ...]
+
+Once all the tasks have finished, the results can be plotted using the
+:func:`plot_options` function. Here we make contour plots of the Asian
+call and Asian put options as function of the volatility and strike price:
+
+.. sourcecode:: ipython
+
+    In [8]: plot_options(sigma_vals, K_vals, prices['acall'])
+
+    In [9]: plt.figure()
+    Out[9]: <matplotlib.figure.Figure object at 0x18c178d0>
+
+    In [10]: plot_options(sigma_vals, K_vals, prices['aput'])
+
+These results are shown in the two figures below. On a 8 core cluster the
+entire calculation (10 strike prices, 10 volatilities, 100,000 paths for each)
+took 30 seconds in parallel, giving a speedup of 7.7x, which is comparable
+to the speedup observed in our previous example.
+
+.. image:: asian_call.*
+
+.. image:: asian_put.*
+
+Conclusion
+==========
+
+To conclude these examples, we summarize the key features of IPython's
+parallel architecture that have been demonstrated:
+
+* Serial code can be parallelized often with only a few extra lines of code.
+  We have used the :class:`MultiEngineClient` and :class:`TaskClient` classes
+  for this purpose.
+* The resulting parallel code can be run without ever leaving the IPython's
+  interactive shell.
+* Any data computed in parallel can be explored interactively through 
+  visualization or further numerical calculations.
+* We have run these examples on a cluster running Windows HPC Server 2008.
+  IPython's built in support for the Windows HPC job scheduler makes it
+  easy to get started with IPython's parallel capabilities.

docs/source/parallel/parallel_winhpc.txt

@@ -0,0 +1,333 b''
+============================================
+Getting started with Windows HPC Server 2008
+============================================
+
+Introduction
+============
+
+The Python programming language is an increasingly popular language for
+numerical computing. This is due to a unique combination of factors. First,
+Python is a high-level and *interactive* language that is well matched to
+interactive numerical work. Second, it is easy (often times trivial) to
+integrate legacy C/C++/Fortran code into Python. Third, a large number of
+high-quality open source projects provide all the needed building blocks for
+numerical computing: numerical arrays (NumPy), algorithms (SciPy), 2D/3D
+Visualization (Matplotlib, Mayavi, Chaco), Symbolic Mathematics (Sage, Sympy)
+and others.
+
+The IPython project is a core part of this open-source toolchain and is
+focused on creating a comprehensive environment for interactive and
+exploratory computing in the Python programming language. It enables all of
+the above tools to be used interactively and consists of two main components:
+
+* An enhanced interactive Python shell with support for interactive plotting
+  and visualization.
+* An architecture for interactive parallel computing.
+
+With these components, it is possible to perform all aspects of a parallel
+computation interactively. This type of workflow is particularly relevant in
+scientific and numerical computing where algorithms, code and data are
+continually evolving as the user/developer explores a problem. The broad
+treads in computing (commodity clusters, multicore, cloud computing, etc.)
+make these capabilities of IPython particularly relevant.
+
+While IPython is a cross platform tool, it has particularly strong support for
+Windows based compute clusters running Windows HPC Server 2008. This document
+describes how to get started with IPython on Windows HPC Server 2008. The
+content and emphasis here is practical: installing IPython, configuring
+IPython to use the Windows job scheduler and running example parallel programs
+interactively. A more complete description of IPython's parallel computing
+capabilities can be found in IPython's online documentation
+(http://ipython.scipy.org/moin/Documentation).
+
+Setting up your Windows cluster
+===============================
+
+This document assumes that you already have a cluster running Windows
+HPC Server 2008. Here is a broad overview of what is involved with setting up
+such a cluster:
+
+1. Install Windows Server 2008 on the head and compute nodes in the cluster.
+2. Setup the network configuration on each host. Each host should have a 
+   static IP address.
+3. On the head node, activate the "Active Directory Domain Services" role 
+   and make the head node the domain controller.
+4. Join the compute nodes to the newly created Active Directory (AD) domain.
+5. Setup user accounts in the domain with shared home directories.
+6. Install the HPC Pack 2008 on the head node to create a cluster.
+7. Install the HPC Pack 2008 on the compute nodes.
+
+More details about installing and configuring Windows HPC Server 2008 can be
+found on the Windows HPC Home Page (http://www.microsoft.com/hpc). Regardless
+of what steps you follow to set up your cluster, the remainder of this
+document will assume that:
+
+* There are domain users that can log on to the AD domain and submit jobs
+  to the cluster scheduler.
+* These domain users have shared home directories. While shared home
+  directories are not required to use IPython, they make it much easier to
+  use IPython.
+
+Installation of IPython and its dependencies
+============================================
+
+IPython and all of its dependencies are freely available and open source.
+These packages provide a powerful and cost-effective approach to numerical and
+scientific computing on Windows. The following dependencies are needed to run
+IPython on Windows:
+
+* Python 2.5 or 2.6 (http://www.python.org)
+* pywin32 (http://sourceforge.net/projects/pywin32/)
+* PyReadline (https://launchpad.net/pyreadline)
+* zope.interface and Twisted (http://twistedmatrix.com)
+* Foolcap (http://foolscap.lothar.com/trac)
+* pyOpenSSL (https://launchpad.net/pyopenssl)
+* IPython (http://ipython.scipy.org)
+
+In addition, the following dependencies are needed to run the demos described
+in this document.
+
+* NumPy and SciPy (http://www.scipy.org)
+* wxPython (http://www.wxpython.org)
+* Matplotlib (http://matplotlib.sourceforge.net/)
+
+The easiest way of obtaining these dependencies is through the Enthought
+Python Distribution (EPD) (http://www.enthought.com/products/epd.php). EPD is
+produced by Enthought, Inc. and contains all of these packages and others in a
+single installer and is available free for academic users. While it is also
+possible to download and install each package individually, this is a tedious
+process. Thus, we highly recommend using EPD to install these packages on
+Windows.
+
+Regardless of how you install the dependencies, here are the steps you will
+need to follow:
+
+1. Install all of the packages listed above, either individually or using EPD
+   on the head node, compute nodes and user workstations.
+
+2. Make sure that :file:`C:\\Python25` and :file:`C:\\Python25\\Scripts` are
+   in the system :envvar:`%PATH%` variable on each node.
+
+3. Install the latest development version of IPython. This can be done by
+   downloading the the development version from the IPython website
+   (http://ipython.scipy.org) and following the installation instructions.
+
+Further details about installing IPython or its dependencies can be found in
+the online IPython documentation (http://ipython.scipy.org/moin/Documentation)
+Once you are finished with the installation, you can try IPython out by
+opening a Windows Command Prompt and typing ``ipython``. This will
+start IPython's interactive shell and you should see something like the
+following screenshot:
+
+.. image:: ipython_shell.*
+
+Starting an IPython cluster
+===========================
+
+To use IPython's parallel computing capabilities, you will need to start an
+IPython cluster. An IPython cluster consists of one controller and multiple
+engines:
+
+IPython controller
+    The IPython controller manages the engines and acts as a gateway between 
+    the engines and the client, which runs in the user's interactive IPython
+    session. The controller is started using the :command:`ipcontroller`
+    command.
+
+IPython engine
+    IPython engines run a user's Python code in parallel on the compute nodes. 
+    Engines are starting using the :command:`ipengine` command.
+
+Once these processes are started, a user can run Python code interactively and
+in parallel on the engines from within the IPython shell using an appropriate
+client. This includes the ability to interact with, plot and visualize data
+from the engines.
+
+IPython has a command line program called :command:`ipcluster` that automates
+all aspects of starting the controller and engines on the compute nodes.
+:command:`ipcluster` has full support for the Windows HPC job scheduler,
+meaning that :command:`ipcluster` can use this job scheduler to start the
+controller and engines. In our experience, the Windows HPC job scheduler is
+particularly well suited for interactive applications, such as IPython. Once
+:command:`ipcluster` is configured properly, a user can start an IPython
+cluster from their local workstation almost instantly, without having to log
+on to the head node (as is typically required by Unix based job schedulers).
+This enables a user to move seamlessly between serial and parallel
+computations.
+
+In this section we show how to use :command:`ipcluster` to start an IPython
+cluster using the Windows HPC Server 2008 job scheduler. To make sure that
+:command:`ipcluster` is installed and working properly, you should first try
+to start an IPython cluster on your local host. To do this, open a Windows
+Command Prompt and type the following command::
+
+    ipcluster start -n 2
+
+You should see a number of messages printed to the screen, ending with 
+"IPython cluster: started". The result should look something like the following
+screenshot:
+
+.. image:: ipcluster_start.*
+
+At this point, the controller and two engines are running on your local host.
+This configuration is useful for testing and for situations where you want to
+take advantage of multiple cores on your local computer. 
+
+Now that we have confirmed that :command:`ipcluster` is working properly, we
+describe how to configure and run an IPython cluster on an actual compute
+cluster running Windows HPC Server 2008. Here is an outline of the needed
+steps:
+
+1. Create a cluster profile using: ``ipcluster create -p mycluster``
+
+2. Edit configuration files in the directory :file:`.ipython\\cluster_mycluster`
+
+3. Start the cluster using: ``ipcluser start -p mycluster -n 32``
+
+Creating a cluster profile
+--------------------------
+
+In most cases, you will have to create a cluster profile to use IPython on a
+cluster. A cluster profile is a name (like "mycluster") that is associated
+with a particular cluster configuration. The profile name is used by
+:command:`ipcluster` when working with the cluster.
+
+Associated with each cluster profile is a cluster directory. This cluster
+directory is a specially named directory (typically located in the
+:file:`.ipython` subdirectory of your home directory) that contains the
+configuration files for a particular cluster profile, as well as log files and
+security keys. The naming convention for cluster directories is:
+:file:`cluster_<profile name>`. Thus, the cluster directory for a profile named
+"foo" would be :file:`.ipython\\cluster_foo`.
+
+To create a new cluster profile (named "mycluster") and the associated cluster
+directory, type the following command at the Windows Command Prompt::
+
+    ipcluster create -p mycluster
+
+The output of this command is shown in the screenshot below. Notice how
+:command:`ipcluster` prints out the location of the newly created cluster
+directory.
+
+.. image:: ipcluster_create.*
+
+Configuring a cluster profile
+-----------------------------
+
+Next, you will need to configure the newly created cluster profile by editing
+the following configuration files in the cluster directory:
+
+* :file:`ipcluster_config.py`
+* :file:`ipcontroller_config.py`
+* :file:`ipengine_config.py`
+
+When :command:`ipcluster` is run, these configuration files are used to 
+determine how the engines and controller will be started. In most cases,
+you will only have to set a few of the attributes in these files.
+
+To configure :command:`ipcluster` to use the Windows HPC job scheduler, you
+will need to edit the following attributes in the file
+:file:`ipcluster_config.py`::
+
+    # Set these at the top of the file to tell ipcluster to use the
+    # Windows HPC job scheduler.
+    c.Global.controller_launcher = \
+        'IPython.kernel.launcher.WindowsHPCControllerLauncher'
+    c.Global.engine_launcher = \
+        'IPython.kernel.launcher.WindowsHPCEngineSetLauncher'
+
+    # Set these to the host name of the scheduler (head node) of your cluster.
+    c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
+    c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
+
+There are a number of other configuration attributes that can be set, but
+in most cases these will be sufficient to get you started.
+
+.. warning::
+    If any of your configuration attributes involve specifying the location
+    of shared directories or files, you must make sure that you use UNC paths
+    like :file:`\\\\host\\share`. It is also important that you specify
+    these paths using raw Python strings: ``r'\\host\share'`` to make sure
+    that the backslashes are properly escaped.
+
+Starting the cluster profile
+----------------------------
+
+Once a cluster profile has been configured, starting an IPython cluster using
+the profile is simple::
+
+    ipcluster start -p mycluster -n 32
+
+The ``-n`` option tells :command:`ipcluster` how many engines to start (in
+this case 32). Stopping the cluster is as simple as typing Control-C.
+
+Using the HPC Job Manager
+-------------------------
+
+When ``ipcluster start`` is run the first time, :command:`ipcluster` creates
+two XML job description files in the cluster directory:
+
+* :file:`ipcontroller_job.xml`
+* :file:`ipengineset_job.xml`
+
+Once these files have been created, they can be imported into the HPC Job
+Manager application. Then, the controller and engines for that profile can be
+started using the HPC Job Manager directly, without using :command:`ipcluster`.
+However, anytime the cluster profile is re-configured, ``ipcluster start``
+must be run again to regenerate the XML job description files. The
+following screenshot shows what the HPC Job Manager interface looks like
+with a running IPython cluster.
+
+.. image:: hpc_job_manager.*
+
+Performing a simple interactive parallel computation
+====================================================
+
+Once you have started your IPython cluster, you can start to use it. To do
+this, open up a new Windows Command Prompt and start up IPython's interactive
+shell by typing::
+
+    ipython
+
+Then you can create a :class:`MultiEngineClient` instance for your profile and
+use the resulting instance to do a simple interactive parallel computation. In
+the code and screenshot that follows, we take a simple Python function and
+apply it to each element of an array of integers in parallel using the
+:meth:`MultiEngineClient.map` method:
+
+.. sourcecode:: ipython
+
+    In [1]: from IPython.kernel.client import *
+
+    In [2]: mec = MultiEngineClient(profile='mycluster')
+
+    In [3]: mec.get_ids()
+    Out[3]: [0, 1, 2, 3, 4, 5, 67, 8, 9, 10, 11, 12, 13, 14]
+
+    In [4]: def f(x):
+       ...:     return x**10
+
+    In [5]: mec.map(f, range(15))  # f is applied in parallel
+    Out[5]:
+    [0,
+     1,
+     1024,
+     59049,
+     1048576,
+     9765625,
+     60466176,
+     282475249,
+     1073741824,
+     3486784401L,
+     10000000000L,
+     25937424601L,
+     61917364224L,
+     137858491849L,
+     289254654976L]
+
+The :meth:`map` method has the same signature as Python's builtin :func:`map`
+function, but runs the calculation in parallel. More involved examples of using
+:class:`MultiEngineClient` are provided in the examples that follow.
+
+.. image:: mec_simple.*
+

docs/source/parallel/winhpc_index.txt

@@ -0,0 +1,14 b''
+========================================
+Using IPython on Windows HPC Server 2008
+========================================
+
+
+Contents
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   parallel_winhpc.txt
+   parallel_demos.txt
+

docs/sphinxext/ipython_directive.py

@@ -0,0 +1,641 b''
+# -*- coding: utf-8 -*-
+"""Sphinx directive to support embedded IPython code.
+
+This directive allows pasting of entire interactive IPython sessions, prompts
+and all, and their code will actually get re-executed at doc build time, with
+all prompts renumbered sequentially.
+
+To enable this directive, simply list it in your Sphinx ``conf.py`` file
+(making sure the directory where you placed it is visible to sphinx, as is
+needed for all Sphinx directives).
+
+By default this directive assumes that your prompts are unchanged IPython ones,
+but this can be customized.  For example, the following code in your Sphinx
+config file will configure this directive for the following input/output
+prompts ``Yade [1]:`` and ``-> [1]:``::
+
+ import ipython_directive as id
+ id.rgxin =re.compile(r'(?:In |Yade )\[(\d+)\]:\s?(.*)\s*')
+ id.rgxout=re.compile(r'(?:Out| ->  )\[(\d+)\]:\s?(.*)\s*')
+ id.fmtin ='Yade [%d]:'
+ id.fmtout=' ->  [%d]:'
+
+ from IPython import Config
+ id.CONFIG = Config(
+   prompt_in1="Yade [\#]:",
+   prompt_in2="     .\D..",
+   prompt_out=" ->  [\#]:"
+ )
+ id.reconfig_shell()
+
+ import ipython_console_highlighting as ich
+ ich.IPythonConsoleLexer.input_prompt=
+    re.compile("(Yade \[[0-9]+\]: )|(   \.\.\.+:)")
+ ich.IPythonConsoleLexer.output_prompt=
+    re.compile("(( ->  )|(Out)\[[0-9]+\]: )|(   \.\.\.+:)")
+ ich.IPythonConsoleLexer.continue_prompt=re.compile("   \.\.\.+:")
+
+
+ToDo
+----
+
+- Turn the ad-hoc test() function into a real test suite.
+- Break up ipython-specific functionality from matplotlib stuff into better
+  separated code.
+- Make sure %bookmarks used internally are removed on exit.
+
+
+Authors
+-------
+
+- John D Hunter: orignal author.
+- Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
+- VáclavŠmilauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import cStringIO
+import imp
+import os
+import re
+import shutil
+import sys
+import warnings
+
+# To keep compatibility with various python versions
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
+# Third-party
+import matplotlib
+import sphinx
+from docutils.parsers.rst import directives
+
+matplotlib.use('Agg')
+
+# Our own
+from IPython import Config, IPythonApp
+from IPython.utils.genutils import Term, Tee
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+sphinx_version = sphinx.__version__.split(".")
+# The split is necessary for sphinx beta versions where the string is
+# '6b1'
+sphinx_version = tuple([int(re.split('[a-z]', x)[0])
+                        for x in sphinx_version[:2]])
+
+COMMENT, INPUT, OUTPUT =  range(3)
+CONFIG = Config()
+rgxin = re.compile('In \[(\d+)\]:\s?(.*)\s*')
+rgxout = re.compile('Out\[(\d+)\]:\s?(.*)\s*')
+fmtin = 'In [%d]:'
+fmtout = 'Out[%d]:'
+
+#-----------------------------------------------------------------------------
+# Functions and class declarations
+#-----------------------------------------------------------------------------
+def block_parser(part):
+    """
+    part is a string of ipython text, comprised of at most one
+    input, one ouput, comments, and blank lines.  The block parser
+    parses the text into a list of::
+
+      blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
+
+    where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
+    data is, depending on the type of token::
+
+      COMMENT : the comment string
+
+      INPUT: the (DECORATOR, INPUT_LINE, REST) where
+         DECORATOR: the input decorator (or None)
+         INPUT_LINE: the input as string (possibly multi-line)
+         REST : any stdout generated by the input line (not OUTPUT)
+
+
+      OUTPUT: the output string, possibly multi-line
+    """
+
+    block = []
+    lines = part.split('\n')
+    N = len(lines)
+    i = 0
+    decorator = None
+    while 1:
+
+        if i==N:
+            # nothing left to parse -- the last line
+            break
+
+        line = lines[i]
+        i += 1
+        line_stripped = line.strip()
+        if line_stripped.startswith('#'):
+            block.append((COMMENT, line))
+            continue
+
+        if line_stripped.startswith('@'):
+            # we're assuming at most one decorator -- may need to
+            # rethink
+            decorator = line_stripped
+            continue
+
+        # does this look like an input line?
+        matchin = rgxin.match(line)
+        if matchin:
+            lineno, inputline = int(matchin.group(1)), matchin.group(2)
+
+            # the ....: continuation string
+            continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
+            Nc = len(continuation)
+            # input lines can continue on for more than one line, if
+            # we have a '\' line continuation char or a function call
+            # echo line 'print'.  The input line can only be
+            # terminated by the end of the block or an output line, so
+            # we parse out the rest of the input line if it is
+            # multiline as well as any echo text
+
+            rest = []
+            while i<N:
+
+                # look ahead; if the next line is blank, or a comment, or
+                # an output line, we're done
+
+                nextline = lines[i]
+                matchout = rgxout.match(nextline)
+                #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
+                if matchout or nextline.startswith('#'):
+                    break
+                elif nextline.startswith(continuation):
+                    inputline += '\n' + nextline[Nc:]
+                else:
+                    rest.append(nextline)
+                i+= 1
+
+            block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
+            continue
+
+        # if it looks like an output line grab all the text to the end
+        # of the block
+        matchout = rgxout.match(line)
+        if matchout:
+            lineno, output = int(matchout.group(1)), matchout.group(2)
+            if i<N-1:
+                output = '\n'.join([output] + lines[i:])
+
+            block.append((OUTPUT, output))
+            break
+
+    return block
+
+
+class EmbeddedSphinxShell(object):
+    """An embedded IPython instance to run inside Sphinx"""
+
+    def __init__(self):
+
+        self.cout = cStringIO.StringIO()
+        Term.cout = self.cout
+        Term.cerr = self.cout
+
+        # For debugging, so we can see normal output, use this:
+        #Term.cout = genutils.Tee(self.cout, channel='stdout') # dbg
+        #Term.cerr = genutils.Tee(self.cout, channel='stderr') # dbg
+
+        # Create config object for IPython
+        config = Config()
+        config.Global.display_banner = False
+        config.Global.exec_lines = ['import numpy as np',
+                                    'from pylab import *'
+                                    ]
+        config.InteractiveShell.autocall = False
+        config.InteractiveShell.autoindent = False
+        config.InteractiveShell.colors = 'NoColor'
+
+        # Merge global config which can be used to override.
+        config._merge(CONFIG)
+
+        # Create and initialize ipython, but don't start its mainloop
+        IP = IPythonApp(override_config=config)
+        IP.initialize()
+
+        # Store a few parts of IPython we'll need.
+        self.IP = IP.shell
+        self.user_ns = self.IP.user_ns
+        self.user_global_ns = self.IP.user_global_ns
+                                    
+        self.input = ''
+        self.output = ''
+
+        self.is_verbatim = False
+        self.is_doctest = False
+        self.is_suppress = False
+
+        # on the first call to the savefig decorator, we'll import
+        # pyplot as plt so we can make a call to the plt.gcf().savefig
+        self._pyplot_imported = False
+
+        # we need bookmark the current dir first so we can save
+        # relative to it
+        self.process_input_line('bookmark ipy_basedir')
+        self.cout.seek(0)
+        self.cout.truncate(0)
+
+    def process_input_line(self, line):
+        """process the input, capturing stdout"""
+        #print "input='%s'"%self.input
+        stdout = sys.stdout
+        try:
+            sys.stdout = self.cout
+            self.IP.push_line(line)
+        finally:
+            sys.stdout = stdout
+
+    # Callbacks for each type of token
+    def process_input(self, data, input_prompt, lineno):
+        """Process data block for INPUT token."""
+        decorator, input, rest = data
+        image_file = None
+        #print 'INPUT:', data  # dbg
+        is_verbatim = decorator=='@verbatim' or self.is_verbatim
+        is_doctest = decorator=='@doctest' or self.is_doctest
+        is_suppress = decorator=='@suppress' or self.is_suppress
+        is_savefig = decorator is not None and \
+                     decorator.startswith('@savefig')
+
+        input_lines = input.split('\n')
+
+        continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
+        Nc = len(continuation)
+
+        if is_savefig:
+            saveargs = decorator.split(' ')
+            filename = saveargs[1]
+            outfile = os.path.join('_static/%s'%filename)
+            # build out an image directive like
+            # .. image:: somefile.png
+            #    :width 4in
+            #
+            # from an input like
+            # savefig somefile.png width=4in
+            imagerows = ['.. image:: %s'%outfile]
+
+            for kwarg in saveargs[2:]:
+                arg, val = kwarg.split('=')
+                arg = arg.strip()
+                val = val.strip()
+                imagerows.append('   :%s: %s'%(arg, val))
+
+            image_file = outfile
+            image_directive = '\n'.join(imagerows)
+
+        # TODO: can we get "rest" from ipython
+        #self.process_input_line('\n'.join(input_lines))
+
+        ret = []
+        is_semicolon = False
+
+        for i, line in enumerate(input_lines):
+            if line.endswith(';'):
+                is_semicolon = True
+
+            if i==0:
+                # process the first input line
+                if is_verbatim:
+                    self.process_input_line('')
+                else:
+                    # only submit the line in non-verbatim mode
+                    self.process_input_line(line)
+                formatted_line = '%s %s'%(input_prompt, line)
+            else:
+                # process a continuation line
+                if not is_verbatim:
+                    self.process_input_line(line)
+
+                formatted_line = '%s %s'%(continuation, line)
+
+            if not is_suppress:
+                ret.append(formatted_line)
+
+        if not is_suppress:
+            if len(rest.strip()):
+                if is_verbatim:
+                    # the "rest" is the standard output of the
+                    # input, which needs to be added in
+                    # verbatim mode
+                    ret.append(rest)
+
+        self.cout.seek(0)
+        output = self.cout.read()
+        if not is_suppress and not is_semicolon:
+            ret.append(output)
+
+        self.cout.truncate(0)
+        return ret, input_lines, output, is_doctest, image_file
+        #print 'OUTPUT', output  # dbg
+
+    def process_output(self, data, output_prompt,
+                       input_lines, output, is_doctest, image_file):
+        """Process data block for OUTPUT token."""
+        if is_doctest:
+            submitted = data.strip()
+            found = output
+            if found is not None:
+                found = found.strip()
+                
+                # XXX - fperez: in 0.11, 'output' never comes with the prompt
+                # in it, just the actual output text.  So I think all this code
+                # can be nuked...
+                ## ind = found.find(output_prompt)
+                ## if ind<0:
+                ##     e='output prompt="%s" does not match out line=%s' % \
+                ##        (output_prompt, found)
+                ##     raise RuntimeError(e)
+                ## found = found[len(output_prompt):].strip()
+
+                if found!=submitted:
+                    e = ('doctest failure for input_lines="%s" with '
+                         'found_output="%s" and submitted output="%s"' %
+                         (input_lines, found, submitted) )
+                    raise RuntimeError(e)
+                #print 'doctest PASSED for input_lines="%s" with found_output="%s" and submitted output="%s"'%(input_lines, found, submitted)
+
+    def process_comment(self, data):
+        """Process data block for COMMENT token."""
+        if not self.is_suppress:
+            return [data]
+
+    def process_block(self, block):
+        """
+        process block from the block_parser and return a list of processed lines
+        """
+
+        ret = []
+        output = None
+        input_lines = None
+
+        m = rgxin.match(str(self.IP.outputcache.prompt1).strip())
+        lineno = int(m.group(1))
+
+        input_prompt = fmtin%lineno
+        output_prompt = fmtout%lineno
+        image_file = None
+        image_directive = None
+        # XXX - This needs a second refactor.  There's too much state being
+        # held globally, which makes for a very awkward interface and large,
+        # hard to test functions.  I've already broken this up at least into
+        # three separate processors to isolate the logic better, but this only
+        # serves to highlight the coupling.  Next we need to clean it up...
+        for token, data in block:
+            if token==COMMENT:
+                out_data = self.process_comment(data)
+            elif token==INPUT:
+                out_data, input_lines, output, is_doctest, image_file= \
+                          self.process_input(data, input_prompt, lineno)
+            elif token==OUTPUT:
+                out_data = \
+                    self.process_output(data, output_prompt,
+                                        input_lines, output, is_doctest,
+                                        image_file)
+            if out_data:
+                ret.extend(out_data)
+
+        if image_file is not None:
+            self.ensure_pyplot()
+            command = 'plt.gcf().savefig("%s")'%image_file
+            print 'SAVEFIG', command  # dbg
+            self.process_input_line('bookmark ipy_thisdir')
+            self.process_input_line('cd -b ipy_basedir')
+            self.process_input_line(command)
+            self.process_input_line('cd -b ipy_thisdir')
+            self.cout.seek(0)
+            self.cout.truncate(0)
+        return ret, image_directive
+
+    def ensure_pyplot(self):
+        if self._pyplot_imported:
+            return
+        self.process_input_line('import matplotlib.pyplot as plt')
+
+# A global instance used below. XXX: not sure why this can't be created inside
+# ipython_directive itself.
+shell = EmbeddedSphinxShell()
+
+def reconfig_shell():
+    """Called after setting module-level variables to re-instantiate
+    with the set values (since shell is instantiated first at import-time
+    when module variables have default values)"""
+    global shell
+    shell = EmbeddedSphinxShell()
+
+
+def ipython_directive(name, arguments, options, content, lineno,
+                      content_offset, block_text, state, state_machine,
+                      ):
+
+    debug = ipython_directive.DEBUG
+    shell.is_suppress = options.has_key('suppress')
+    shell.is_doctest = options.has_key('doctest')
+    shell.is_verbatim = options.has_key('verbatim')
+
+    #print 'ipy', shell.is_suppress, options
+    parts = '\n'.join(content).split('\n\n')
+    lines = ['.. sourcecode:: ipython', '']
+
+    figures = []
+    for part in parts:
+        block = block_parser(part)
+
+        if len(block):
+            rows, figure = shell.process_block(block)
+            for row in rows:
+                lines.extend(['    %s'%line for line in row.split('\n')])
+
+            if figure is not None:
+                figures.append(figure)
+
+    for figure in figures:
+        lines.append('')
+        lines.extend(figure.split('\n'))
+        lines.append('')
+
+    #print lines
+    if len(lines)>2:
+        if debug:
+            print '\n'.join(lines)
+        else:
+            #print 'INSERTING %d lines'%len(lines)
+            state_machine.insert_input(
+                lines, state_machine.input_lines.source(0))
+
+    return []
+
+ipython_directive.DEBUG = False
+ipython_directive.DEBUG = True  # dbg
+
+# Enable as a proper Sphinx directive
+def setup(app):
+    setup.app = app
+    options = {'suppress': directives.flag,
+               'doctest': directives.flag,
+               'verbatim': directives.flag,
+               }
+
+    app.add_directive('ipython', ipython_directive, True, (0, 2, 0), **options)
+
+
+# Simple smoke test, needs to be converted to a proper automatic test.
+def test():
+
+    examples = [
+        r"""
+In [9]: pwd
+Out[9]: '/home/jdhunter/py4science/book'
+
+In [10]: cd bookdata/
+/home/jdhunter/py4science/book/bookdata
+
+In [2]: from pylab import *
+
+In [2]: ion()
+
+In [3]: im = imread('stinkbug.png')
+
+@savefig mystinkbug.png width=4in
+In [4]: imshow(im)
+Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
+        
+""",
+        r"""
+
+In [1]: x = 'hello world'
+
+# string methods can be
+# used to alter the string
+@doctest
+In [2]: x.upper()
+Out[2]: 'HELLO WORLD'
+
+@verbatim
+In [3]: x.st<TAB>
+x.startswith  x.strip
+""",
+    r"""
+
+In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
+   .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
+
+In [131]: print url.split('&')
+['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
+
+In [60]: import urllib
+
+""",
+    r"""\
+
+In [133]: import numpy.random
+
+@suppress
+In [134]: numpy.random.seed(2358)
+
+@doctest
+In [135]: np.random.rand(10,2)
+Out[135]:
+array([[ 0.64524308,  0.59943846],
+       [ 0.47102322,  0.8715456 ],
+       [ 0.29370834,  0.74776844],
+       [ 0.99539577,  0.1313423 ],
+       [ 0.16250302,  0.21103583],
+       [ 0.81626524,  0.1312433 ],
+       [ 0.67338089,  0.72302393],
+       [ 0.7566368 ,  0.07033696],
+       [ 0.22591016,  0.77731835],
+       [ 0.0072729 ,  0.34273127]])
+
+""",
+
+    r"""
+In [106]: print x
+jdh
+
+In [109]: for i in range(10):
+   .....:     print i
+   .....:
+   .....:
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+""",
+
+        r"""
+
+In [144]: from pylab import *
+
+In [145]: ion()
+
+# use a semicolon to suppress the output
+@savefig test_hist.png width=4in
+In [151]: hist(np.random.randn(10000), 100);
+
+
+@savefig test_plot.png width=4in
+In [151]: plot(np.random.randn(10000), 'o');
+   """,
+
+        r"""
+# use a semicolon to suppress the output
+In [151]: plt.clf()
+
+@savefig plot_simple.png width=4in
+In [151]: plot([1,2,3])
+
+@savefig hist_simple.png width=4in
+In [151]: hist(np.random.randn(10000), 100);
+
+""",
+     r"""
+# update the current fig
+In [151]: ylabel('number')
+
+In [152]: title('normal distribution')
+
+
+@savefig hist_with_text.png
+In [153]: grid(True)
+
+        """,
+        ]
+
+    #ipython_directive.DEBUG = True  # dbg
+    #options = dict(suppress=True)  # dbg
+    options = dict()
+    for example in examples:
+        content = example.split('\n')
+        ipython_directive('debug', arguments=None, options=options,
+                          content=content, lineno=0,
+                          content_offset=None, block_text=None,
+                          state=None, state_machine=None,
+                          )
+
+# Run test suite as a script
+if __name__=='__main__':
+    if not os.path.isdir('_static'):
+        os.mkdir('_static')
+    test()
+    print 'All OK? Check figures in _static/'

IPython/__init__.py

@@ -16,18 +16,17 b' IPython is a set of tools for interactive and exploratory computing in Python.'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import absolute_import
 
 import os
 import sys
-from IPython.core import release
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
-
-if sys.version[0:3] < '2.4':
-    raise ImportError('Python Version 2.4 or above is required for IPython.')
+if sys.version[0:3] < '2.5':
+    raise ImportError('Python Version 2.5 or above is required for IPython.')
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
@@ -39,11 +38,16 b' sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))'
 #-----------------------------------------------------------------------------
 
 # In some cases, these are causing circular imports.
-from IPython.core.iplib import InteractiveShell
-from IPython.core.embed import embed
-from IPython.core.error import TryNext
+from .config.loader import Config
+from .core import release
+from .core.application import Application
+from .core.ipapp import IPythonApp
+from .core.embed import embed
+from .core.error import TryNext
+from .core.iplib import InteractiveShell
+from .testing import test
 
-from IPython.lib import (
+from .lib import (
     enable_wx, disable_wx,
     enable_gtk, disable_gtk,
     enable_qt4, disable_qt4,
@@ -61,4 +65,3 b' for author, email in release.authors.values():'
 __license__  = release.license
 __version__  = release.version
 __revision__ = release.revision
-

IPython/config/default/ipython_config.py

@@ -13,7 +13,7 b' c = get_config()'
 
 # Set this to determine the detail of what is logged at startup.
 # The default is 30 and possible values are 0,10,20,30,40,50.
-c.Global.log_level = 20
+# c.Global.log_level = 20
 
 # This should be a list of importable Python modules that have an
 # load_in_ipython(ip) method.  This method gets called when the extension
@@ -35,7 +35,7 b' c.Global.log_level = 20'
 # These files are run in IPython in the user's namespace.  Files with a .py
 # extension need to be pure Python.  Files with a .ipy extension can have
 # custom IPython syntax (like magics, etc.).  
-# These files need to be in the cwd, the ipythondir or be absolute paths.
+# These files need to be in the cwd, the ipython_dir or be absolute paths.
 # c.Global.exec_files = [
 #     'mycode.py',
 #     'fancy.ipy'
@@ -71,9 +71,9 b' c.Global.log_level = 20'
 
 # c.InteractiveShell.logstart = True
 
-# c.InteractiveShell.logfile = 'ipython_log.py'
+# c.InteractiveShell.logfile = u'ipython_log.py'
 
-# c.InteractiveShell.logappend = 'mylog.py'
+# c.InteractiveShell.logappend = u'mylog.py'
 
 # c.InteractiveShell.object_info_string_level = 0
 

IPython/config/loader.py

@@ -1,10 +1,10 b''
-#!/usr/bin/env python
-# encoding: utf-8
+# coding: utf-8
 """A simple configuration system.
 
-Authors:
-
+Authors
+-------
 * Brian Granger
+* Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
@@ -37,7 +37,25 b' class ConfigError(Exception):'
 class ConfigLoaderError(ConfigError):
     pass
 
-
+#-----------------------------------------------------------------------------
+# Argparse fix
+#-----------------------------------------------------------------------------
+# Unfortunately argparse by default prints help messages to stderr instead of
+# stdout.  This makes it annoying to capture long help screens at the command
+# line, since one must know how to pipe stderr, which many users don't know how
+# to do.  So we override the print_help method with one that defaults to
+# stdout and use our class instead.
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Simple argparse subclass that prints help to stdout by default."""
+    
+    def print_help(self, file=None):
+        if file is None:
+            file = sys.stdout
+        return super(ArgumentParser, self).print_help(file)
+    
+    print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
+    
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
@@ -244,8 +262,14 b' class PyFileConfigLoader(FileConfigLoader):'
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
-            sub_config = loader.load_config()
-            self.config._merge(sub_config)
+            try:
+                sub_config = loader.load_config()
+            except IOError:
+                # Pass silently if the sub config is not there. This happens
+                # when a user us using a profile, but not the default config.
+                pass
+            else:
+                self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
@@ -268,26 +292,55 b' class CommandLineConfigLoader(ConfigLoader):'
     """
 
 
-class NoConfigDefault(object): pass
-NoConfigDefault = NoConfigDefault()
+class __NoConfigDefault(object): pass
+NoConfigDefault = __NoConfigDefault()
+
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
+    #: Global default for arguments (see argparse docs for details)
+    argument_default = NoConfigDefault
     
-    # arguments = [(('-f','--file'),dict(type=str,dest='file'))]
-    arguments = ()
-
-    def __init__(self, *args, **kw):
+    def __init__(self, argv=None, arguments=(), *args, **kw):
         """Create a config loader for use with argparse.
 
-        The args and kwargs arguments here are passed onto the constructor
-        of :class:`argparse.ArgumentParser`.
+        With the exception of ``argv`` and ``arguments``, other args and kwargs
+        arguments here are passed onto the constructor of
+        :class:`argparse.ArgumentParser`.
+
+        Parameters
+        ----------
+
+        argv : optional, list
+          If given, used to read command-line arguments from, otherwise
+          sys.argv[1:] is used.
+
+        arguments : optional, tuple
+          Description of valid command-line arguments, to be called in sequence
+          with parser.add_argument() to configure the parser.
         """
         super(CommandLineConfigLoader, self).__init__()
+        if argv == None:
+            argv = sys.argv[1:]
+        self.argv = argv
+        self.arguments = arguments
         self.args = args
-        self.kw = kw
+        kwargs = dict(argument_default=self.argument_default)
+        kwargs.update(kw)
+        self.kw = kwargs
 
     def load_config(self, args=None):
-        """Parse command line arguments and return as a Struct."""
+        """Parse command line arguments and return as a Struct.
+
+        Parameters
+        ----------
+
+        args : optional, list
+          If given, a list with the structure of sys.argv[1:] to parse arguments
+          from.  If not given, the instance's self.argv attribute (given at
+          construction time) is used."""
+        
+        if args is None:
+            args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
@@ -300,25 +353,21 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
             return []
 
     def _create_parser(self):
-        self.parser = argparse.ArgumentParser(*self.args, **self.kw)
+        self.parser = ArgumentParser(*self.args, **self.kw)
         self._add_arguments()
         self._add_other_arguments()
 
-    def _add_other_arguments(self):
-        pass
-
     def _add_arguments(self):
         for argument in self.arguments:
-            if not argument[1].has_key('default'):
-                argument[1]['default'] = NoConfigDefault
             self.parser.add_argument(*argument[0],**argument[1])
 
-    def _parse_args(self, args=None):
-        """self.parser->self.parsed_data"""
-        if args is None:
-            self.parsed_data, self.extra_args = self.parser.parse_known_args()
-        else:
-            self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
+    def _add_other_arguments(self):
+        """Meant for subclasses to add their own arguments."""
+        pass
+
+    def _parse_args(self, args):
+        """self.parser->self.parsed_data""" 
+        self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
@@ -326,4 +375,3 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
             if v is not NoConfigDefault:
                 exec_str = 'self.config.' + k + '= v'
                 exec exec_str in locals(), globals()
-

IPython/config/tests/test_loader.py

@@ -37,17 +37,18 b' from IPython.config.loader import ('
 
 
 pyfile = """
-a = 10
-b = 20
-Foo.Bar.value = 10
-Foo.Bam.value = range(10)
-D.C.value = 'hi there'
+c = get_config()
+c.a = 10
+c.b = 20
+c.Foo.Bar.value = 10
+c.Foo.Bam.value = range(10)
+c.D.C.value = 'hi there'
 """
 
 class TestPyFileCL(TestCase):
 
     def test_basic(self):
-        fd, fname = mkstemp()
+        fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
@@ -65,15 +66,13 b' class TestArgParseCL(TestCase):'
 
     def test_basic(self):
 
-        class MyLoader(ArgParseConfigLoader):
-            arguments = (
+        arguments = (
                 (('-f','--foo'), dict(dest='Global.foo', type=str)),
                 (('-b',), dict(dest='MyClass.bar', type=int)),
                 (('-n',), dict(dest='n', action='store_true')),
                 (('Global.bam',), dict(type=str))
             )
-
-        cl = MyLoader()
+        cl = ArgParseConfigLoader(arguments=arguments)
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEquals(config.Global.foo, 'hi')
         self.assertEquals(config.MyClass.bar, 10)

IPython/core/application.py

@@ -1,7 +1,12 b''
-#!/usr/bin/env python
 # encoding: utf-8
 """
-An application for IPython
+An application for IPython.
+
+All top-level applications should use the classes in this module for
+handling configuration and creating componenets.
+
+The job of an :class:`Application` is to create the master configuration 
+object and then create the components, passing the config to them.
 
 Authors:
 
@@ -26,62 +31,131 b' Notes'
 import logging
 import os
 import sys
-import traceback
-from copy import deepcopy
 
-from IPython.utils.genutils import get_ipython_dir, filefind
+from IPython.core import release, crashhandler
+from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
-    NoConfigDefault
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-
-class IPythonArgParseConfigLoader(ArgParseConfigLoader):
-    """Default command line options for IPython based applications."""
-
-    def _add_other_arguments(self):
-        self.parser.add_argument('-ipythondir',dest='Global.ipythondir',type=str,
-            help='Set to override default location of Global.ipythondir.',
-            default=NoConfigDefault,
-            metavar='Global.ipythondir')
-        self.parser.add_argument('-p','-profile',dest='Global.profile',type=str,
-            help='The string name of the ipython profile to be used.',
-            default=NoConfigDefault,
-            metavar='Global.profile')
-        self.parser.add_argument('-log_level',dest="Global.log_level",type=int,
-            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            default=NoConfigDefault)
-        self.parser.add_argument('-config_file',dest='Global.config_file',type=str,
-            help='Set the config file name to override default.',
-            default=NoConfigDefault,
-            metavar='Global.config_file')
-
-
 class ApplicationError(Exception):
     pass
 
 
+app_cl_args = (
+    (('--ipython-dir', ), dict(
+        dest='Global.ipython_dir',type=unicode,
+        help=
+        """Set to override default location of the IPython directory
+        IPYTHON_DIR, stored as Global.ipython_dir.  This can also be specified
+        through the environment variable IPYTHON_DIR.""",
+        metavar='Global.ipython_dir') ),
+    (('-p', '--profile',), dict(
+        dest='Global.profile',type=unicode,
+        help=
+        """The string name of the ipython profile to be used.  Assume that your
+        config file is ipython_config-<name>.py (looks in current dir first,
+        then in IPYTHON_DIR). This is a quick way to keep and load multiple
+        config files for different tasks, especially if include your basic one
+        in your more specialized ones.  You can keep a basic
+        IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
+        include this one and load extra things for particular tasks.""",
+        metavar='Global.profile') ),
+    (('--log-level',), dict(
+        dest="Global.log_level",type=int,
+        help='Set the log level (0,10,20,30,40,50).  Default is 30.',
+        metavar='Global.log_level')),
+    (('--config-file',), dict(
+        dest='Global.config_file',type=unicode,
+        help=
+        """Set the config file name to override default.  Normally IPython
+        loads ipython_config.py (from current directory) or
+        IPYTHON_DIR/ipython_config.py.  If the loading of your config file
+        fails, IPython starts with a bare bones configuration (no modules
+        loaded at all).""",
+        metavar='Global.config_file')),
+    )
+
 class Application(object):
-    """Load a config, construct an app and run it.
+    """Load a config, construct components and set them running.
+
+    The configuration of an application can be done via four different Config
+    objects, which are loaded and ultimately merged into a single one used from
+    that point on by the app.  These are:
+
+    1. default_config: internal defaults, implemented in code.
+    2. file_config: read from the filesystem.
+    3. command_line_config: read from the system's command line flags.
+    4. constructor_config: passed parametrically to the constructor.
+
+    During initialization, 3 is actually read before 2, since at the
+    command-line one may override the location of the file to be read.  But the
+    above is the order in which the merge is made.
+
+    There is a final config object can be created and passed to the
+    constructor: override_config.  If it exists, this completely overrides the
+    configs 2-4 above (the default is still used to ensure that all needed
+    fields at least are created).  This makes it easier to create
+    parametrically (e.g. in testing or sphinx plugins) objects with a known
+    configuration, that are unaffected by whatever arguments may be present in
+    sys.argv or files in the user's various directories.
     """
 
-    config_file_name = 'ipython_config.py'
-    name = 'ipython'
-
-    def __init__(self):
+    name = u'ipython'
+    description = 'IPython: an enhanced interactive Python shell.'
+    #: usage message printed by argparse. If None, auto-generate
+    usage = None
+    config_file_name = u'ipython_config.py'
+    #: Track the default and actual separately because some messages are
+    #: only printed if we aren't using the default.
+    default_config_file_name = config_file_name
+    default_log_level = logging.WARN
+    #: Set by --profile option
+    profile_name = None
+    #: User's ipython directory, typically ~/.ipython/
+    ipython_dir = None
+    #: internal defaults, implemented in code.
+    default_config = None
+    #: read from the filesystem
+    file_config = None
+    #: read from the system's command line flags
+    command_line_config = None
+    #: passed parametrically to the constructor.
+    constructor_config = None
+    #: final override, if given supercedes file/command/constructor configs
+    override_config = None
+    #: A reference to the argv to be used (typically ends up being sys.argv[1:])
+    argv = None
+    #: Default command line arguments.  Subclasses should create a new tuple
+    #: that *includes* these.
+    cl_arguments = app_cl_args
+
+    #: extra arguments computed by the command-line loader
+    extra_args = None
+
+    # Private attributes
+    _exiting = False
+    _initialized = False
+
+    # Class choices for things that will be instantiated at runtime.
+    _CrashHandler = crashhandler.CrashHandler
+
+    def __init__(self, argv=None, constructor_config=None, override_config=None):
+        self.argv = sys.argv[1:] if argv is None else argv
+        self.constructor_config = constructor_config
+        self.override_config = override_config
         self.init_logger()
-        self.default_config_file_name = self.config_file_name
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
-        self.log.setLevel(logging.WARN)
+        self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
@@ -95,28 +169,80 b' class Application(object):'
 
     log_level = property(_get_log_level, _set_log_level)
 
+    def initialize(self):
+        """Initialize the application.
+
+        Loads all configuration information and sets all application state, but
+        does not start any relevant processing (typically some kind of event
+        loop).
+
+        Once this method has been called, the application is flagged as
+        initialized and the method becomes a no-op."""
+        
+        if self._initialized:
+            return
+
+        # The first part is protected with an 'attempt' wrapper, that will log
+        # failures with the basic system traceback machinery.  Once our crash
+        # handler is in place, we can let any subsequent exception propagate,
+        # as our handler will log it with much better detail than the default.
+        self.attempt(self.create_crash_handler)
+
+        # Configuration phase
+        # Default config (internally hardwired in application code)
+        self.create_default_config()
+        self.log_default_config()
+        self.set_default_config_log_level()
+
+        if self.override_config is None:
+            # Command-line config
+            self.pre_load_command_line_config()
+            self.load_command_line_config()
+            self.set_command_line_config_log_level()
+            self.post_load_command_line_config()
+            self.log_command_line_config()
+
+        # Find resources needed for filesystem access, using information from
+        # the above two
+        self.find_ipython_dir()
+        self.find_resources()
+        self.find_config_file_name()
+        self.find_config_file_paths()
+
+        if self.override_config is None:
+            # File-based config
+            self.pre_load_file_config()
+            self.load_file_config()
+            self.set_file_config_log_level()
+            self.post_load_file_config()
+            self.log_file_config()
+
+        # Merge all config objects into a single one the app can then use
+        self.merge_configs()
+        self.log_master_config()
+
+        # Construction phase
+        self.pre_construct()
+        self.construct()
+        self.post_construct()
+
+        # Done, flag as such and
+        self._initialized = True
+
     def start(self):
         """Start the application."""
-        self.attempt(self.create_default_config)
-        self.attempt(self.pre_load_command_line_config)
-        self.attempt(self.load_command_line_config, action='abort')
-        self.attempt(self.post_load_command_line_config)
-        self.attempt(self.find_ipythondir)
-        self.attempt(self.find_config_file_name)
-        self.attempt(self.find_config_file_paths)
-        self.attempt(self.pre_load_file_config)
-        self.attempt(self.load_file_config)
-        self.attempt(self.post_load_file_config)
-        self.attempt(self.merge_configs)
-        self.attempt(self.pre_construct)
-        self.attempt(self.construct)
-        self.attempt(self.post_construct)
-        self.attempt(self.start_app)
+        self.initialize()
+        self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
+    def create_crash_handler(self):
+        """Create a crash handler, typically setting sys.excepthook to it."""
+        self.crash_handler = self._CrashHandler(self, self.name)
+        sys.excepthook = self.crash_handler
+
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
@@ -126,66 +252,91 b' class Application(object):'
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
-        self.default_config = Config()
-        self.default_config.Global.ipythondir = get_ipython_dir()
+        c = Config()
+        c.Global.ipython_dir = get_ipython_dir()
+        c.Global.log_level = self.log_level
+        self.default_config = c
+
+    def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
+    def set_default_config_log_level(self):
+        try:
+            self.log_level = self.default_config.Global.log_level
+        except AttributeError:
+            # Fallback to the default_log_level class attribute
+            pass
+
     def create_command_line_config(self):
         """Create and return a command line config loader."""
-        return IPythonArgParseConfigLoader(description=self.name)
+        return ArgParseConfigLoader(self.argv, self.cl_arguments,
+                                    description=self.description, 
+                                    version=release.version,
+                                    usage=self.usage,
+                                    )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
-        """Load the command line config.
-
-        This method also sets ``self.debug``.
-        """
-
+        """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
+    def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
-            pass # Use existing value which is set in Application.init_logger.
-        self.log.debug("Command line config loaded:")
-        self.log.debug(repr(self.command_line_config))
+            pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
-    def find_ipythondir(self):
+    def log_command_line_config(self):
+        self.log.debug("Command line config loaded:")
+        self.log.debug(repr(self.command_line_config))
+
+    def find_ipython_dir(self):
         """Set the IPython directory.
 
-        This sets ``self.ipythondir``, but the actual value that is passed
-        to the application is kept in either ``self.default_config`` or
-        ``self.command_line_config``.  This also added ``self.ipythondir`` to
-        ``sys.path`` so config files there can be references by other config
+        This sets ``self.ipython_dir``, but the actual value that is passed to
+        the application is kept in either ``self.default_config`` or
+        ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
+        ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
-            self.ipythondir = self.command_line_config.Global.ipythondir
+            self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
-            self.ipythondir = self.default_config.Global.ipythondir
-        sys.path.append(os.path.abspath(self.ipythondir))
-        if not os.path.isdir(self.ipythondir):
-            os.makedirs(self.ipythondir, mode = 0777)
-        self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
+            self.ipython_dir = self.default_config.Global.ipython_dir
+        sys.path.append(os.path.abspath(self.ipython_dir))
+        if not os.path.isdir(self.ipython_dir):
+            os.makedirs(self.ipython_dir, mode=0777)
+        self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
+
+    def find_resources(self):
+        """Find other resources that need to be in place.
+
+        Things like cluster directories need to be in place to find the
+        config file.  These happen right after the IPython directory has
+        been set.
+        """
+        pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
-        If a profile has been set at the command line, this will resolve
-        it.  The search paths for the config file are set in
-        :meth:`find_config_file_paths` and then passed to the config file
-        loader where they are resolved to an absolute path.
+        This must set ``self.config_file_name`` to the filename of the
+        config file to use (just the filename). The search paths for the
+        config file are set in :meth:`find_config_file_paths` and then passed
+        to the config file loader where they are resolved to an absolute path.
+
+        If a profile has been set at the command line, this will resolve it.
         """
 
         try:
@@ -195,15 +346,24 b' class Application(object):'
 
         try:
             self.profile_name = self.command_line_config.Global.profile
-            name_parts = self.config_file_name.split('.')
-            name_parts.insert(1, '_' + self.profile_name + '.')
-            self.config_file_name = ''.join(name_parts)
         except AttributeError:
             pass
+        else:
+            name_parts = self.config_file_name.split('.')
+            name_parts.insert(1, u'_' + self.profile_name + u'.')
+            self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
-        """Set the search paths for resolving the config file."""
-        self.config_file_paths = (os.getcwd(), self.ipythondir)
+        """Set the search paths for resolving the config file.
+
+        This must set ``self.config_file_paths`` to a sequence of search
+        paths to pass to the config file loader.
+        """
+        # Include our own profiles directory last, so that users can still find
+        # our shipped copies of builtin profiles even if they don't have them
+        # in their local ipython directory.
+        prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
+        self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
@@ -216,7 +376,8 b' class Application(object):'
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
-        self.log.debug("Attempting to load config file: <%s>" % self.config_file_name)
+        self.log.debug("Attempting to load config file: %s" %
+                       self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
@@ -225,19 +386,18 b' class Application(object):'
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
-                self.log.warn("Config file not found, skipping: <%s>" % \
+                self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
-            self.log.warn("Error loading config file: <%s>" % \
-                           self.config_file_name, exc_info=True)
+            self.log.warn("Error loading config file: %s" %
+                          self.config_file_name, exc_info=True)
             self.file_config = Config()
-        else:
-            self.log.debug("Config file loaded: <%s>" % loader.full_filename)
-            self.log.debug(repr(self.file_config))
+
+    def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
-        # line.
+        # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
@@ -248,13 +408,31 b' class Application(object):'
         """Do actions after the config file is loaded."""
         pass
 
+    def log_file_config(self):
+        if hasattr(self.file_config.Global, 'config_file'):
+            self.log.debug("Config file loaded: %s" %
+                           self.file_config.Global.config_file)
+            self.log.debug(repr(self.file_config))
+
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
-        config._merge(self.file_config)
-        config._merge(self.command_line_config)
+        if self.override_config is None:
+            config._merge(self.file_config)
+            config._merge(self.command_line_config)
+            if self.constructor_config is not None:
+                config._merge(self.constructor_config)
+        else:
+            config._merge(self.override_config)
+        # XXX fperez - propose to Brian we rename master_config to simply
+        # config, I think this is going to be heavily used in examples and
+        # application code and the name is shorter/easier to find/remember.
+        # For now, just alias it...
         self.master_config = config
+        self.config = config
+
+    def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
@@ -280,21 +458,31 b' class Application(object):'
 
     def abort(self):
         """Abort the starting of the application."""
-        self.log.critical("Aborting application: %s" % self.name, exc_info=True)
-        sys.exit(1)
+        if self._exiting:
+            pass
+        else:
+            self.log.critical("Aborting application: %s" % self.name, exc_info=True)
+            self._exiting = True
+            sys.exit(1)
 
-    def exit(self):
-        self.log.critical("Aborting application: %s" % self.name)
-        sys.exit(1)
+    def exit(self, exit_status=0):
+        if self._exiting:
+            pass
+        else:
+            self.log.debug("Exiting application: %s" % self.name)
+            self._exiting = True
+            sys.exit(exit_status)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
-            self.exit()
+            raise
         except:
             if action == 'abort':
+                self.log.critical("Aborting application: %s" % self.name,
+                                  exc_info=True)
                 self.abort()
+                raise
             elif action == 'exit':
-                self.exit()
-      
+                self.exit(0)

IPython/core/builtin_trap.py

@@ -31,8 +31,8 b' from IPython.utils.autoattr import auto_attr'
 #-----------------------------------------------------------------------------
 
 
-class BuiltinUndefined(object): pass
-BuiltinUndefined = BuiltinUndefined()
+class __BuiltinUndefined(object): pass
+BuiltinUndefined = __BuiltinUndefined()
 
 
 class BuiltinTrap(Component):
@@ -86,6 +86,7 b' class BuiltinTrap(Component):'
         """Store ipython references in the __builtin__ namespace."""
         self.add_builtin('exit', Quitter(self.shell, 'exit'))
         self.add_builtin('quit', Quitter(self.shell, 'quit'))
+        self.add_builtin('get_ipython', self.shell.get_ipython)
 
         # Recursive reload function
         try:

IPython/core/completer.py

@@ -44,7 +44,6 b' its input.'
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
-
 """
 
 #*****************************************************************************
@@ -54,14 +53,19 b' used, and this module (and the readline module) are silently inactive.'
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
+#       Copyright (C) 2008-2010 IPython Development Team
+#       Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
-#       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 import __builtin__
 import __main__
 import glob
@@ -73,23 +77,57 b' import shlex'
 import sys
 import types
 
+import IPython.utils.rlineimpl as readline
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
-
-import IPython.utils.rlineimpl as readline
-from IPython.utils.ipstruct import Struct
 from IPython.utils import generics
-
-# Python 2.4 offers sets as a builtin
-try:
-    set()
-except NameError:
-    from sets import Set as set
-
 from IPython.utils.genutils import debugx, dir2
 
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Public API
 __all__ = ['Completer','IPCompleter']
 
+if sys.platform == 'win32':
+    PROTECTABLES = ' '
+else:
+    PROTECTABLES = ' ()'
+
+#-----------------------------------------------------------------------------
+# Main functions and classes
+#-----------------------------------------------------------------------------
+
+def protect_filename(s):
+    """Escape a string to protect certain characters."""
+    
+    return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
+                    for ch in s])
+
+
+def single_dir_expand(matches):
+    "Recursively expand match lists containing a single dir."
+
+    if len(matches) == 1 and os.path.isdir(matches[0]):
+        # Takes care of links to directories also.  Use '/'
+        # explicitly, even under Windows, so that name completions
+        # don't end up escaped.
+        d = matches[0]
+        if d[-1] in ['/','\\']:
+            d = d[:-1]
+
+        subdirs = os.listdir(d)
+        if subdirs:
+            matches = [ (d + '/' + p) for p in subdirs]
+            return single_dir_expand(matches)
+        else:
+            return matches
+    else:
+        return matches
+
+class Bunch: pass
+
 class Completer:
     def __init__(self,namespace=None,global_namespace=None):
         """Create a new completer for the command line.
@@ -152,6 +190,7 b' class Completer:'
         defined in self.namespace or self.global_namespace that match.
 
         """
+        #print 'Completer->global_matches, txt=%r' % text # dbg
         matches = []
         match_append = matches.append
         n = len(text)
@@ -179,6 +218,7 b' class Completer:'
         """
         import re
 
+        #print 'Completer->attr_matches, txt=%r' % text # dbg
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
@@ -205,6 +245,7 b' class Completer:'
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
+
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
@@ -235,7 +276,7 b' class IPCompleter(Completer):'
         to complete. """
 
         Completer.__init__(self,namespace,global_namespace)
-        self.magic_prefix = shell.name+'.magic_'
+
         self.magic_escape = ESC_MAGIC
         self.readline = readline
         delims = self.readline.get_completer_delims()
@@ -244,7 +285,8 b' class IPCompleter(Completer):'
         self.get_line_buffer = self.readline.get_line_buffer
         self.get_endidx = self.readline.get_endidx
         self.omit__names = omit__names
-        self.merge_completions = shell.readline_merge_completions        
+        self.merge_completions = shell.readline_merge_completions
+        self.shell = shell.shell
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
@@ -263,11 +305,13 b' class IPCompleter(Completer):'
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
+
+        # All active matcher routines for completion
         self.matchers = [self.python_matches,
                          self.file_matches,
+                         self.magic_matches,
                          self.alias_matches,
                          self.python_func_kw_matches]
-
     
     # Code contributed by Alex Schmolck, for ipython/emacs integration
     def all_completions(self, text):
@@ -278,9 +322,8 b' class IPCompleter(Completer):'
         try:
             for i in xrange(sys.maxint):
                 res = self.complete(text, i)
-
-                if not res: break
-
+                if not res:
+                    break
                 comp_append(res)
         #XXX workaround for ``notDefined.<tab>``
         except NameError:
@@ -316,41 +359,12 b' class IPCompleter(Completer):'
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
-        if sys.platform == 'win32':
-            protectables = ' '
-        else:
-            protectables = ' ()'
-
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
             
-        def protect_filename(s):
-            return "".join([(ch in protectables and '\\' + ch or ch)
-                            for ch in s])
-
-        def single_dir_expand(matches):
-            "Recursively expand match lists containing a single dir."
-            
-            if len(matches) == 1 and os.path.isdir(matches[0]):
-                # Takes care of links to directories also.  Use '/'
-                # explicitly, even under Windows, so that name completions
-                # don't end up escaped.
-                d = matches[0]
-                if d[-1] in ['/','\\']:
-                    d = d[:-1]
-
-                subdirs = os.listdir(d)
-                if subdirs:
-                    matches = [ (d + '/' + p) for p in subdirs]
-                    return single_dir_expand(matches)
-                else:
-                    return matches
-            else:
-                return matches
-        
         lbuf = self.lbuf
         open_quotes = 0  # track strings with open quotes
         try:
@@ -402,13 +416,24 b' class IPCompleter(Completer):'
         #print 'mm',matches  # dbg
         return single_dir_expand(matches)
 
+    def magic_matches(self, text):
+        """Match magics"""
+        #print 'Completer->magic_matches:',text,'lb',self.lbuf # dbg
+        # Get all shell magics now rather than statically, so magics loaded at
+        # runtime show up too
+        magics = self.shell.lsmagic()
+        pre = self.magic_escape
+        baretext = text.lstrip(pre)
+        return [ pre+m for m in magics if m.startswith(baretext)]
+
     def alias_matches(self, text):
         """Match internal system aliases"""        
         #print 'Completer->alias_matches:',text,'lb',self.lbuf # dbg
         
         # if we are not in the first 'item', alias matching 
         # doesn't make sense - unless we are starting with 'sudo' command.
-        if ' ' in self.lbuf.lstrip() and not self.lbuf.lstrip().startswith('sudo'):
+        if ' ' in self.lbuf.lstrip() and \
+               not self.lbuf.lstrip().startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
@@ -420,7 +445,7 b' class IPCompleter(Completer):'
     def python_matches(self,text):
         """Match attributes or global python names"""
 
-        #print 'Completer->python_matches, txt=<%s>' % text # dbg
+        #print 'Completer->python_matches, txt=%r' % text # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
@@ -439,11 +464,7 b' class IPCompleter(Completer):'
                 matches = []
         else:
             matches = self.global_matches(text)
-            # this is so completion finds magics when automagic is on:
-            if (matches == [] and 
-                 not text.startswith(os.sep) and
-                 not ' ' in self.lbuf):
-                matches = self.attr_matches(self.magic_prefix+text)
+
         return matches
 
     def _default_arguments(self, obj):
@@ -514,9 +535,11 b' class IPCompleter(Completer):'
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
-            try: namedArgs = self._default_arguments(eval(callableMatch,
+            try:
+                namedArgs = self._default_arguments(eval(callableMatch,
                                                          self.namespace))
-            except: continue
+            except:
+                continue
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
@@ -528,7 +551,7 b' class IPCompleter(Completer):'
         if not line.strip():
             return None
 
-        event = Struct()
+        event = Bunch()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
@@ -540,11 +563,9 b' class IPCompleter(Completer):'
             try_magic = self.custom_completers.s_matches(
               self.magic_escape + cmd)            
         else:
-            try_magic = []
-        
+            try_magic = []        
         
-        for c in itertools.chain(
-                                 self.custom_completers.s_matches(cmd),
+        for c in itertools.chain(self.custom_completers.s_matches(cmd),
                                  try_magic,
                                  self.custom_completers.flat_matches(self.lbuf)):
             #print "try",c # dbg
@@ -555,7 +576,8 b' class IPCompleter(Completer):'
                 if withcase:
                     return withcase
                 # if none, then case insensitive ones are ok too
-                return [r for r in res if r.lower().startswith(text.lower())]
+                text_low = text.lower()
+                return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
             
@@ -598,14 +620,11 b' class IPCompleter(Completer):'
             return None
         
         magic_escape = self.magic_escape
-        magic_prefix = self.magic_prefix
 
         self.lbuf = self.full_lbuf[:self.get_endidx()]
 
         try:
-            if text.startswith(magic_escape):
-                text = text.replace(magic_escape,magic_prefix)
-            elif text.startswith('~'):
+            if text.startswith('~'):
                 text = os.path.expanduser(text)
             if state == 0:
                 custom_res = self.dispatch_custom_completer(text)
@@ -625,13 +644,10 b' class IPCompleter(Completer):'
                             self.matches = matcher(text)
                             if self.matches:
                                 break
-                    def uniq(alist):
-                        set = {}
-                        return [set.setdefault(e,e) for e in alist if e not in set]
-                    self.matches = uniq(self.matches)                
+                    self.matches = list(set(self.matches))
             try:
-                ret = self.matches[state].replace(magic_prefix,magic_escape)                
-                return ret
+                #print "MATCH: %r" % self.matches[state] # dbg
+                return self.matches[state]
             except IndexError:
                 return None
         except:

IPython/core/component.py

@@ -157,7 +157,7 b' def masquerade_as(instance, cls):'
     cls.register_instance(instance)
 
 
-class ComponentNameGenerator(object):
+class __ComponentNameGenerator(object):
     """A Singleton to generate unique component names."""
 
     def __init__(self, prefix):
@@ -170,7 +170,7 b' class ComponentNameGenerator(object):'
         return "%s%s" % (self.prefix, count)
 
 
-ComponentNameGenerator = ComponentNameGenerator('ipython.component')
+ComponentNameGenerator = __ComponentNameGenerator('ipython.component')
 
 
 class MetaComponent(MetaHasTraits, MetaComponentTracker):
@@ -237,14 +237,20 b' class Component(HasTraits):'
             self.config = config
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
-            # will share config.
+            # will share config. In fact, I did find such a side effect in
+            # _config_changed below. If a config attribute value was a mutable type
+            # all instances of a component were getting the same copy, effectively
+            # making that a class attribute.
             # self.config = deepcopy(config)
         else:
             if self.parent is not None:
                 self.config = self.parent.config
                 # We used to deepcopy, but for now we are trying to just save
                 # by reference.  This *could* have side effects as all components
-                # will share config.
+                # will share config. In fact, I did find such a side effect in
+                # _config_changed below. If a config attribute value was a mutable type
+                # all instances of a component were getting the same copy, effectively
+                # making that a class attribute.
                 # self.config = deepcopy(self.parent.config)
 
         self.created = datetime.datetime.now()
@@ -296,14 +302,29 b' class Component(HasTraits):'
             if new._has_section(sname):
                 my_config = new[sname]
                 for k, v in traits.items():
+                    # Don't allow traitlets with config=True to start with
+                    # uppercase.  Otherwise, they are confused with Config
+                    # subsections.  But, developers shouldn't have uppercase
+                    # attributes anyways! (PEP 6)
+                    if k[0].upper()==k[0] and not k.startswith('_'):
+                        raise ComponentError('Component traitlets with '
+                        'config=True must start with a lowercase so they are '
+                        'not confused with Config subsections: %s.%s' % \
+                        (self.__class__.__name__, k))
                     try:
+                        # Here we grab the value from the config
+                        # If k has the naming convention of a config
+                        # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass
                     else:
                         # print "Setting %s.%s from %s.%s=%r" % \
                         #     (self.__class__.__name__,k,sname,k,config_value)
-                        setattr(self, k, config_value)
+                        # We have to do a deepcopy here if we don't deepcopy the entire
+                        # config object. If we don't, a mutable config_value will be
+                        # shared by all instances, effectively making it a class attribute.
+                        setattr(self, k, deepcopy(config_value))
 
     @property
     def children(self):

IPython/core/crashhandler.py

@@ -28,10 +28,8 b' from IPython.core import release'
 from IPython.core import ultratb
 from IPython.external.Itpl import itpl
 
-from IPython.utils.genutils import *
-
 #****************************************************************************
-class CrashHandler:
+class CrashHandler(object):
     """Customizable crash handlers for IPython-based systems.
 
     Instances of this class provide a __call__ method which can be used as a
@@ -41,15 +39,15 b' class CrashHandler:'
 
     """
 
-    def __init__(self,IP,app_name,contact_name,contact_email,
-                 bug_tracker,crash_report_fname,
-                 show_crash_traceback=True):
+    def __init__(self,app, app_name, contact_name=None, contact_email=None, 
+                 bug_tracker=None, crash_report_fname='CrashReport.txt', 
+                 show_crash_traceback=True, call_pdb=False):
         """New crash handler.
 
         Inputs:
 
-        - IP: a running IPython instance, which will be queried at crash time
-        for internal information.
+        - app: a running application instance, which will be queried at crash
+        time for internal information.
 
         - app_name: a string containing the name of your application.
 
@@ -77,13 +75,16 b' class CrashHandler:'
         """
 
         # apply args into instance
-        self.IP = IP  # IPython instance
+        self.app = app
         self.app_name = app_name
         self.contact_name = contact_name
         self.contact_email = contact_email
         self.bug_tracker = bug_tracker
         self.crash_report_fname = crash_report_fname
         self.show_crash_traceback = show_crash_traceback
+        self.section_sep = '\n\n'+'*'*75+'\n\n'
+        self.call_pdb = call_pdb
+        #self.call_pdb = True # dbg
         
         # Hardcoded defaults, which can be overridden either by subclasses or
         # at runtime for the instance.
@@ -124,7 +125,7 b' $self.bug_tracker'
         #color_scheme = 'Linux'   # dbg
         
         try:
-            rptdir = self.IP.config.IPYTHONDIR
+            rptdir = self.app.ipython_dir
         except:
             rptdir = os.getcwd()
         if not os.path.isdir(rptdir):
@@ -134,8 +135,14 b' $self.bug_tracker'
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         TBhandler = ultratb.VerboseTB(color_scheme=color_scheme,
-                                           long_header=1)
-        traceback = TBhandler.text(etype,evalue,etb,context=31)
+                                      long_header=1,
+                                      call_pdb=self.call_pdb,
+                                      )
+        if self.call_pdb:
+            TBhandler(etype,evalue,etb)
+            return
+        else:
+            traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
         if self.show_crash_traceback:
@@ -155,74 +162,66 b' $self.bug_tracker'
         # Construct report on disk
         report.write(self.make_report(traceback))
         report.close()
-        raw_input("Press enter to exit:")
+        raw_input("Hit <Enter> to quit this message (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
-
-        sec_sep = '\n\n'+'*'*75+'\n\n'
-
+        import platform
+        
+        sec_sep = self.section_sep
+        
         report = []
         rpt_add = report.append
         
         rpt_add('*'*75+'\n\n'+'IPython post-mortem report\n\n')
-        rpt_add('IPython version: %s \n\n' % release.version)
-        rpt_add('BZR revision   : %s \n\n' % release.revision)
-        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s' %
+        rpt_add('IPython version: %s \n' % release.version)
+        rpt_add('BZR revision   : %s \n' % release.revision)
+        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s\n' %
                      (os.name,sys.platform) )
-        rpt_add(sec_sep+'Current user configuration structure:\n\n')
-        rpt_add(pformat(self.IP.dict()))
-        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
+        rpt_add('               : %s\n' % platform.platform())
+        rpt_add('Python info    : %s\n' % sys.version)	     
+
         try:
-            rpt_add(sec_sep+"History of session input:")
-            for line in self.IP.user_ns['_ih']:
-                rpt_add(line)
-            rpt_add('\n*** Last line of input (may not be in above history):\n')
-            rpt_add(self.IP._last_input_line+'\n')
+            config = pformat(self.app.config)
+            rpt_add(sec_sep+'Current user configuration structure:\n\n')
+            rpt_add(config)
         except:
             pass
+        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 
+
 class IPythonCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
     
-    def __init__(self,IP):
+    def __init__(self, app, app_name='IPython'):
 
         # Set here which of the IPython authors should be listed as contact
         AUTHOR_CONTACT = 'Fernando'
         
         # Set argument defaults
-        app_name = 'IPython'
         bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
         contact_name,contact_email = release.authors[AUTHOR_CONTACT][:2]
         crash_report_fname = 'IPython_crash_report.txt'
         # Call parent constructor
-        CrashHandler.__init__(self,IP,app_name,contact_name,contact_email,
+        CrashHandler.__init__(self,app,app_name,contact_name,contact_email,
                               bug_tracker,crash_report_fname)
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
-        sec_sep = '\n\n'+'*'*75+'\n\n'
-
-        report = []
+        sec_sep = self.section_sep
+        # Start with parent report
+        report = [super(IPythonCrashHandler, self).make_report(traceback)]
+        # Add interactive-specific info we may have
         rpt_add = report.append
-        
-        rpt_add('*'*75+'\n\n'+'IPython post-mortem report\n\n')
-        rpt_add('IPython version: %s \n\n' % release.version)
-        rpt_add('BZR revision   : %s \n\n' % release.revision)
-        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s' %
-                     (os.name,sys.platform) )
-        rpt_add(sec_sep+'Current user configuration structure:\n\n')
-        # rpt_add(pformat(self.IP.dict()))
-        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
         try:
             rpt_add(sec_sep+"History of session input:")
-            for line in self.IP.user_ns['_ih']:
+            for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
-            rpt_add(self.IP._last_input_line+'\n')
+            rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 

IPython/core/debugger.py

@@ -70,6 +70,7 b' def BdbQuit_excepthook(et,ev,tb):'
 def BdbQuit_IPython_excepthook(self,et,ev,tb):
     print 'Exiting Debugger.'
 
+
 class Tracer(object):
     """Class for local debugging, similar to pdb.set_trace.
 
@@ -105,12 +106,10 b' class Tracer(object):'
         from the Python standard library for usage details.
         """
 
-        global __IPYTHON__
         try:
-            __IPYTHON__
-        except NameError:
+            ip = ipapi.get()
+        except:
             # Outside of ipython, we set our own exception hook manually
-            __IPYTHON__ = ipapi.get()
             BdbQuit_excepthook.excepthook_ori = sys.excepthook
             sys.excepthook = BdbQuit_excepthook
             def_colors = 'NoColor'
@@ -122,9 +121,8 b' class Tracer(object):'
                 pass
         else:
             # In ipython, we use its custom exception handler mechanism
-            ip = ipapi.get()
             def_colors = ip.colors
-            ip.set_custom_exc((bdb.BdbQuit,),BdbQuit_IPython_excepthook)
+            ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
@@ -138,6 +136,7 b' class Tracer(object):'
         
         self.debugger.set_trace(sys._getframe().f_back)
 
+
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the do_... commands that hook into the help system.
@@ -149,6 +148,7 b' def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):'
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
+
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
@@ -164,143 +164,98 b' def _file_lines(fname):'
         outfile.close()
         return out
 
+
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
-    if sys.version[:3] >= '2.5' or has_pydb:
-        def __init__(self,color_scheme='NoColor',completekey=None,
-                     stdin=None, stdout=None):
+    def __init__(self,color_scheme='NoColor',completekey=None,
+                 stdin=None, stdout=None):
 
-            # Parent constructor:
-            if has_pydb and completekey is None:
-                OldPdb.__init__(self,stdin=stdin,stdout=Term.cout)
-            else:
-                OldPdb.__init__(self,completekey,stdin,stdout)
-                
-            self.prompt = prompt # The default prompt is '(Pdb)'
+        # Parent constructor:
+        if has_pydb and completekey is None:
+            OldPdb.__init__(self,stdin=stdin,stdout=Term.cout)
+        else:
+            OldPdb.__init__(self,completekey,stdin,stdout)
             
-            # IPython changes...
-            self.is_pydb = has_pydb
-
-            if self.is_pydb:
-
-                # iplib.py's ipalias seems to want pdb's checkline
-                # which located in pydb.fn
-                import pydb.fns
-                self.checkline = lambda filename, lineno: \
-                                 pydb.fns.checkline(self, filename, lineno)
-
-                self.curframe = None
-                self.do_restart = self.new_do_restart
-
-                self.old_all_completions = __IPYTHON__.Completer.all_completions
-                __IPYTHON__.Completer.all_completions=self.all_completions
-
-                self.do_list = decorate_fn_with_doc(self.list_command_pydb,
-                                                    OldPdb.do_list)
-                self.do_l     = self.do_list
-                self.do_frame = decorate_fn_with_doc(self.new_do_frame,
-                                                     OldPdb.do_frame)
-
-            self.aliases = {}
-
-            # Create color table: we copy the default one from the traceback
-            # module and add a few attributes needed for debugging
-            self.color_scheme_table = exception_colors()
+        self.prompt = prompt # The default prompt is '(Pdb)'
+        
+        # IPython changes...
+        self.is_pydb = has_pydb
 
-            # shorthands 
-            C = coloransi.TermColors
-            cst = self.color_scheme_table
+        self.shell = ipapi.get()
 
-            cst['NoColor'].colors.breakpoint_enabled = C.NoColor
-            cst['NoColor'].colors.breakpoint_disabled = C.NoColor
+        if self.is_pydb:
 
-            cst['Linux'].colors.breakpoint_enabled = C.LightRed
-            cst['Linux'].colors.breakpoint_disabled = C.Red
+            # iplib.py's ipalias seems to want pdb's checkline
+            # which located in pydb.fn
+            import pydb.fns
+            self.checkline = lambda filename, lineno: \
+                             pydb.fns.checkline(self, filename, lineno)
 
-            cst['LightBG'].colors.breakpoint_enabled = C.LightRed
-            cst['LightBG'].colors.breakpoint_disabled = C.Red
+            self.curframe = None
+            self.do_restart = self.new_do_restart
 
-            self.set_colors(color_scheme)
+            self.old_all_completions = self.shell.Completer.all_completions
+            self.shell.Completer.all_completions=self.all_completions
 
-            # Add a python parser so we can syntax highlight source while
-            # debugging.
-            self.parser = PyColorize.Parser()
+            self.do_list = decorate_fn_with_doc(self.list_command_pydb,
+                                                OldPdb.do_list)
+            self.do_l     = self.do_list
+            self.do_frame = decorate_fn_with_doc(self.new_do_frame,
+                                                 OldPdb.do_frame)
 
+        self.aliases = {}
 
-    else:
-        # Ugly hack: for Python 2.3-2.4, we can't call the parent constructor,
-        # because it binds readline and breaks tab-completion.  This means we
-        # have to COPY the constructor here.
-        def __init__(self,color_scheme='NoColor'):
-            bdb.Bdb.__init__(self)
-            cmd.Cmd.__init__(self,completekey=None) # don't load readline
-            self.prompt = 'ipdb> ' # The default prompt is '(Pdb)'
-            self.aliases = {}
-
-            # These two lines are part of the py2.4 constructor, let's put them
-            # unconditionally here as they won't cause any problems in 2.3.
-            self.mainpyfile = ''
-            self._wait_for_mainpyfile = 0
-
-            # Read $HOME/.pdbrc and ./.pdbrc
-            try:
-                self.rcLines = _file_lines(os.path.join(os.environ['HOME'],
-                                                        ".pdbrc"))
-            except KeyError:
-                self.rcLines = []
-            self.rcLines.extend(_file_lines(".pdbrc"))
+        # Create color table: we copy the default one from the traceback
+        # module and add a few attributes needed for debugging
+        self.color_scheme_table = exception_colors()
 
-            # Create color table: we copy the default one from the traceback
-            # module and add a few attributes needed for debugging
-            self.color_scheme_table = exception_colors()
+        # shorthands 
+        C = coloransi.TermColors
+        cst = self.color_scheme_table
 
-            # shorthands 
-            C = coloransi.TermColors
-            cst = self.color_scheme_table
+        cst['NoColor'].colors.breakpoint_enabled = C.NoColor
+        cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
-            cst['NoColor'].colors.breakpoint_enabled = C.NoColor
-            cst['NoColor'].colors.breakpoint_disabled = C.NoColor
+        cst['Linux'].colors.breakpoint_enabled = C.LightRed
+        cst['Linux'].colors.breakpoint_disabled = C.Red
 
-            cst['Linux'].colors.breakpoint_enabled = C.LightRed
-            cst['Linux'].colors.breakpoint_disabled = C.Red
+        cst['LightBG'].colors.breakpoint_enabled = C.LightRed
+        cst['LightBG'].colors.breakpoint_disabled = C.Red
 
-            cst['LightBG'].colors.breakpoint_enabled = C.LightRed
-            cst['LightBG'].colors.breakpoint_disabled = C.Red
+        self.set_colors(color_scheme)
 
-            self.set_colors(color_scheme)
+        # Add a python parser so we can syntax highlight source while
+        # debugging.
+        self.parser = PyColorize.Parser()
 
-            # Add a python parser so we can syntax highlight source while
-            # debugging.
-            self.parser = PyColorize.Parser()
-        
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
 
     def interaction(self, frame, traceback):
-        __IPYTHON__.set_completer_frame(frame)
+        self.shell.set_completer_frame(frame)
         OldPdb.interaction(self, frame, traceback)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
 
     def new_do_quit(self, arg):
         
         if hasattr(self, 'old_all_completions'):
-            __IPYTHON__.Completer.all_completions=self.old_all_completions
+            self.shell.Completer.all_completions=self.old_all_completions
         
         
         return OldPdb.do_quit(self, arg)
@@ -314,7 +269,7 b' class Pdb(OldPdb):'
         return self.do_quit(arg)
 
     def postloop(self):
-        __IPYTHON__.set_completer_frame(None)
+        self.shell.set_completer_frame(None)
 
     def print_stack_trace(self):
         try:
@@ -331,7 +286,7 b' class Pdb(OldPdb):'
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
-        __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
@@ -500,7 +455,7 b' class Pdb(OldPdb):'
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
-        __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
@@ -509,16 +464,49 b' class Pdb(OldPdb):'
         """The debugger interface to magic_pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pdef(arg, namespaces=namespaces)
+        self.shell.magic_pdef(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """The debugger interface to magic_pdoc"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pdoc(arg, namespaces=namespaces)
+        self.shell.magic_pdoc(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """The debugger equivalant of ?obj"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pinfo("pinfo %s" % arg, namespaces=namespaces)
+        self.shell.magic_pinfo("pinfo %s" % arg, namespaces=namespaces)
+
+    def checkline(self, filename, lineno):
+        """Check whether specified line seems to be executable.
+
+        Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
+        line or EOF). Warning: testing is not comprehensive.
+        """
+        #######################################################################
+        # XXX Hack!  Use python-2.5 compatible code for this call, because with
+        # all of our changes, we've drifted from the pdb api in 2.6.  For now,
+        # changing:
+        #
+        #line = linecache.getline(filename, lineno, self.curframe.f_globals)
+        # to:
+        #
+        line = linecache.getline(filename, lineno)
+        #
+        # does the trick.  But in reality, we need to fix this by reconciling
+        # our updates with the new Pdb APIs in Python 2.6.
+        #
+        # End hack.  The rest of this method is copied verbatim from 2.6 pdb.py
+        #######################################################################
+        
+        if not line:
+            print >>self.stdout, 'End of file'
+            return 0
+        line = line.strip()
+        # Don't allow setting breakpoint at a blank line
+        if (not line or (line[0] == '#') or
+             (line[:3] == '"""') or line[:3] == "'''"):
+            print >>self.stdout, '*** Blank or comment'
+            return 0
+        return lineno

IPython/core/display_trap.py

@@ -46,11 +46,11 b' class DisplayTrap(Component):'
         # Only turn off the trap when the outermost call to __exit__ is made.
         self._nested_level = 0
 
-    @auto_attr
-    def shell(self):
-        return Component.get_instances(
-            root=self.root,
-            klass='IPython.core.iplib.InteractiveShell')[0]
+    # @auto_attr
+    # def shell(self):
+    #     return Component.get_instances(
+    #         root=self.root,
+    #         klass='IPython.core.iplib.InteractiveShell')[0]
 
     def __enter__(self):
         if self._nested_level == 0:

IPython/core/embed.py

@@ -68,7 +68,7 b' class InteractiveShellEmbed(InteractiveShell):'
     # is True by default.
     display_banner = CBool(True)
 
-    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
+    def __init__(self, parent=None, config=None, ipython_dir=None, usage=None,
                  user_ns=None, user_global_ns=None,
                  banner1=None, banner2=None, display_banner=None,
                  custom_exceptions=((),None), exit_msg=''):
@@ -76,7 +76,7 b' class InteractiveShellEmbed(InteractiveShell):'
         self.save_sys_ipcompleter()
 
         super(InteractiveShellEmbed,self).__init__(
-            parent=parent, config=config, ipythondir=ipythondir, usage=usage, 
+            parent=parent, config=config, ipython_dir=ipython_dir, usage=usage, 
             user_ns=user_ns, user_global_ns=user_global_ns,
             banner1=banner1, banner2=banner2, display_banner=display_banner,
             custom_exceptions=custom_exceptions)
@@ -233,14 +233,6 b' class InteractiveShellEmbed(InteractiveShell):'
             for var in local_varnames:
                 delvar(var,None)
 
-    def set_completer_frame(self, frame=None):
-        if frame:
-            self.Completer.namespace = frame.f_locals
-            self.Completer.global_namespace = frame.f_globals
-        else:
-            self.Completer.namespace = self.user_ns
-            self.Completer.global_namespace = self.user_global_ns
-
 
 _embedded_shell = None
 

IPython/core/history.py

@@ -14,20 +14,25 b" def magic_history(self, parameter_s = ''):"
     %history       -> print at most 40 inputs (some may be multi-line)\\
     %history n     -> print at most n inputs\\
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
-    
-    Each input's number <n> is shown, and is accessible as the
-    automatically generated variable _i<n>.  Multi-line statements are
-    printed starting at a new line for easy copy/paste.
-    
 
-    Options:
+    By default, input history is printed without line numbers so it can be
+    directly pasted into an editor.
 
-      -n: do NOT print line numbers. This is useful if you want to get a
-      printout of many lines which can be directly pasted into a text
-      editor.
+    With -n, each input's number <n> is shown, and is accessible as the
+    automatically generated variable _i<n> as well as In[<n>].  Multi-line
+    statements are printed starting at a new line for easy copy/paste.
+
+    Options:
 
+      -n: print line numbers for each input.
       This feature is only available if numbered prompts are in use.
 
+      -o: also print outputs for each input.
+
+      -p: print classic '>>>' python prompts before each input.  This is useful
+       for making documentation, and in conjunction with -o, for producing
+       doctest-ready output.
+
       -t: (default) print the 'translated' history, as IPython understands it.
       IPython filters your input and converts it all into valid Python source
       before executing it (things like magics or aliases are turned into
@@ -50,7 +55,7 b" def magic_history(self, parameter_s = ''):"
     if not self.outputcache.do_full_cache:
         print 'This feature is only available if numbered prompts are in use.'
         return
-    opts,args = self.parse_options(parameter_s,'gntsrf:',mode='list')
+    opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
 
     # Check if output to specific file was requested.
     try:
@@ -97,9 +102,12 b" def magic_history(self, parameter_s = ''):"
         warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
         print self.magic_hist.__doc__
         return
+    
     width = len(str(final))
     line_sep = ['','\n']
-    print_nums = not opts.has_key('n')
+    print_nums = 'n' in opts
+    print_outputs = 'o' in opts
+    pyprompts = 'p' in opts
     
     found = False
     if pattern is not None:
@@ -123,7 +131,19 b" def magic_history(self, parameter_s = ''):"
         if print_nums:
             print >> outfile, \
                   '%s:%s' % (str(in_num).ljust(width),line_sep[multiline]),
-        print >> outfile, inline,
+        if pyprompts:
+            print >> outfile, '>>>',
+            if multiline:
+                lines = inline.splitlines()
+                print >> outfile, '\n... '.join(lines)
+                print >> outfile, '... '
+            else:
+                print >> outfile, inline,
+        else:
+            print >> outfile, inline,
+        output = self.shell.user_ns['Out'].get(in_num)
+        if output is not None:
+            print repr(output)
 
     if close_at_end:
         outfile.close()
@@ -245,10 +265,10 b' class ShadowHist(object):'
 
 
 def init_ipython(ip):
-    import ipy_completers
-
     ip.define_magic("rep",rep_f)        
     ip.define_magic("hist",magic_hist)            
     ip.define_magic("history",magic_history)
 
-    ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
+    # XXX - ipy_completers are in quarantine, need to be updated to new apis
+    #import ipy_completers
+    #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')

IPython/core/hooks.py

@@ -137,8 +137,7 b' class CommandChainDispatcher:'
         for prio,cmd in self.chain:
             #print "prio",prio,"cmd",cmd #dbg
             try:
-                ret = cmd(*args, **kw)
-                return ret
+                return cmd(*args, **kw)
             except TryNext, exc:
                 if exc.args or exc.kwargs:
                     args = exc.args

IPython/core/ipapi.py

@@ -18,16 +18,19 b' has been made into a component, this module will be sent to deathrow.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.core.error import TryNext, UsageError
+from IPython.core.error import TryNext, UsageError, IPythonCoreError
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
+
 def get():
     """Get the most recently created InteractiveShell instance."""
     from IPython.core.iplib import InteractiveShell
     insts = InteractiveShell.get_instances()
+    if len(insts)==0:
+        return None
     most_recent = insts[0]
     for inst in insts[1:]:
         if inst.created > most_recent.created:

IPython/core/ipapp.py

@@ -1,19 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
-The main IPython application object
+The :class:`~IPython.core.application.Application` object for the command
+line :command:`ipython` program.
 
-Authors:
+Authors
+-------
 
 * Brian Granger
 * Fernando Perez
-
-Notes
------
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -22,321 +21,405 b' Notes'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import absolute_import
 
 import logging
 import os
 import sys
-import warnings
 
-from IPython.core.application import Application, IPythonArgParseConfigLoader
-from IPython.core import release
+from IPython.core import crashhandler
+from IPython.core.application import Application
 from IPython.core.iplib import InteractiveShell
 from IPython.config.loader import (
-    NoConfigDefault, 
     Config,
-    ConfigError,
-    PyFileConfigLoader
+    PyFileConfigLoader,
+#    NoConfigDefault,
 )
-
 from IPython.lib import inputhook
-
-from IPython.utils.ipstruct import Struct
 from IPython.utils.genutils import filefind, get_ipython_dir
+from . import usage
 
 #-----------------------------------------------------------------------------
-# Utilities and helpers
+# Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
-
-ipython_desc = """
-A Python shell with automatic history (input and output), dynamic object
-introspection, easier configuration, command completion, access to the system
-shell and more.
-"""
-
-def pylab_warning():
-    msg = """
-
-IPython's -pylab mode has been disabled until matplotlib supports this version
-of IPython.  This version of IPython has greatly improved GUI integration that 
-matplotlib will soon be able to take advantage of.  This will eventually
-result in greater stability and a richer API for matplotlib under IPython.
-However during this transition, you will either need to use an older version
-of IPython, or do the following to use matplotlib interactively::
-
-    import matplotlib
-    matplotlib.interactive(True)
-    matplotlib.use('wxagg')  # adjust for your backend
-    %gui -a wx               # adjust for your GUI
-    from matplotlib import pyplot as plt
-
-See the %gui magic for information on the new interface.
-"""
-    warnings.warn(msg, category=DeprecationWarning, stacklevel=1)
-
-
-#-----------------------------------------------------------------------------
-# Main classes and functions
-#-----------------------------------------------------------------------------
+default_config_file_name = u'ipython_config.py'
 
 cl_args = (
-    (('-autocall',), dict(
-        type=int, dest='InteractiveShell.autocall', default=NoConfigDefault,
-        help='Set the autocall value (0,1,2).',
+    (('--autocall',), dict(
+        type=int, dest='InteractiveShell.autocall',
+        help=
+        """Make IPython automatically call any callable object even if you
+        didn't type explicit parentheses. For example, 'str 43' becomes
+        'str(43)' automatically.  The value can be '0' to disable the feature,
+        '1' for 'smart' autocall, where it is not applied if there are no more
+        arguments on the line, and '2' for 'full' autocall, where all callable
+        objects are automatically called (even if no arguments are present).
+        The default is '1'.""",
         metavar='InteractiveShell.autocall')
     ),
-    (('-autoindent',), dict(
-        action='store_true', dest='InteractiveShell.autoindent', default=NoConfigDefault,
+    (('--autoindent',), dict(
+        action='store_true', dest='InteractiveShell.autoindent',
         help='Turn on autoindenting.')
     ),
-    (('-noautoindent',), dict(
-        action='store_false', dest='InteractiveShell.autoindent', default=NoConfigDefault,
+    (('--no-autoindent',), dict(
+        action='store_false', dest='InteractiveShell.autoindent',
         help='Turn off autoindenting.')
     ),
-    (('-automagic',), dict(
-        action='store_true', dest='InteractiveShell.automagic', default=NoConfigDefault,
-        help='Turn on the auto calling of magic commands.')
-    ),
-    (('-noautomagic',), dict(
-        action='store_false', dest='InteractiveShell.automagic', default=NoConfigDefault,
+    (('--automagic',), dict(
+        action='store_true', dest='InteractiveShell.automagic',
+        help='Turn on the auto calling of magic commands.'
+        'Type %%magic at  the  IPython  prompt  for  more information.')
+     ),
+    (('--no-automagic',), dict(
+        action='store_false', dest='InteractiveShell.automagic',
         help='Turn off the auto calling of magic commands.')
     ),
-    (('-autoedit_syntax',), dict(
-        action='store_true', dest='InteractiveShell.autoedit_syntax', default=NoConfigDefault,
+    (('--autoedit-syntax',), dict(
+        action='store_true', dest='InteractiveShell.autoedit_syntax',
         help='Turn on auto editing of files with syntax errors.')
     ),
-    (('-noautoedit_syntax',), dict(
-        action='store_false', dest='InteractiveShell.autoedit_syntax', default=NoConfigDefault,
+    (('--no-autoedit-syntax',), dict(
+        action='store_false', dest='InteractiveShell.autoedit_syntax',
         help='Turn off auto editing of files with syntax errors.')
     ),
-    (('-banner',), dict(
-        action='store_true', dest='Global.display_banner', default=NoConfigDefault,
+    (('--banner',), dict(
+        action='store_true', dest='Global.display_banner',
         help='Display a banner upon starting IPython.')
     ),
-    (('-nobanner',), dict(
-        action='store_false', dest='Global.display_banner', default=NoConfigDefault,
+    (('--no-banner',), dict(
+        action='store_false', dest='Global.display_banner',
         help="Don't display a banner upon starting IPython.")
     ),
-    (('-cache_size',), dict(
-        type=int, dest='InteractiveShell.cache_size', default=NoConfigDefault,
-        help="Set the size of the output cache.",
+    (('--cache-size',), dict(
+        type=int, dest='InteractiveShell.cache_size',
+        help=
+        """Set the size of the output cache.  The default is 1000, you can
+        change it permanently in your config file.  Setting it to 0 completely
+        disables the caching system, and the minimum value accepted is 20 (if
+        you provide a value less than 20, it is reset to 0 and a warning is
+        issued).  This limit is defined because otherwise you'll spend more
+        time re-flushing a too small cache than working.
+        """,
         metavar='InteractiveShell.cache_size')
     ),
-    (('-classic',), dict(
-        action='store_true', dest='Global.classic', default=NoConfigDefault,
+    (('--classic',), dict(
+        action='store_true', dest='Global.classic',
         help="Gives IPython a similar feel to the classic Python prompt.")
     ),
-    (('-colors',), dict(
-        type=str, dest='InteractiveShell.colors', default=NoConfigDefault,
+    (('--colors',), dict(
+        type=str, dest='InteractiveShell.colors',
         help="Set the color scheme (NoColor, Linux, and LightBG).",
         metavar='InteractiveShell.colors')
     ),
-    (('-color_info',), dict(
-        action='store_true', dest='InteractiveShell.color_info', default=NoConfigDefault,
-        help="Enable using colors for info related things.")
+    (('--color-info',), dict(
+        action='store_true', dest='InteractiveShell.color_info',
+        help=
+        """IPython can display information about objects via a set of func-
+        tions, and optionally can use colors for this, syntax highlighting
+        source code and various other elements.  However, because this
+        information is passed through a pager (like 'less') and many pagers get
+        confused with color codes, this option is off by default.  You can test
+        it and turn it on permanently in your ipython_config.py file if it
+        works for you.  Test it and turn it on permanently if it works with
+        your system.  The magic function %%color_info allows you to toggle this
+        inter- actively for testing."""
+        )
     ),
-    (('-nocolor_info',), dict(
-        action='store_false', dest='InteractiveShell.color_info', default=NoConfigDefault,
+    (('--no-color-info',), dict(
+        action='store_false', dest='InteractiveShell.color_info',
         help="Disable using colors for info related things.")
     ),
-    (('-confirm_exit',), dict(
-        action='store_true', dest='InteractiveShell.confirm_exit', default=NoConfigDefault,
-        help="Prompt the user when existing.")
-    ),
-    (('-noconfirm_exit',), dict(
-        action='store_false', dest='InteractiveShell.confirm_exit', default=NoConfigDefault,
-        help="Don't prompt the user when existing.")
-    ),
-    (('-deep_reload',), dict(
-        action='store_true', dest='InteractiveShell.deep_reload', default=NoConfigDefault,
-        help="Enable deep (recursive) reloading by default.")
+    (('--confirm-exit',), dict(
+        action='store_true', dest='InteractiveShell.confirm_exit',
+        help=
+        """Set to confirm when you try to exit IPython with an EOF (Control-D
+        in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
+        '%%Exit', you can force a direct exit without any confirmation.
+        """
+        )
     ),
-    (('-nodeep_reload',), dict(
-        action='store_false', dest='InteractiveShell.deep_reload', default=NoConfigDefault,
+    (('--no-confirm-exit',), dict(
+        action='store_false', dest='InteractiveShell.confirm_exit',
+        help="Don't prompt the user when exiting.")
+    ),
+    (('--deep-reload',), dict(
+        action='store_true', dest='InteractiveShell.deep_reload',
+        help=
+        """Enable deep (recursive) reloading by default. IPython can use the
+        deep_reload module which reloads changes in modules recursively (it
+        replaces the reload() function, so you don't need to change anything to
+        use it). deep_reload() forces a full reload of modules whose code may
+        have changed, which the default reload() function does not.  When
+        deep_reload is off, IPython will use the normal reload(), but
+        deep_reload will still be available as dreload(). This fea- ture is off
+        by default [which means that you have both normal reload() and
+        dreload()].""")
+    ),
+    (('--no-deep-reload',), dict(
+        action='store_false', dest='InteractiveShell.deep_reload',
         help="Disable deep (recursive) reloading by default.")
     ),
-    (('-editor',), dict(
-        type=str, dest='InteractiveShell.editor', default=NoConfigDefault,
+    (('--editor',), dict(
+        type=str, dest='InteractiveShell.editor',
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
         metavar='InteractiveShell.editor')
     ),
-    (('-log','-l'), dict(
-        action='store_true', dest='InteractiveShell.logstart', default=NoConfigDefault,
-        help="Start logging to the default file (./ipython_log.py).")
+    (('--log','-l'), dict(
+        action='store_true', dest='InteractiveShell.logstart',
+        help="Start logging to the default log file (./ipython_log.py).")
     ),
-    (('-logfile','-lf'), dict(
-        type=str, dest='InteractiveShell.logfile', default=NoConfigDefault,
-        help="Start logging to logfile.",
+    (('--logfile','-lf'), dict(
+        type=unicode, dest='InteractiveShell.logfile',
+        help="Start logging to logfile with this name.",
         metavar='InteractiveShell.logfile')
     ),
-    (('-logappend','-la'), dict(
-        type=str, dest='InteractiveShell.logappend', default=NoConfigDefault,
-        help="Start logging to logappend in append mode.",
+    (('--log-append','-la'), dict(
+        type=unicode, dest='InteractiveShell.logappend',
+        help="Start logging to the given file in append mode.",
         metavar='InteractiveShell.logfile')
     ),
-    (('-pdb',), dict(
-        action='store_true', dest='InteractiveShell.pdb', default=NoConfigDefault,
+    (('--pdb',), dict(
+        action='store_true', dest='InteractiveShell.pdb',
         help="Enable auto calling the pdb debugger after every exception.")
     ),
-    (('-nopdb',), dict(
-        action='store_false', dest='InteractiveShell.pdb', default=NoConfigDefault,
+    (('--no-pdb',), dict(
+        action='store_false', dest='InteractiveShell.pdb',
         help="Disable auto calling the pdb debugger after every exception.")
     ),
-    (('-pprint',), dict(
-        action='store_true', dest='InteractiveShell.pprint', default=NoConfigDefault,
+    (('--pprint',), dict(
+        action='store_true', dest='InteractiveShell.pprint',
         help="Enable auto pretty printing of results.")
     ),
-    (('-nopprint',), dict(
-        action='store_false', dest='InteractiveShell.pprint', default=NoConfigDefault,
+    (('--no-pprint',), dict(
+        action='store_false', dest='InteractiveShell.pprint',
         help="Disable auto auto pretty printing of results.")
     ),
-    (('-prompt_in1','-pi1'), dict(
-        type=str, dest='InteractiveShell.prompt_in1', default=NoConfigDefault,
-        help="Set the main input prompt ('In [\#]: ')",
+    (('--prompt-in1','-pi1'), dict(
+        type=str, dest='InteractiveShell.prompt_in1',
+        help=
+        """Set the main input prompt ('In [\#]: ').  Note that if you are using
+        numbered prompts, the number is represented with a '\#' in the string.
+        Don't forget to quote strings with spaces embedded in them. Most
+        bash-like escapes can be used to customize IPython's prompts, as well
+        as a few additional ones which are IPython-spe- cific.  All valid
+        prompt escapes are described in detail in the Customization section of
+        the IPython manual.""",
         metavar='InteractiveShell.prompt_in1')
     ),
-    (('-prompt_in2','-pi2'), dict(
-        type=str, dest='InteractiveShell.prompt_in2', default=NoConfigDefault,
-        help="Set the secondary input prompt (' .\D.: ')",
+    (('--prompt-in2','-pi2'), dict(
+        type=str, dest='InteractiveShell.prompt_in2',
+        help=
+        """Set the secondary input prompt (' .\D.: ').  Similar to the previous
+        option, but used for the continuation prompts. The special sequence
+        '\D' is similar to '\#', but with all digits replaced by dots (so you
+        can have your continuation prompt aligned with your input prompt).
+        Default: ' .\D.: ' (note three spaces at the start for alignment with
+        'In [\#]')""",
         metavar='InteractiveShell.prompt_in2')
     ),
-    (('-prompt_out','-po'), dict(
-        type=str, dest='InteractiveShell.prompt_out', default=NoConfigDefault,
+    (('--prompt-out','-po'), dict(
+        type=str, dest='InteractiveShell.prompt_out',
         help="Set the output prompt ('Out[\#]:')",
         metavar='InteractiveShell.prompt_out')
     ),
-    (('-quick',), dict(
-        action='store_true', dest='Global.quick', default=NoConfigDefault,
+    (('--quick',), dict(
+        action='store_true', dest='Global.quick',
         help="Enable quick startup with no config files.")
     ),
-    (('-readline',), dict(
-        action='store_true', dest='InteractiveShell.readline_use', default=NoConfigDefault,
+    (('--readline',), dict(
+        action='store_true', dest='InteractiveShell.readline_use',
         help="Enable readline for command line usage.")
     ),
-    (('-noreadline',), dict(
-        action='store_false', dest='InteractiveShell.readline_use', default=NoConfigDefault,
+    (('--no-readline',), dict(
+        action='store_false', dest='InteractiveShell.readline_use',
         help="Disable readline for command line usage.")
     ),
-    (('-screen_length','-sl'), dict(
-        type=int, dest='InteractiveShell.screen_length', default=NoConfigDefault,
-        help='Number of lines on screen, used to control printing of long strings.',
+    (('--screen-length','-sl'), dict(
+        type=int, dest='InteractiveShell.screen_length',
+        help=
+        """Number of lines of your screen, used to control printing of very
+        long strings.  Strings longer than this number of lines will be sent
+        through a pager instead of directly printed.  The default value for
+        this is 0, which means IPython will auto-detect your screen size every
+        time it needs to print certain potentially long strings (this doesn't
+        change the behavior of the 'print' keyword, it's only triggered
+        internally). If for some reason this isn't working well (it needs
+        curses support), specify it yourself. Otherwise don't change the
+        default.""",
         metavar='InteractiveShell.screen_length')
     ),
-    (('-separate_in','-si'), dict(
-        type=str, dest='InteractiveShell.separate_in', default=NoConfigDefault,
-        help="Separator before input prompts.  Default '\n'.",
+    (('--separate-in','-si'), dict(
+        type=str, dest='InteractiveShell.separate_in',
+        help="Separator before input prompts.  Default '\\n'.",
         metavar='InteractiveShell.separate_in')
     ),
-    (('-separate_out','-so'), dict(
-        type=str, dest='InteractiveShell.separate_out', default=NoConfigDefault,
+    (('--separate-out','-so'), dict(
+        type=str, dest='InteractiveShell.separate_out',
         help="Separator before output prompts.  Default 0 (nothing).",
         metavar='InteractiveShell.separate_out')
     ),
-    (('-separate_out2','-so2'), dict(
-        type=str, dest='InteractiveShell.separate_out2', default=NoConfigDefault,
+    (('--separate-out2','-so2'), dict(
+        type=str, dest='InteractiveShell.separate_out2',
         help="Separator after output prompts.  Default 0 (nonight).",
         metavar='InteractiveShell.separate_out2')
     ),
-    (('-nosep',), dict(
-        action='store_true', dest='Global.nosep', default=NoConfigDefault,
+    (('-no-sep',), dict(
+        action='store_true', dest='Global.nosep',
         help="Eliminate all spacing between prompts.")
     ),
-    (('-term_title',), dict(
-        action='store_true', dest='InteractiveShell.term_title', default=NoConfigDefault,
+    (('--term-title',), dict(
+        action='store_true', dest='InteractiveShell.term_title',
         help="Enable auto setting the terminal title.")
     ),
-    (('-noterm_title',), dict(
-        action='store_false', dest='InteractiveShell.term_title', default=NoConfigDefault,
+    (('--no-term-title',), dict(
+        action='store_false', dest='InteractiveShell.term_title',
         help="Disable auto setting the terminal title.")
     ),
-    (('-xmode',), dict(
-        type=str, dest='InteractiveShell.xmode', default=NoConfigDefault,
-        help="Exception mode ('Plain','Context','Verbose')",
+    (('--xmode',), dict(
+        type=str, dest='InteractiveShell.xmode',
+        help=
+        """Exception reporting mode ('Plain','Context','Verbose').  Plain:
+        similar to python's normal traceback printing.  Context: prints 5 lines
+        of context source code around each line in the traceback.  Verbose:
+        similar to Context, but additionally prints the variables currently
+        visible where the exception happened (shortening their strings if too
+        long).  This can potentially be very slow, if you happen to have a huge
+        data structure whose string representation is complex to compute.
+        Your computer may appear to freeze for a while with cpu usage at 100%%.
+        If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
+        it more than once).
+        """,
         metavar='InteractiveShell.xmode')
     ),
-    (('-ext',), dict(
-        type=str, dest='Global.extra_extension', default=NoConfigDefault,
+    (('--ext',), dict(
+        type=str, dest='Global.extra_extension',
         help="The dotted module name of an IPython extension to load.",
         metavar='Global.extra_extension')
     ),
     (('-c',), dict(
-        type=str, dest='Global.code_to_run', default=NoConfigDefault,
+        type=str, dest='Global.code_to_run',
         help="Execute the given command string.",
         metavar='Global.code_to_run')
     ),
     (('-i',), dict(
-        action='store_true', dest='Global.force_interact', default=NoConfigDefault,
-        help="If running code from the command line, become interactive afterwards.")
-    ),
-    (('-wthread',), dict(
-        action='store_true', dest='Global.wthread', default=NoConfigDefault,
-        help="Enable wxPython event loop integration.")
-    ),
-    (('-q4thread','-qthread'), dict(
-        action='store_true', dest='Global.q4thread', default=NoConfigDefault,
-        help="Enable Qt4 event loop integration. Qt3 is no longer supported.")
-    ),
-    (('-gthread',), dict(
-        action='store_true', dest='Global.gthread', default=NoConfigDefault,
-        help="Enable GTK event loop integration.")
-    ),
-    # # These are only here to get the proper deprecation warnings
-    (('-pylab',), dict(
-        action='store_true', dest='Global.pylab', default=NoConfigDefault,
-        help="Disabled.  Pylab has been disabled until matplotlib supports this version of IPython.")
-    )
-)
-
-
-class IPythonAppCLConfigLoader(IPythonArgParseConfigLoader):
+        action='store_true', dest='Global.force_interact',
+        help=
+        "If running code from the command line, become interactive afterwards."
+        )
+    ),
 
-    arguments = cl_args
+    # Options to start with GUI control enabled from the beginning
+    (('--gui',), dict(
+        type=str, dest='Global.gui',
+        help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
+        metavar='gui-mode')
+    ),
 
+    (('--pylab','-pylab'), dict(
+        type=str, dest='Global.pylab',
+        nargs='?', const='auto', metavar='gui-mode',
+        help="Pre-load matplotlib and numpy for interactive use. "+
+        "If no value is given, the gui backend is matplotlib's, else use "+
+        "one of:  ['tk', 'qt', 'wx', 'gtk'].")
+    ),
+    
+    # Legacy GUI options.  Leave them in for backwards compatibility, but the
+    # 'thread' names are really a misnomer now.
+    (('--wthread','-wthread'), dict(
+        action='store_true', dest='Global.wthread',
+        help="Enable wxPython event loop integration "+
+             "(DEPRECATED, use --gui wx)")
+    ),
+    (('--q4thread','--qthread','-q4thread','-qthread'), dict(
+        action='store_true', dest='Global.q4thread',
+        help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+
+             "(DEPRECATED, use --gui qt)")
+    ),
+    (('--gthread','-gthread'), dict(
+        action='store_true', dest='Global.gthread',
+        help="Enable GTK event loop integration. "+
+             "(DEPRECATED, use --gui gtk)")
+    ),
+)
 
-_default_config_file_name = 'ipython_config.py'
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
 
 class IPythonApp(Application):
-    name = 'ipython'
-    config_file_name = _default_config_file_name
+    name = u'ipython'
+    #: argparse formats better the 'usage' than the 'description' field
+    description = None
+    #: usage message printed by argparse. If None, auto-generate
+    usage = usage.cl_usage
+
+    config_file_name = default_config_file_name
+
+    cl_arguments = Application.cl_arguments + cl_args
+
+    # Private and configuration attributes
+    _CrashHandler = crashhandler.IPythonCrashHandler
+    
+    def __init__(self, argv=None,
+                 constructor_config=None, override_config=None,
+                 **shell_params):
+        """Create a new IPythonApp.
+
+        See the parent class for details on how configuration is handled.
+
+        Parameters
+        ----------
+        argv : optional, list
+          If given, used as the command-line argv environment to read arguments
+          from.
+
+        constructor_config : optional, Config
+          If given, additional config that is merged last, after internal
+          defaults, command-line and file-based configs.
+
+        override_config : optional, Config
+          If given, config that overrides all others unconditionally (except
+          for internal defaults, which ensure that all parameters exist).
+
+        shell_params : optional, dict
+          All other keywords are passed to the :class:`iplib.InteractiveShell`
+          constructor. 
+        """
+        super(IPythonApp, self).__init__(argv, constructor_config,
+                                         override_config)
+        self.shell_params = shell_params
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
-        self.default_config.Global.display_banner = True
+        # Eliminate multiple lookups
+        Global = self.default_config.Global
+
+        # Set all default values
+        Global.display_banner = True
         
         # If the -c flag is given or a file is given to run at the cmd line
         # like "ipython foo.py", normally we exit without starting the main
         # loop.  The force_interact config variable allows a user to override
         # this and interact.  It is also set by the -i cmd line flag, just
         # like Python.
-        self.default_config.Global.force_interact = False
+        Global.force_interact = False
 
         # By default always interact by starting the IPython mainloop.
-        self.default_config.Global.interact = True
-
-        # Let the parent class set the default, but each time log_level
-        # changes from config, we need to update self.log_level as that is
-        # what updates the actual log level in self.log.
-        self.default_config.Global.log_level = self.log_level
+        Global.interact = True
 
         # No GUI integration by default
-        self.default_config.Global.wthread = False
-        self.default_config.Global.q4thread = False
-        self.default_config.Global.gthread = False
-
-    def create_command_line_config(self):
-        """Create and return a command line config loader."""
-        return IPythonAppCLConfigLoader(
-            description=ipython_desc,
-            version=release.version)
-
-    def post_load_command_line_config(self):
-        """Do actions after loading cl config."""
-        clc = self.command_line_config
-
-        # Display the deprecation warnings about threaded shells
-        if hasattr(clc.Global, 'pylab'):
-            pylab_warning()
-            del clc.Global['pylab']
+        Global.gui = False
+        # Pylab off by default
+        Global.pylab = False
+
+        # Deprecated versions of gui support that used threading, we support
+        # them just for bacwards compatibility as an alternate spelling for
+        # '--gui X'
+        Global.qthread = False
+        Global.q4thread = False
+        Global.wthread = False
+        Global.gthread = False
 
     def load_file_config(self):
         if hasattr(self.command_line_config.Global, 'quick'):
@@ -379,8 +462,7 b' class IPythonApp(Application):'
         # unless the -i flag (Global.force_interact) is true.
         code_to_run = config.Global.get('code_to_run','')
         file_to_run = False
-        if len(self.extra_args)>=1:
-            if self.extra_args[0]:
+        if self.extra_args and self.extra_args[0]:
                 file_to_run = True
         if file_to_run or code_to_run:
             if not config.Global.force_interact:
@@ -392,10 +474,8 b' class IPythonApp(Application):'
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance
-        self.shell = InteractiveShell(
-            parent=None,
-            config=self.master_config
-        )
+        self.shell = InteractiveShell(None, self.master_config,
+                                      **self.shell_params )
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
@@ -414,29 +494,52 b' class IPythonApp(Application):'
         if self.log_level <= logging.INFO: print
 
         # Now a variety of things that happen after the banner is printed.
-        self._enable_gui()
+        self._enable_gui_pylab()
         self._load_extensions()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
+        self._configure_xmode()
+
+    def _enable_gui_pylab(self):
+        """Enable GUI event loop integration, taking pylab into account."""
+        Global = self.master_config.Global
+
+        # Select which gui to use
+        if Global.gui:
+            gui = Global.gui
+        # The following are deprecated, but there's likely to be a lot of use
+        # of this form out there, so we might as well support it for now.  But
+        # the --gui option above takes precedence.
+        elif Global.wthread:
+            gui = inputhook.GUI_WX
+        elif Global.qthread:
+            gui = inputhook.GUI_QT
+        elif Global.gthread:
+            gui = inputhook.GUI_GTK
+        else:
+            gui = None
+
+        # Using --pylab will also require gui activation, though which toolkit
+        # to use may be chosen automatically based on mpl configuration.
+        if Global.pylab:
+            activate = self.shell.enable_pylab
+            if Global.pylab == 'auto':
+                gui = None
+            else:
+                gui = Global.pylab
+        else:
+            # Enable only GUI integration, no pylab
+            activate = inputhook.enable_gui
 
-    def _enable_gui(self):
-        """Enable GUI event loop integration."""
-        config = self.master_config
-        try:
-            # Enable GUI integration
-            if config.Global.wthread:
-                self.log.info("Enabling wx GUI event loop integration")
-                inputhook.enable_wx(app=True)
-            elif config.Global.q4thread:
-                self.log.info("Enabling Qt4 GUI event loop integration")
-                inputhook.enable_qt4(app=True)
-            elif config.Global.gthread:
-                self.log.info("Enabling GTK GUI event loop integration")
-                inputhook.enable_gtk(app=True)
-        except:
-            self.log.warn("Error in enabling GUI event loop integration:")
-            self.shell.showtraceback()
+        if gui or Global.pylab:
+            try:
+                self.log.info("Enabling GUI event loop integration, "
+                              "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
+                activate(gui)
+            except:
+                self.log.warn("Error in enabling GUI event loop integration:")
+                self.shell.showtraceback()
 
     def _load_extensions(self):
         """Load all IPython extensions in Global.extensions.
@@ -477,9 +580,9 b' class IPythonApp(Application):'
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
-        full_filename = filefind(fname, ['.', self.ipythondir])
+        full_filename = filefind(fname, [u'.', self.ipython_dir])
         if os.path.isfile(full_filename):
-            if full_filename.endswith('.py'):
+            if full_filename.endswith(u'.py'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile(full_filename, self.shell.user_ns)
             elif full_filename.endswith('.ipy'):
@@ -521,26 +624,32 b' class IPythonApp(Application):'
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
+    def _configure_xmode(self):
+        # XXX - shouldn't this be read from the config?  I'm still a little
+        # lost with all the details of handling the new config guys...
+        self.shell.InteractiveTB.set_mode(mode=self.shell.xmode)
+        
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
+        else:
+            self.log.debug("IPython not interactive, start_app is no-op...")
 
 
-def load_default_config(ipythondir=None):
-    """Load the default config file from the default ipythondir.
+def load_default_config(ipython_dir=None):
+    """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
-    if ipythondir is None:
-        ipythondir = get_ipython_dir()
-    cl = PyFileConfigLoader(_default_config_file_name, ipythondir)
+    if ipython_dir is None:
+        ipython_dir = get_ipython_dir()
+    cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
     config = cl.load_config()
     return config
 
 
 def launch_new_instance():
-    """Create a run a full blown IPython instance"""
+    """Create and run a full blown IPython instance"""
     app = IPythonApp()
     app.start()
-

IPython/core/iplib.py

@@ -17,6 +17,7 b' Main IPython Component'
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
+from __future__ import absolute_import
 
 import __builtin__
 import StringIO
@@ -31,34 +32,37 b' import sys'
 import tempfile
 from contextlib import nested
 
-from IPython.core import ultratb
 from IPython.core import debugger, oinspect
-from IPython.core import shadowns
 from IPython.core import history as ipcorehist
 from IPython.core import prefilter
+from IPython.core import shadowns
+from IPython.core import ultratb
 from IPython.core.alias import AliasManager
 from IPython.core.builtin_trap import BuiltinTrap
+from IPython.core.component import Component
 from IPython.core.display_trap import DisplayTrap
+from IPython.core.error import TryNext, UsageError
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
-from IPython.core.prompts import CachedOutput
 from IPython.core.prefilter import PrefilterManager
-from IPython.core.component import Component
+from IPython.core.prompts import CachedOutput
+from IPython.core.pylabtools import pylab_activate
 from IPython.core.usage import interactive_usage, default_banner
-from IPython.core.error import TryNext, UsageError
-
-from IPython.utils import pickleshare
 from IPython.external.Itpl import ItplNS
+from IPython.lib.inputhook import enable_gui
 from IPython.lib.backgroundjobs import BackgroundJobManager
-from IPython.utils.ipstruct import Struct
 from IPython.utils import PyColorize
-from IPython.utils.genutils import *
+from IPython.utils import pickleshare
 from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.ipstruct import Struct
 from IPython.utils.platutils import toggle_set_term_title, set_term_title
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 
+# XXX - need to clean up this import * line
+from IPython.utils.genutils import *
+
 # from IPython.utils import growl
 # growl.start("IPython")
 
@@ -70,7 +74,6 b' from IPython.utils.traitlets import ('
 # Globals
 #-----------------------------------------------------------------------------
 
-
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
@@ -78,12 +81,10 b' raw_input_original = raw_input'
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
-
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
-
 ini_spaces_re = re.compile(r'^(\s+)')
 
 
@@ -113,6 +114,8 b' def softspace(file, newvalue):'
     return oldvalue
 
 
+def no_op(*a, **kw): pass
+
 class SpaceInInput(exceptions.Exception): pass
 
 class Bunch: pass
@@ -162,6 +165,15 b' def get_default_editor():'
     return ed
 
 
+def get_default_colors():
+    if sys.platform=='darwin':
+        return "LightBG"
+    elif os.name=='nt':
+        return 'Linux'
+    else:
+        return 'Linux'
+
+
 class SeparateStr(Str):
     """A Str subclass to validate separate_in, separate_out, etc.
 
@@ -174,6 +186,57 b' class SeparateStr(Str):'
         return super(SeparateStr, self).validate(obj, value)
 
 
+def make_user_namespaces(user_ns=None, user_global_ns=None):
+    """Return a valid local and global user interactive namespaces.
+
+    This builds a dict with the minimal information needed to operate as a
+    valid IPython user namespace, which you can pass to the various
+    embedding classes in ipython. The default implementation returns the
+    same dict for both the locals and the globals to allow functions to
+    refer to variables in the namespace. Customized implementations can
+    return different dicts. The locals dictionary can actually be anything
+    following the basic mapping protocol of a dict, but the globals dict
+    must be a true dict, not even a subclass. It is recommended that any
+    custom object for the locals namespace synchronize with the globals
+    dict somehow.
+
+    Raises TypeError if the provided globals namespace is not a true dict.
+
+    Parameters
+    ----------
+    user_ns : dict-like, optional
+        The current user namespace. The items in this namespace should
+        be included in the output. If None, an appropriate blank
+        namespace should be created.
+    user_global_ns : dict, optional
+        The current user global namespace. The items in this namespace
+        should be included in the output. If None, an appropriate
+        blank namespace should be created.
+
+    Returns
+    -------
+        A pair of dictionary-like object to be used as the local namespace
+        of the interpreter and a dict to be used as the global namespace.
+    """
+
+    if user_ns is None:
+        # Set __name__ to __main__ to better match the behavior of the
+        # normal interpreter.
+        user_ns = {'__name__'     :'__main__',
+                   '__builtins__' : __builtin__,
+                  }
+    else:
+        user_ns.setdefault('__name__','__main__')
+        user_ns.setdefault('__builtins__',__builtin__)
+
+    if user_global_ns is None:
+        user_global_ns = user_ns
+    if type(user_global_ns) is not dict:
+        raise TypeError("user_global_ns must be a true dict; got %r"
+            % type(user_global_ns))
+
+    return user_ns, user_global_ns
+
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
@@ -182,7 +245,7 b' class SeparateStr(Str):'
 class InteractiveShell(Component, Magic):
     """An enhanced, interactive shell for Python."""
 
-    autocall = Enum((0,1,2), config=True)
+    autocall = Enum((0,1,2), default_value=1, config=True)
     autoedit_syntax = CBool(False, config=True)
     autoindent = CBool(True, config=True)
     automagic = CBool(True, config=True)
@@ -192,7 +255,7 b' class InteractiveShell(Component, Magic):'
     cache_size = Int(1000, config=True)
     color_info = CBool(True, config=True)
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'), 
-                             default_value='LightBG', config=True)
+                             default_value=get_default_colors(), config=True)
     confirm_exit = CBool(True, config=True)
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True)
@@ -206,7 +269,7 b' class InteractiveShell(Component, Magic):'
     embedded_active = CBool(False)
     editor = Str(get_default_editor(), config=True)
     filename = Str("<ipython console>")
-    ipythondir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
+    ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
     logstart = CBool(False, config=True)
     logfile = Str('', config=True)
     logappend = Str('', config=True)
@@ -264,7 +327,7 b' class InteractiveShell(Component, Magic):'
     # Subclasses with thread support should override this as needed.
     isthreaded = False
 
-    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
+    def __init__(self, parent=None, config=None, ipython_dir=None, usage=None,
                  user_ns=None, user_global_ns=None,
                  banner1=None, banner2=None, display_banner=None,
                  custom_exceptions=((),None)):
@@ -274,7 +337,7 b' class InteractiveShell(Component, Magic):'
         super(InteractiveShell, self).__init__(parent, config=config)
 
         # These are relatively independent and stateless
-        self.init_ipythondir(ipythondir)
+        self.init_ipython_dir(ipython_dir)
         self.init_instance_attrs()
         self.init_term_title()
         self.init_usage(usage)
@@ -320,6 +383,7 b' class InteractiveShell(Component, Magic):'
         self.hooks.late_startup_hook()
 
     def get_ipython(self):
+        """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
@@ -332,7 +396,7 b' class InteractiveShell(Component, Magic):'
     def _banner2_changed(self):
         self.compute_banner()
 
-    def _ipythondir_changed(self, name, new):
+    def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0777)
         if not os.path.isdir(self.ipython_extension_dir):
@@ -340,7 +404,7 b' class InteractiveShell(Component, Magic):'
 
     @property
     def ipython_extension_dir(self):
-        return os.path.join(self.ipythondir, 'extensions')
+        return os.path.join(self.ipython_dir, 'extensions')
 
     @property
     def usable_screen_length(self):
@@ -372,19 +436,19 b' class InteractiveShell(Component, Magic):'
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
-    def init_ipythondir(self, ipythondir):
-        if ipythondir is not None:
-            self.ipythondir = ipythondir
-            self.config.Global.ipythondir = self.ipythondir
+    def init_ipython_dir(self, ipython_dir):
+        if ipython_dir is not None:
+            self.ipython_dir = ipython_dir
+            self.config.Global.ipython_dir = self.ipython_dir
             return
 
-        if hasattr(self.config.Global, 'ipythondir'):
-            self.ipythondir = self.config.Global.ipythondir
+        if hasattr(self.config.Global, 'ipython_dir'):
+            self.ipython_dir = self.config.Global.ipython_dir
         else:
-            self.ipythondir = get_ipython_dir()
+            self.ipython_dir = get_ipython_dir()
 
         # All children can just read this
-        self.config.Global.ipythondir = self.ipythondir
+        self.config.Global.ipython_dir = self.ipython_dir
 
     def init_instance_attrs(self):
         self.jobs = BackgroundJobManager()
@@ -805,8 +869,7 b' class InteractiveShell(Component, Magic):'
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
-        user_ns, user_global_ns = self.make_user_namespaces(user_ns,
-            user_global_ns)
+        user_ns, user_global_ns = make_user_namespaces(user_ns, user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
@@ -816,7 +879,7 b' class InteractiveShell(Component, Magic):'
         # An auxiliary namespace that checks what parts of the user_ns were
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
-        # doesn't need to be seaparately tracked in the ns_table
+        # doesn't need to be separately tracked in the ns_table.
         self.user_config_ns = {}
 
         # A namespace to keep track of internal data structures to prevent
@@ -891,55 +954,6 b' class InteractiveShell(Component, Magic):'
         else:
             sys.modules[main_name] = FakeModule(self.user_ns)
 
-    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
-        """Return a valid local and global user interactive namespaces.
-
-        This builds a dict with the minimal information needed to operate as a
-        valid IPython user namespace, which you can pass to the various
-        embedding classes in ipython. The default implementation returns the
-        same dict for both the locals and the globals to allow functions to
-        refer to variables in the namespace. Customized implementations can
-        return different dicts. The locals dictionary can actually be anything
-        following the basic mapping protocol of a dict, but the globals dict
-        must be a true dict, not even a subclass. It is recommended that any
-        custom object for the locals namespace synchronize with the globals
-        dict somehow.
-
-        Raises TypeError if the provided globals namespace is not a true dict.
-
-        :Parameters:
-            user_ns : dict-like, optional
-                The current user namespace. The items in this namespace should
-                be included in the output. If None, an appropriate blank
-                namespace should be created.
-            user_global_ns : dict, optional
-                The current user global namespace. The items in this namespace
-                should be included in the output. If None, an appropriate
-                blank namespace should be created.
-
-        :Returns:
-            A tuple pair of dictionary-like object to be used as the local namespace
-            of the interpreter and a dict to be used as the global namespace.
-        """
-
-        if user_ns is None:
-            # Set __name__ to __main__ to better match the behavior of the
-            # normal interpreter.
-            user_ns = {'__name__'     :'__main__',
-                       '__builtins__' : __builtin__,
-                      }
-        else:
-            user_ns.setdefault('__name__','__main__')
-            user_ns.setdefault('__builtins__',__builtin__)
-
-        if user_global_ns is None:
-            user_global_ns = user_ns
-        if type(user_global_ns) is not dict:
-            raise TypeError("user_global_ns must be a true dict; got %r"
-                % type(user_global_ns))
-
-        return user_ns, user_global_ns
-
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
@@ -952,26 +966,43 b' class InteractiveShell(Component, Magic):'
         method.  If they were not empty before, data will simply be added to
         therm.
         """
-        # Store myself as the public api!!!
-        self.user_ns['get_ipython'] = self.get_ipython
+        # This function works in two parts: first we put a few things in
+        # user_ns, and we sync that contents into user_config_ns so that these
+        # initial variables aren't shown by %who.  After the sync, we add the
+        # rest of what we *do* want the user to see with %who even on a new
+        # session.
+        ns = {}
+        
+        # Put 'help' in the user namespace
+        try:
+            from site import _Helper
+            ns['help'] = _Helper()
+        except ImportError:
+            warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
-        self.user_ns['_ih'] = self.input_hist
-        self.user_ns['_oh'] = self.output_hist
-        self.user_ns['_dh'] = self.dir_hist
+        ns['_ih'] = self.input_hist
+        ns['_oh'] = self.output_hist
+        ns['_dh'] = self.dir_hist
+
+        ns['_sh'] = shadowns
+
+        # Sync what we've added so far to user_config_ns so these aren't seen
+        # by %who
+        self.user_config_ns.update(ns)
+
+        # Now, continue adding more contents
 
         # user aliases to input and output histories
-        self.user_ns['In']  = self.input_hist
-        self.user_ns['Out'] = self.output_hist
+        ns['In']  = self.input_hist
+        ns['Out'] = self.output_hist
 
-        self.user_ns['_sh'] = shadowns
+        # Store myself as the public api!!!
+        ns['get_ipython'] = self.get_ipython
+        
+        # And update the real user's namespace
+        self.user_ns.update(ns)
 
-        # Put 'help' in the user namespace
-        try:
-            from site import _Helper
-            self.user_ns['help'] = _Helper()
-        except ImportError:
-            warn('help() not available - check site.py')
 
     def reset(self):
         """Clear all internal namespaces.
@@ -1070,7 +1101,7 b' class InteractiveShell(Component, Magic):'
             histfname = 'history-%s' % self.profile
         else:
             histfname = 'history'
-        self.histfile = os.path.join(self.ipythondir, histfname)
+        self.histfile = os.path.join(self.ipython_dir, histfname)
 
         # Fill the history zero entry, user counter starts at 1
         self.input_hist.append('\n')
@@ -1078,21 +1109,18 b' class InteractiveShell(Component, Magic):'
 
     def init_shadow_hist(self):
         try:
-            self.db = pickleshare.PickleShareDB(self.ipythondir + "/db")
+            self.db = pickleshare.PickleShareDB(self.ipython_dir + "/db")
         except exceptions.UnicodeDecodeError:
-            print "Your ipythondir can't be decoded to unicode!"
+            print "Your ipython_dir can't be decoded to unicode!"
             print "Please set HOME environment variable to something that"
             print r"only has ASCII characters, e.g. c:\home"
-            print "Now it is", self.ipythondir
+            print "Now it is", self.ipython_dir
             sys.exit()
         self.shadowhist = ipcorehist.ShadowHist(self.db)
 
     def savehist(self):
         """Save input history to a file (via readline library)."""
 
-        if not self.has_readline:
-            return
-        
         try:
             self.readline.write_history_file(self.histfile)
         except:
@@ -1102,12 +1130,11 b' class InteractiveShell(Component, Magic):'
     def reloadhist(self):
         """Reload the input history from disk file."""
 
-        if self.has_readline:
-            try:
-                self.readline.clear_history()
-                self.readline.read_history_file(self.shell.histfile)
-            except AttributeError:
-                pass
+        try:
+            self.readline.clear_history()
+            self.readline.read_history_file(self.shell.histfile)
+        except AttributeError:
+            pass
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
@@ -1141,37 +1168,14 b' class InteractiveShell(Component, Magic):'
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
-        # IPython itself shouldn't crash. This will produce a detailed
-        # post-mortem if it does.  But we only install the crash handler for
-        # non-threaded shells, the threaded ones use a normal verbose reporter
-        # and lose the crash handler.  This is because exceptions in the main
-        # thread (such as in GUI code) propagate directly to sys.excepthook,
-        # and there's no point in printing crash dumps for every user exception.
-        if self.isthreaded:
-            ipCrashHandler = ultratb.FormattedTB()
-        else:
-            from IPython.core import crashhandler
-            ipCrashHandler = crashhandler.IPythonCrashHandler(self)
-        self.set_crash_handler(ipCrashHandler)
+        # The instance will store a pointer to the system-wide exception hook,
+        # so that runtime code (such as magics) can access it.  This is because
+        # during the read-eval loop, it may get temporarily overwritten.
+        self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
-    def set_crash_handler(self, crashHandler):
-        """Set the IPython crash handler.
-
-        This must be a callable with a signature suitable for use as
-        sys.excepthook."""
-
-        # Install the given crash handler as the Python exception hook
-        sys.excepthook = crashHandler
-        
-        # The instance will store a pointer to this, so that runtime code
-        # (such as magics) can access it.  This is because during the
-        # read-eval loop, it gets temporarily overwritten (to deal with GUI
-        # frameworks).
-        self.sys_excepthook = sys.excepthook
-
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
@@ -1239,7 +1243,8 b' class InteractiveShell(Component, Magic):'
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
-    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None):
+    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
+                      exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
@@ -1250,18 +1255,24 b' class InteractiveShell(Component, Magic):'
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
-
-        
-        # Though this won't be called by syntax errors in the input line,
-        # there may be SyntaxError cases whith imported code.
         
         try:
             if exc_tuple is None:
                 etype, value, tb = sys.exc_info()
             else:
                 etype, value, tb = exc_tuple
+
+            if etype is None:
+                if hasattr(sys, 'last_type'):
+                    etype, value, tb = sys.last_type, sys.last_value, \
+                                       sys.last_traceback
+                else:
+                    self.write('No traceback available to show.\n')
+                    return
     
             if etype is SyntaxError:
+                # Though this won't be called by syntax errors in the input
+                # line, there may be SyntaxError cases whith imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 print "UsageError:", value
@@ -1277,12 +1288,20 b' class InteractiveShell(Component, Magic):'
                 if etype in self.custom_exceptions:
                     self.CustomTB(etype,value,tb)
                 else:
-                    self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
-                    if self.InteractiveTB.call_pdb and self.has_readline:
-                        # pdb mucks up readline, fix it back
-                        self.set_completer()
+                    if exception_only:
+                        m = ('An exception has occurred, use %tb to see the '
+                             'full traceback.')
+                        print m
+                        self.InteractiveTB.show_exception_only(etype, value)
+                    else:
+                        self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
+                        if self.InteractiveTB.call_pdb:
+                            # pdb mucks up readline, fix it back
+                            self.set_completer()
+                        
         except KeyboardInterrupt:
-            self.write("\nKeyboardInterrupt\n")
+            self.write("\nKeyboardInterrupt\n")        
+        
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
@@ -1295,7 +1314,7 b' class InteractiveShell(Component, Magic):'
         """
         etype, value, last_traceback = sys.exc_info()
 
-        # See note about these variables in showtraceback() below
+        # See note about these variables in showtraceback() above
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = last_traceback
@@ -1426,9 +1445,7 b' class InteractiveShell(Component, Magic):'
             return outcomps
 
     def set_custom_completer(self,completer,pos=0):
-        """set_custom_completer(completer,pos=0)
-
-        Adds a new custom completer function.
+        """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
@@ -1438,9 +1455,18 b' class InteractiveShell(Component, Magic):'
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_completer(self):
-        """reset readline's completer to be our own."""
+        """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.complete)
 
+    def set_completer_frame(self, frame=None):
+        """Set the frame of the completer."""
+        if frame:
+            self.Completer.namespace = frame.f_locals
+            self.Completer.global_namespace = frame.f_globals
+        else:
+            self.Completer.namespace = self.user_ns
+            self.Completer.global_namespace = self.user_global_ns
+
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
@@ -1448,20 +1474,25 b' class InteractiveShell(Component, Magic):'
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
+        if self.readline_use:
+            import IPython.utils.rlineimpl as readline
+                  
         self.rl_next_input = None
         self.rl_do_indent = False
 
-        if not self.readline_use:
-            return
-
-        import IPython.utils.rlineimpl as readline
-                  
-        if not readline.have_readline:
-            self.has_readline = 0
+        if not self.readline_use or not readline.have_readline:
+            self.has_readline = False
             self.readline = None
-            # no point in bugging windows users with this every time:
-            warn('Readline services not available on this platform.')
+            # Set a number of methods that depend on readline to be no-op
+            self.savehist = no_op
+            self.reloadhist = no_op
+            self.set_completer = no_op
+            self.set_custom_completer = no_op
+            self.set_completer_frame = no_op
+            warn('Readline services not available or not loaded.')
         else:
+            self.has_readline = True
+            self.readline = readline
             sys.modules['readline'] = readline
             import atexit
             from IPython.core.completer import IPCompleter
@@ -1496,8 +1527,6 b' class InteractiveShell(Component, Magic):'
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
-            self.has_readline = 1
-            self.readline = readline
             # save this in sys so embedded copies can restore it properly
             sys.ipcompleter = self.Completer.complete
             self.set_completer()
@@ -1569,6 +1598,9 b' class InteractiveShell(Component, Magic):'
         # Set user colors (don't do it in the constructor above so that it
         # doesn't crash if colors option is invalid)
         self.magic_colors(self.colors)
+        # History was moved to a separate module
+        from . import history
+        history.init_ipython(self)
 
     def magic(self,arg_s):
         """Call a magic function by name.
@@ -1587,7 +1619,6 b' class InteractiveShell(Component, Magic):'
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
-
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
@@ -1826,7 +1857,8 b' class InteractiveShell(Component, Magic):'
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
-                    self.readline_startup_hook(None)
+                    if self.has_readline:
+                        self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
             except bdb.BdbQuit:
@@ -1843,10 +1875,13 b' class InteractiveShell(Component, Magic):'
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
-            
+
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
+        # Turn off the exit flag, so the mainloop can be restarted if desired
+        self.exit_now = False
+
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
@@ -1862,7 +1897,8 b' class InteractiveShell(Component, Magic):'
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
-            If True, then don't print errors for non-zero exit statuses.
+            If True, then silence SystemExit for non-zero status (it is always
+            silenced for zero status, as it is so common).
         """
         kw.setdefault('exit_ignore', False)
 
@@ -1887,40 +1923,21 b' class InteractiveShell(Component, Magic):'
 
         with prepended_to_syspath(dname):
             try:
-                if sys.platform == 'win32' and sys.version_info < (2,5,1):
-                    # Work around a bug in Python for Windows.  The bug was
-                    # fixed in in Python 2.5 r54159 and 54158, but that's still
-                    # SVN Python as of March/07.  For details, see:
-                    # http://projects.scipy.org/ipython/ipython/ticket/123
-                    try:
-                        globs,locs = where[0:2]
-                    except:
-                        try:
-                            globs = locs = where[0]
-                        except:
-                            globs = locs = globals()
-                    exec file(fname) in globs,locs
-                else:
-                    execfile(fname,*where)
-            except SyntaxError:
-                self.showsyntaxerror()
-                warn('Failure executing file: <%s>' % fname)
+                execfile(fname,*where)
             except SystemExit, status:
-                # Code that correctly sets the exit status flag to success (0)
-                # shouldn't be bothered with a traceback.  Note that a plain
-                # sys.exit() does NOT set the message to 0 (it's empty) so that
-                # will still get a traceback.  Note that the structure of the
-                # SystemExit exception changed between Python 2.4 and 2.5, so
-                # the checks must be done in a version-dependent way.
-                show = False
-                if status.message!=0 and not kw['exit_ignore']:
-                    show = True
-                if show:
-                    self.showtraceback()
-                    warn('Failure executing file: <%s>' % fname)
+                # If the call was made with 0 or None exit status (sys.exit(0)
+                # or sys.exit() ), don't bother showing a traceback, as both of
+                # these are considered normal by the OS:
+                # > python -c'import sys;sys.exit(0)'; echo $?
+                # 0
+                # > python -c'import sys;sys.exit()'; echo $?
+                # 0
+                # For other exit status, we show the exception unless
+                # explicitly silenced, but only in short form.
+                if status.code not in (0, None) and not kw['exit_ignore']:
+                    self.showtraceback(exception_only=True)
             except:
                 self.showtraceback()
-                warn('Failure executing file: <%s>' % fname)
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
@@ -2134,9 +2151,8 b' class InteractiveShell(Component, Magic):'
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
-            self.showtraceback()
-            warn("Type %exit or %quit to exit IPython "
-                 "(%Exit or %Quit do so unconditionally).",level=1)
+            self.showtraceback(exception_only=True)
+            warn("To exit: use any of 'exit', 'quit', %Exit or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
@@ -2278,6 +2294,8 b' class InteractiveShell(Component, Magic):'
     def get_component(self, name=None, klass=None):
         """Fetch a component by name and klass in my tree."""
         c = Component.get_instances(root=self, name=name, klass=klass)
+        if len(c) == 0:
+            return None
         if len(c) == 1:
             return c[0]
         else:
@@ -2309,7 +2327,7 b' class InteractiveShell(Component, Magic):'
         You can put your extension modules anywhere you want, as long as
         they can be imported by Python's standard import mechanism.  However,
         to make it easy to write extensions, you can also put your extensions
-        in ``os.path.join(self.ipythondir, 'extensions')``.  This directory
+        in ``os.path.join(self.ipython_dir, 'extensions')``.  This directory
         is added to ``sys.path`` automatically.
         """
         from IPython.utils.syspathcontext import prepended_to_syspath
@@ -2318,7 +2336,7 b' class InteractiveShell(Component, Magic):'
             with prepended_to_syspath(self.ipython_extension_dir):
                 __import__(module_str)
         mod = sys.modules[module_str]
-        self._call_load_ipython_extension(mod)
+        return self._call_load_ipython_extension(mod)
 
     def unload_extension(self, module_str):
         """Unload an IPython extension by its module name.
@@ -2350,11 +2368,11 b' class InteractiveShell(Component, Magic):'
 
     def _call_load_ipython_extension(self, mod):
         if hasattr(mod, 'load_ipython_extension'):
-            mod.load_ipython_extension(self)
+            return mod.load_ipython_extension(self)
 
     def _call_unload_ipython_extension(self, mod):
         if hasattr(mod, 'unload_ipython_extension'):
-            mod.unload_ipython_extension(self)
+            return mod.unload_ipython_extension(self)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
@@ -2362,6 +2380,10 b' class InteractiveShell(Component, Magic):'
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(self, config=self.config)
+        # Ultimately this will be refactored in the new interpreter code, but
+        # for now, we should expose the main prefilter method (there's legacy
+        # code out there that may rely on this).
+        self.prefilter = self.prefilter_manager.prefilter_lines
 
     #-------------------------------------------------------------------------
     # Utilities
@@ -2427,11 +2449,46 b' class InteractiveShell(Component, Magic):'
         return ask_yes_no(prompt,default)
 
     #-------------------------------------------------------------------------
+    # Things related to GUI support and pylab
+    #-------------------------------------------------------------------------
+
+    def enable_pylab(self, gui=None):
+        """Activate pylab support at runtime.
+
+        This turns on support for matplotlib, preloads into the interactive
+        namespace all of numpy and pylab, and configures IPython to correcdtly
+        interact with the GUI event loop.  The GUI backend to be used can be
+        optionally selected with the optional :param:`gui` argument.
+
+        Parameters
+        ----------
+        gui : optional, string
+
+          If given, dictates the choice of matplotlib GUI backend to use
+          (should be one of IPython's supported backends, 'tk', 'qt', 'wx' or
+          'gtk'), otherwise we use the default chosen by matplotlib (as
+          dictated by the matplotlib build-time options plus the user's
+          matplotlibrc configuration file).
+        """
+        # We want to prevent the loading of pylab to pollute the user's
+        # namespace as shown by the %who* magics, so we execute the activation
+        # code in an empty namespace, and we update *both* user_ns and
+        # user_config_ns with this information.
+        ns = {}
+        gui = pylab_activate(ns, gui)
+        self.user_ns.update(ns)
+        self.user_config_ns.update(ns)
+        # Now we must activate the gui pylab wants to use, and fix %run to take
+        # plot updates into account
+        enable_gui(gui)
+        self.magic_run = self._pylab_magic_run
+
+    #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
 
     def ask_exit(self):
-        """ Call for exiting. Can be overiden and used as a callback. """
+        """ Ask the shell to exit. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):

IPython/core/magic.py

@@ -21,6 +21,7 b' import os'
 import pdb
 import pydoc
 import sys
+import shutil
 import re
 import tempfile
 import time
@@ -43,21 +44,26 b' except ImportError:'
 
 # Homebrewed
 import IPython
-from IPython.utils import wildcard
+import IPython.utils.generics
+
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
+from IPython.core.error import UsageError
 from IPython.core.fakemodule import FakeModule
+from IPython.core.macro import Macro
+from IPython.core.page import page
 from IPython.core.prefilter import ESC_MAGIC
+from IPython.core.pylabtools import mpl_runner
+from IPython.lib.inputhook import enable_gui
 from IPython.external.Itpl import Itpl, itpl, printpl,itplns
+from IPython.testing import decorators as testdec
+from IPython.utils import platutils
+from IPython.utils import wildcard
 from IPython.utils.PyColorize import Parser
 from IPython.utils.ipstruct import Struct
-from IPython.core.macro import Macro
+
+# XXX - We need to switch to explicit imports here with genutils
 from IPython.utils.genutils import *
-from IPython.core.page import page
-from IPython.utils import platutils
-import IPython.utils.generics
-from IPython.core.error import UsageError
-from IPython.testing import decorators as testdec
 
 #***************************************************************************
 # Utility functions
@@ -79,10 +85,16 b' def compress_dhist(dh):'
         done.add(h)
 
     return newhead + tail        
-        
+
 
 #***************************************************************************
 # Main class implementing Magic functionality
+
+# XXX - for some odd reason, if Magic is made a new-style class, we get errors
+# on construction of the main InteractiveShell object.  Something odd is going
+# on with super() calls, Component and the MRO... For now leave it as-is, but
+# eventually this needs to be clarified.
+
 class Magic:
     """Magic functions for InteractiveShell.
 
@@ -334,7 +346,7 b' python-profiler package from non-free.""")'
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
-        posix = kw.get('posix',True)
+        posix = kw.get('posix', os.name == 'posix')
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
@@ -863,7 +875,7 b' Currently the magic system has the following functions:\\n"""'
                     show_all=opt('a'),ignore_case=ignore_case)
         except:
             shell.showtraceback()
-
+        
     def magic_who_ls(self, parameter_s=''):
         """Return a sorted list of all interactive variables.
 
@@ -873,17 +885,15 b' Currently the magic system has the following functions:\\n"""'
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_config_ns = self.shell.user_config_ns
-        out = []
+        out = [ i for i in user_ns
+                if not i.startswith('_') \
+                and not (i in internal_ns or i in user_config_ns) ]
+
         typelist = parameter_s.split()
+        if typelist:
+            typeset = set(typelist)
+            out = [i for i in out if type(i).__name__ in typeset]
 
-        for i in user_ns:
-            if not (i.startswith('_') or i.startswith('_i')) \
-                   and not (i in internal_ns or i in user_config_ns):
-                if typelist:
-                    if type(user_ns[i]).__name__ in typelist:
-                        out.append(i)
-                else:
-                    out.append(i)
         out.sort()
         return out
         
@@ -1268,7 +1278,6 b' Currently the magic system has the following functions:\\n"""'
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
-        
         self.shell.debugger(force=True)
 
     @testdec.skip_doctest
@@ -1571,7 +1580,7 b' Currently the magic system has the following functions:\\n"""'
             return
 
         if filename.lower().endswith('.ipy'):
-            self.safe_execfile_ipy(filename)
+            self.shell.safe_execfile_ipy(filename)
             return
         
         # Control the response to exit() calls made by the script being run
@@ -2522,20 +2531,7 b' Defaulting color scheme to \'NoColor\'"""'
         self.shell.pprint = 1 - self.shell.pprint
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.pprint]
-        
-    def magic_exit(self, parameter_s=''):
-        """Exit IPython, confirming if configured to do so.
-
-        You can configure whether IPython asks for confirmation upon exit by
-        setting the confirm_exit flag in the ipythonrc file."""
-
-        self.shell.exit()
-
-    def magic_quit(self, parameter_s=''):
-        """Exit IPython, confirming if configured to do so (like %exit)"""
-
-        self.shell.exit()
-        
+                
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
@@ -2685,11 +2681,12 b' Defaulting color scheme to \'NoColor\'"""'
                             else:
                                 syscmdlist.append(ff)
             else:
+                no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
-                        if isexec(ff) and base.lower() not in self.shell.no_alias:
+                        if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
@@ -3365,7 +3362,7 b' Defaulting color scheme to \'NoColor\'"""'
         # By default, echo back to terminal unless quiet mode is requested
         if not opts.has_key('q'):
             write = self.shell.write
-            write(block)
+            write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
@@ -3378,34 +3375,6 b' Defaulting color scheme to \'NoColor\'"""'
         qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
         
         page(qr)
-        
-    def magic_upgrade(self,arg):
-        """ Upgrade your IPython installation
-        
-        This will copy the config files that don't yet exist in your 
-        ipython dir from the system config dir. Use this after upgrading 
-        IPython if you don't wish to delete your .ipython dir.
-
-        Call with -nolegacy to get rid of ipythonrc* files (recommended for
-        new users)
-
-        """
-        ip = self.getapi()
-        ipinstallation = path(IPython.__file__).dirname()
-        upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'utils' / 'upgradedir.py')
-        src_config = ipinstallation / 'config' / 'userconfig'
-        userdir = path(ip.config.IPYTHONDIR)
-        cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
-        print ">",cmd
-        shell(cmd)
-        if arg == '-nolegacy':
-            legacy = userdir.files('ipythonrc*')
-            print "Nuking legacy files:",legacy
-            
-            [p.remove() for p in legacy]
-            suffix = (sys.platform == 'win32' and '.ini' or '')
-            (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
-
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
@@ -3427,8 +3396,6 b' Defaulting color scheme to \'NoColor\'"""'
         your existing IPython session.
         """
 
-        # XXX - Fix this to have cleaner activate/deactivate calls.
-        from IPython.extensions import InterpreterPasteInput as ipaste
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
@@ -3451,8 +3418,6 b' Defaulting color scheme to \'NoColor\'"""'
 
         if mode == False:
             # turn on
-            ipaste.activate_prefilter()
-
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
@@ -3466,13 +3431,11 b' Defaulting color scheme to \'NoColor\'"""'
                                   oc.prompt_out.pad_left = False
 
             shell.pprint = False
-
+            
             shell.magic_xmode('Plain')
 
         else:
             # turn off
-            ipaste.deactivate_prefilter()
-
             oc.prompt1.p_template = shell.prompt_in1
             oc.prompt2.p_template = shell.prompt_in2
             oc.prompt_out.p_template = shell.prompt_out
@@ -3485,7 +3448,7 b' Defaulting color scheme to \'NoColor\'"""'
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
-            rc.pprint = dstore.rc_pprint
+            shell.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
@@ -3503,7 +3466,7 b' Defaulting color scheme to \'NoColor\'"""'
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
-        are supports:  wxPython, PyQt4, PyGTK, and Tk::
+        are supported:  wxPython, PyQt4, PyGTK, and Tk::
 
             %gui wx      # enable wxPython event loop integration
             %gui qt4|qt  # enable PyQt4 event loop integration
@@ -3522,25 +3485,13 b' Defaulting color scheme to \'NoColor\'"""'
 
         This is highly recommended for most users.
         """
-        from IPython.lib import inputhook
-        if "-a" in parameter_s:
-            app = True
-        else:
-            app = False
-        if not parameter_s:
-            inputhook.clear_inputhook()
-        elif 'wx' in parameter_s:
-            return inputhook.enable_wx(app)
-        elif ('qt4' in parameter_s) or ('qt' in parameter_s):
-            return inputhook.enable_qt4(app)
-        elif 'gtk' in parameter_s:
-            return inputhook.enable_gtk(app)
-        elif 'tk' in parameter_s:
-            return inputhook.enable_tk(app)
+        opts, arg = self.parse_options(parameter_s,'a')
+        if arg=='': arg = None
+        return enable_gui(arg, 'a' in opts)
 
     def magic_load_ext(self, module_str):
         """Load an IPython extension by its module name."""
-        self.load_extension(module_str)
+        return self.load_extension(module_str)
 
     def magic_unload_ext(self, module_str):
         """Unload an IPython extension by its module name."""
@@ -3550,4 +3501,113 b' Defaulting color scheme to \'NoColor\'"""'
         """Reload an IPython extension by its module name."""
         self.reload_extension(module_str)
 
+    @testdec.skip_doctest
+    def magic_install_profiles(self, s):
+        """Install the default IPython profiles into the .ipython dir.
+
+        If the default profiles have already been installed, they will not
+        be overwritten. You can force overwriting them by using the ``-o``
+        option::
+
+            In [1]: %install_profiles -o
+        """
+        if '-o' in s:
+            overwrite = True
+        else:
+            overwrite = False
+        from IPython.config import profile
+        profile_dir = os.path.split(profile.__file__)[0]
+        ipython_dir = self.ipython_dir
+        files = os.listdir(profile_dir)
+
+        to_install = []
+        for f in files:
+            if f.startswith('ipython_config'):
+                src = os.path.join(profile_dir, f)
+                dst = os.path.join(ipython_dir, f)
+                if (not os.path.isfile(dst)) or overwrite:
+                    to_install.append((f, src, dst))
+        if len(to_install)>0:
+            print "Installing profiles to: ", ipython_dir
+            for (f, src, dst) in to_install:
+                shutil.copy(src, dst)
+                print "    %s" % f
+
+    def magic_install_default_config(self, s):
+        """Install IPython's default config file into the .ipython dir.
+
+        If the default config file (:file:`ipython_config.py`) is already
+        installed, it will not be overwritten. You can force overwriting
+        by using the ``-o`` option::
+
+            In [1]: %install_default_config
+        """
+        if '-o' in s:
+            overwrite = True
+        else:
+            overwrite = False
+        from IPython.config import default
+        config_dir = os.path.split(default.__file__)[0]
+        ipython_dir = self.ipython_dir
+        default_config_file_name = 'ipython_config.py'
+        src = os.path.join(config_dir, default_config_file_name)
+        dst = os.path.join(ipython_dir, default_config_file_name)
+        if (not os.path.isfile(dst)) or overwrite:
+            shutil.copy(src, dst)
+            print "Installing default config file: %s" % dst
+
+    # Pylab support: simple wrappers that activate pylab, load gui input
+    # handling and modify slightly %run
+
+    @testdec.skip_doctest
+    def _pylab_magic_run(self, parameter_s=''):
+        Magic.magic_run(self, parameter_s,
+                        runner=mpl_runner(self.shell.safe_execfile))
+
+    _pylab_magic_run.__doc__ = magic_run.__doc__
+
+    @testdec.skip_doctest
+    def magic_pylab(self, s):
+        """Load numpy and matplotlib to work interactively.
+
+        %pylab [GUINAME]
+
+        This function lets you activate pylab (matplotlib, numpy and
+        interactive support) at any point during an IPython session.
+
+        It will import at the top level numpy as np, pyplot as plt, matplotlib,
+        pylab and mlab, as well as all names from numpy and pylab.
+
+        Parameters
+        ----------
+        guiname : optional
+          One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk' or
+          'tk').  If given, the corresponding Matplotlib backend is used,
+          otherwise matplotlib's default (which you can override in your
+          matplotlib config file) is used.
+
+        Examples
+        --------
+        In this case, where the MPL default is TkAgg:
+        In [2]: %pylab
+
+        Welcome to pylab, a matplotlib-based Python environment.
+        Backend in use: TkAgg
+        For more information, type 'help(pylab)'.
+
+        But you can explicitly request a different backend:
+        In [3]: %pylab qt
+
+        Welcome to pylab, a matplotlib-based Python environment.
+        Backend in use: Qt4Agg
+        For more information, type 'help(pylab)'.
+        """
+        self.shell.enable_pylab(s)
+
+    def magic_tb(self, s):
+        """Print the last traceback with the currently active exception mode.
+
+        See %xmode for changing exception reporting modes."""
+        self.shell.showtraceback()
+
 # end Magic

IPython/core/prefilter.py

@@ -39,7 +39,7 b' from IPython.core.splitinput import split_user_input'
 from IPython.core.page import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool
-from IPython.utils.genutils import make_quoted_expr
+from IPython.utils.genutils import make_quoted_expr, Term
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
@@ -158,7 +158,7 b' class LineInfo(object):'
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
-            self._oinfo = ip._ofind(self.ifun)
+            self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):                                                         
@@ -362,7 +362,7 b' class PrefilterManager(Component):'
                 line = transformer.transform(line, continue_prompt)
         return line
 
-    def prefilter_line(self, line, continue_prompt):
+    def prefilter_line(self, line, continue_prompt=False):
         """Prefilter a single input line as text.
 
         This method prefilters a single line of text by calling the
@@ -416,7 +416,7 b' class PrefilterManager(Component):'
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
-    def prefilter_lines(self, lines, continue_prompt):
+    def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of
@@ -427,11 +427,19 b' class PrefilterManager(Component):'
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
-        out = []
-        for line in lines.rstrip('\n').split('\n'):
-            out.append(self.prefilter_line(line, continue_prompt))
-        return '\n'.join(out)
-
+        llines = lines.rstrip('\n').split('\n')
+        # We can get multiple lines in one shot, where multiline input 'blends'
+        # into one line, in cases like recalling from the readline history
+        # buffer.  We need to make sure that in such cases, we correctly
+        # communicate downstream which line is first and which are continuation
+        # ones.
+        if len(llines) > 1:
+            out = '\n'.join([self.prefilter_line(line, lnum>0)
+                             for lnum, line in enumerate(llines) ])
+        else:
+            out = self.prefilter_line(llines[0], continue_prompt)
+            
+        return out
 
 #-----------------------------------------------------------------------------
 # Prefilter transformers
@@ -508,6 +516,47 b' class AssignMagicTransformer(PrefilterTransformer):'
         return line
 
 
+_classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
+
+class PyPromptTransformer(PrefilterTransformer):
+    """Handle inputs that start with '>>> ' syntax."""
+
+    priority = Int(50, config=True)
+
+    def transform(self, line, continue_prompt):
+
+        if not line or line.isspace() or line.strip() == '...':
+            # This allows us to recognize multiple input prompts separated by
+            # blank lines and pasted in a single chunk, very common when
+            # pasting doctests or long tutorial passages.
+            return ''
+        m = _classic_prompt_re.match(line)
+        if m:
+            return line[len(m.group(0)):]
+        else:
+            return line
+
+
+_ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
+
+class IPyPromptTransformer(PrefilterTransformer):
+    """Handle inputs that start classic IPython prompt syntax."""
+
+    priority = Int(50, config=True)
+
+    def transform(self, line, continue_prompt):
+
+        if not line or line.isspace() or line.strip() == '...':
+            # This allows us to recognize multiple input prompts separated by
+            # blank lines and pasted in a single chunk, very common when
+            # pasting doctests or long tutorial passages.
+            return ''
+        m = _ipy_prompt_re.match(line)
+        if m:
+            return line[len(m.group(0)):]
+        else:
+            return line
+
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
@@ -755,9 +804,17 b' class PrefilterHandler(Component):'
         line = line_info.line
         continue_prompt = line_info.continue_prompt
 
-        if (continue_prompt and self.shell.autoindent and line.isspace() and
-            (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2 or
-             (self.shell.buffer[-1]).isspace() )):
+        if (continue_prompt and
+            self.shell.autoindent and
+            line.isspace() and
+            
+            (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2
+             or
+             not self.shell.buffer
+             or
+             (self.shell.buffer[-1]).isspace()
+             )
+            ):
             line = ''
 
         self.shell.log(line, line, continue_prompt)
@@ -845,12 +902,11 b' class AutoHandler(PrefilterHandler):'
         pre     = line_info.pre
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
-
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
-            self.log(line,line,continue_prompt)
+            self.shell.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
@@ -967,7 +1023,9 b' class EmacsHandler(PrefilterHandler):'
 
 _default_transformers = [
     AssignSystemTransformer,
-    AssignMagicTransformer
+    AssignMagicTransformer,
+    PyPromptTransformer,
+    IPyPromptTransformer,
 ]
 
 _default_checkers = [
@@ -992,4 +1050,3 b' _default_handlers = ['
     HelpHandler,
     EmacsHandler
 ]
-

IPython/core/prompts.py

@@ -549,18 +549,23 b' class CachedOutput:'
             # print "Got prompt: ", outprompt
             if self.do_full_cache:
                 cout_write(outprompt)
-            else:
-                print "self.do_full_cache = False"
 
-            # and now call a possibly user-defined print mechanism
-            manipulated_val = self.display(arg)
+            # and now call a possibly user-defined print mechanism. Note that
+            # self.display typically prints as a side-effect, we don't do any
+            # printing to stdout here.
+            try:
+                manipulated_val = self.display(arg)
+            except TypeError:
+                # If the user's display hook didn't return a string we can
+                # print, we're done.  Happens commonly if they return None
+                cout_write('\n')
+                return
             
             # user display hooks can change the variable to be stored in
             # output history
-            
             if manipulated_val is not None:
                 arg = manipulated_val
-                
+
             # avoid recursive reference when displaying _oh/Out
             if arg is not self.user_ns['_oh']:
                 self.update(arg)

IPython/core/quitter.py

@@ -1,10 +1,10 b''
-#!/usr/bin/env python
-# encoding: utf-8
+# coding: utf-8
 """
 A simple class for quitting IPython.
 
-Authors:
-
+Authors
+-------
+* Fernando Perez
 * Brian Granger
 """
 
@@ -19,6 +19,7 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
+import sys
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
@@ -30,9 +31,13 b' class Quitter(object):'
         self.shell = shell
         self.name = name
         
-    def __repr__(self):
+    def __str__(self):
         return 'Type %s() to exit.' % self.name
-    __str__ = __repr__
 
     def __call__(self):
-        self.shell.exit()
  No newline at end of file
+        self.shell.ask_exit()
+
+    # Repr MUST return a string, else display like pprint hooks get confused
+    def __repr__(self):
+        self.shell.ask_exit()
+        return 'Bye.'

IPython/core/release.py

@@ -23,7 +23,7 b" name = 'ipython'"
 development = True    # change this to False to do a release
 version_base = '0.11'
 branch = 'ipython'
-revision = '1205'
+revision = '1321'
 
 if development:
     if branch == 'ipython':

IPython/core/tests/tclass.py

@@ -1,16 +1,12 b''
-"""Simple script to instantiate a class for testing %run"""
+"""Simple script to be run *twice*, to check reference counting bugs.
 
-import sys
-
-# An external test will check that calls to f() work after %run
-class foo: pass
+See test_run for details."""
 
-def f():
-    return foo()
+import sys
 
-# We also want to ensure that while objects remain available for immediate
-# access, objects from *previous* runs of the same script get collected, to
-# avoid accumulating massive amounts of old references.
+# We want to ensure that while objects remain available for immediate access,
+# objects from *previous* runs of the same script get collected, to avoid
+# accumulating massive amounts of old references.
 class C(object):
     def __init__(self,name):
         self.name = name
@@ -18,6 +14,7 b' class C(object):'
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
 
+
 try:
     name = sys.argv[1]
 except IndexError:
@@ -25,3 +22,9 b' except IndexError:'
 else:
     if name.startswith('C'):
         c = C(name)
+
+#print >> sys.stderr, "ARGV:", sys.argv  # dbg
+
+# This next print statement is NOT debugging, we're making the check on a
+# completely separate process so we verify by capturing stdout:
+print 'ARGV 1-:', sys.argv[1:]

IPython/core/tests/test_iplib.py

@@ -13,68 +13,233 b' import tempfile'
 import nose.tools as nt
 
 # our own packages
-from IPython.core import iplib
-from IPython.core import ipapi
-from IPython.core.oldusersetup import user_setup
+from IPython.testing import decorators as dec
+from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
-# Useful global ipapi object and main IPython one.  Unfortunately we have a
-# long precedent of carrying the 'ipapi' global object which is injected into
-# the system namespace as _ip, but that keeps a pointer to the actual IPython
-# InteractiveShell instance, which is named IP.  Since in testing we do need
-# access to the real thing (we want to probe beyond what ipapi exposes), make
-# here a global reference to each.  In general, things that are exposed by the
-# ipapi instance should be read from there, but we also will often need to use
-# the actual IPython one.
-
-# Get the public instance of IPython, and if it's None, make one so we can use
-# it for testing
-ip = ipapi.get()
-if ip is None:
-    # IPython not running yet,  make one from the testing machinery for
-    # consistency when the test suite is being run via iptest
-    from IPython.testing.plugin import ipdoctest
-    ip = ipapi.get()
+# Get the public instance of IPython
+ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_reset():
     """reset must clear most namespaces."""
-    ip.reset()  # first, it should run without error
-    # Then, check that most namespaces end up empty
+    # The number of variables in the private user_config_ns is not zero, but it
+    # should be constant regardless of what we do
+    nvars_config_ns = len(ip.user_config_ns)
+
+    # Check that reset runs without error
+    ip.reset()
+
+    # Once we've reset it (to clear of any junk that might have been there from
+    # other tests, we can count how many variables are in the user's namespace
+    nvars_user_ns = len(ip.user_ns)
+
+    # Now add a few variables to user_ns, and check that reset clears them
+    ip.user_ns['x'] = 1
+    ip.user_ns['y'] = 1
+    ip.reset()
+    
+    # Finally, check that all namespaces have only as many variables as we
+    # expect to find in them:
     for ns in ip.ns_refs_table:
         if ns is ip.user_ns:
-            # The user namespace is reset with some data, so we can't check for
-            # it being empty
-            continue
-        nt.assert_equals(len(ns),0)
-
-
-# make sure that user_setup can be run re-entrantly in 'install' mode.
-def test_user_setup():
-    # use a lambda to pass kwargs to the generator
-    user_setup = lambda a,k: user_setup(*a,**k)
-    kw = dict(mode='install', interactive=False)
-
-    # Call the user setup and verify that the directory exists
-    yield user_setup, (ip.config.IPYTHONDIR,''), kw
-    yield os.path.isdir, ip.config.IPYTHONDIR
-
-    # Now repeat the operation with a non-existent directory. Check both that
-    # the call succeeds and that the directory is created.
-    tmpdir = tempfile.mktemp(prefix='ipython-test-')
-    # Use a try with an empty except because try/finally doesn't work with a 
-    # yield in Python 2.4.
-    try:
-        yield user_setup, (tmpdir,''), kw
-        yield os.path.isdir, tmpdir
-    except:
-        pass
-    # Clean up the temp dir once done
-    shutil.rmtree(tmpdir)
-    
  No newline at end of file
+            nvars_expected = nvars_user_ns
+        elif ns is ip.user_config_ns:
+            nvars_expected = nvars_config_ns
+        else:
+            nvars_expected = 0
+            
+        yield nt.assert_equals(len(ns), nvars_expected)
+
+
+# Tests for reporting of exceptions in various modes, handling of SystemExit,
+# and %tb functionality.  This is really a mix of testing ultraTB and iplib.
+
+def doctest_tb_plain():
+    """
+In [18]: xmode plain
+Exception reporting mode: Plain
+
+In [19]: run simpleerr.py
+Traceback (most recent call last):
+  ...line 32, in <module>
+    bar(mode)
+  ...line 16, in bar
+    div0()
+  ...line 8, in div0
+    x/y
+ZeroDivisionError: integer division or modulo by zero
+    """
+
+
+def doctest_tb_context():
+    """
+In [3]: xmode context
+Exception reporting mode: Context
+
+In [4]: run simpleerr.py
+---------------------------------------------------------------------------
+ZeroDivisionError                         Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode)
+     14     "bar"
+     15     if mode=='div':
+---> 16         div0()
+     17     elif mode=='exit':
+     18         try:
+<BLANKLINE>
+... in div0()
+      6     x = 1
+      7     y = 0
+----> 8     x/y
+      9 
+     10 def sysexit(stat, mode):
+<BLANKLINE>
+ZeroDivisionError: integer division or modulo by zero
+"""
+
+
+def doctest_tb_verbose():
+    """
+In [5]: xmode verbose
+Exception reporting mode: Verbose
+
+In [6]: run simpleerr.py
+---------------------------------------------------------------------------
+ZeroDivisionError                         Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+        global bar = <function bar at ...>
+        global mode = 'div'
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode='div')
+     14     "bar"
+     15     if mode=='div':
+---> 16         div0()
+        global div0 = <function div0 at ...>
+     17     elif mode=='exit':
+     18         try:
+<BLANKLINE>
+... in div0()
+      6     x = 1
+      7     y = 0
+----> 8     x/y
+        x = 1
+        y = 0
+      9 
+     10 def sysexit(stat, mode):
+<BLANKLINE>
+ZeroDivisionError: integer division or modulo by zero
+      """
+
+
+def doctest_tb_sysexit():
+    """
+In [17]: %xmode plain
+Exception reporting mode: Plain
+
+In [18]: %run simpleerr.py exit
+An exception has occurred, use %tb to see the full traceback.
+SystemExit: (1, 'Mode = exit')
+
+In [19]: %run simpleerr.py exit 2
+An exception has occurred, use %tb to see the full traceback.
+SystemExit: (2, 'Mode = exit')
+
+In [20]: %tb
+Traceback (most recent call last):
+  File ... in <module>
+    bar(mode)
+  File ... line 22, in bar
+    sysexit(stat, mode)
+  File ... line 11, in sysexit
+    raise SystemExit(stat, 'Mode = %s' % mode)
+SystemExit: (2, 'Mode = exit')
+
+In [21]: %xmode context
+Exception reporting mode: Context
+
+In [22]: %tb
+---------------------------------------------------------------------------
+SystemExit                                Traceback (most recent call last)
+<BLANKLINE>
+...<module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+     33 
+     34 
+<BLANKLINE>
+...bar(mode)
+     20         except:
+     21             stat = 1
+---> 22         sysexit(stat, mode)
+     23     else:
+     24         raise ValueError('Unknown mode')
+<BLANKLINE>
+...sysexit(stat, mode)
+      9 
+     10 def sysexit(stat, mode):
+---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
+     12 
+     13 def bar(mode):
+<BLANKLINE>
+SystemExit: (2, 'Mode = exit')
+
+In [23]: %xmode verbose
+Exception reporting mode: Verbose
+
+In [24]: %tb
+---------------------------------------------------------------------------
+SystemExit                                Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+        global bar = <function bar at ...>
+        global mode = 'exit'
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode='exit')
+     20         except:
+     21             stat = 1
+---> 22         sysexit(stat, mode)
+        global sysexit = <function sysexit at ...>
+        stat = 2
+        mode = 'exit'
+     23     else:
+     24         raise ValueError('Unknown mode')
+<BLANKLINE>
+... in sysexit(stat=2, mode='exit')
+      9 
+     10 def sysexit(stat, mode):
+---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
+        global SystemExit = undefined
+        stat = 2
+        mode = 'exit'
+     12 
+     13 def bar(mode):
+<BLANKLINE>
+SystemExit: (2, 'Mode = exit')
+    """

IPython/core/tests/test_magic.py

@@ -2,22 +2,31 b''
 
 Needs to be run by nose (to make ipython session available).
 """
+from __future__ import absolute_import
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
+# third-party
 import nose.tools as nt
 
+# our own
+from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
-
+#-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
@@ -37,6 +46,19 b' def test_rehashx():'
     yield (nt.assert_true, len(scoms) > 10)
 
 
+def test_magic_parse_options():
+    """Test that we don't mangle paths when parsing magic options."""
+    ip = get_ipython()
+    path = 'c:\\x'
+    opts = ip.parse_options('-f %s' % path,'f:')[0]
+    # argv splitting is os-dependent
+    if os.name == 'posix':
+        expected = 'c:x'
+    else:
+        expected = path
+    nt.assert_equals(opts['f'], expected)
+
+    
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
@@ -45,35 +67,94 b' def doctest_hist_f():'
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
+
+    In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
-    XXX - This test is not recording the output correctly.  Not sure why...
+    XXX - This test is not recording the output correctly.  For some reason, in
+    testing mode the raw history isn't getting populated.  No idea why.
+    Disabling the output checking for now, though at least we do run it.
 
-    In [20]: 'hist' in _ip.lsmagic()
-    Out[20]: True
+    In [1]: 'hist' in _ip.lsmagic()
+    Out[1]: True
 
-    In [6]: x=1
+    In [2]: x=1
 
-    In [7]: %hist -n -r 2
-    x=1  # random
-    hist -n -r 2  # random
+    In [3]: %hist -r 2
+    x=1 # random
+    %hist -r 2
     """
 
-# This test is known to fail on win32.
-# See ticket https://bugs.launchpad.net/bugs/366334
-def test_obj_del():
-    _ip = get_ipython()
-    """Test that object's __del__ methods are called on exit."""
-    test_dir = os.path.dirname(__file__)
-    del_file = os.path.join(test_dir,'obj_del.py')
-    ipython_cmd = find_cmd('ipython')
-    out = _ip.getoutput('%s %s' % (ipython_cmd, del_file))
-    nt.assert_equals(out,'obj_del.py: object A deleted')
-
+def doctest_hist_op():
+    """Test %hist -op
+
+    In [1]: class b:
+       ...:         pass
+       ...: 
+
+    In [2]: class s(b):
+       ...:         def __str__(self):
+       ...:             return 's'
+       ...: 
+
+    In [3]: 
+
+    In [4]: class r(b):
+       ...:         def __repr__(self):
+       ...:             return 'r'
+       ...: 
+
+    In [5]: class sr(s,r): pass
+       ...: 
+
+    In [6]: 
+
+    In [7]: bb=b()
+
+    In [8]: ss=s()
+
+    In [9]: rr=r()
+
+    In [10]: ssrr=sr()
+
+    In [11]: bb
+    Out[11]: <...b instance at ...>
+
+    In [12]: ss
+    Out[12]: <...s instance at ...>
+
+    In [13]: 
+
+    In [14]: %hist -op
+    >>> class b:
+    ...     pass
+    ... 
+    >>> class s(b):
+    ...     def __str__(self):
+    ...         return 's'
+    ... 
+    >>> 
+    >>> class r(b):
+    ...     def __repr__(self):
+    ...         return 'r'
+    ... 
+    >>> class sr(s,r): pass
+    >>> 
+    >>> bb=b()
+    >>> ss=s()
+    >>> rr=r()
+    >>> ssrr=sr()
+    >>> bb
+    <...b instance at ...>
+    >>> ss
+    <...s instance at ...>
+    >>> 
+    >>> get_ipython().magic("hist -op")
+    """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
@@ -97,8 +178,12 b' def test_shist():'
     yield nt.assert_equal,s.get(2),'world'
     
     shutil.rmtree(tfile)
+
     
[e-mail]_not_numpy
+# XXX failing for now, until we get clearcmd out of quarantine.  But we should
+# fix this and revert the skip to happen only if numpy is not around.
+#@dec.skipif_not_numpy
[e-mail]
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
@@ -109,162 +194,8 b' def test_numpy_clear_array_undec():'
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
[e-mail]()
-def test_fail_dec(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]('This one shouldn not run')
-def test_fail_dec2(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]
-def test_fail_dec3(*a,**k):
-    yield nt.assert_true, False
-
-
-def doctest_refbug():
-    """Very nasty problem with references held by multiple runs of a script.
-    See: https://bugs.launchpad.net/ipython/+bug/269966
-
-    In [1]: _ip.clear_main_mod_cache()
-    
-    In [2]: run refbug
-
-    In [3]: call_f()
-    lowercased: hello
-
-    In [4]: run refbug
-
-    In [5]: call_f()
-    lowercased: hello
-    lowercased: hello
-    """
-
-#-----------------------------------------------------------------------------
-# Tests for %run
-#-----------------------------------------------------------------------------
-
-# %run is critical enough that it's a good idea to have a solid collection of
-# tests for it, some as doctests and some as normal tests.
-
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass foo
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [4]: run tclass C-first_pass
-
-    In [5]: run tclass C-second_pass
-    tclass.py: deleting object: C-first_pass
-    """
-
-def doctest_run_builtins():
-    """Check that %run doesn't damage __builtins__ via a doctest.
-
-    This is similar to the test_run_builtins, but I want *both* forms of the
-    test to catch any possible glitches in our testing machinery, since that
-    modifies %run somewhat.  So for this, we have both a normal test (below)
-    and a doctest (this one).
-
-    In [1]: import tempfile
-
-    In [2]: bid1 = id(__builtins__)
-
-    In [3]: fname = tempfile.mkstemp()[1]
-
-    In [3]: f = open(fname,'w')
-
-    In [4]: f.write('pass\\n')
-
-    In [5]: f.flush()
-
-    In [6]: print type(__builtins__)
-    <type 'module'>
-
-    In [7]: %run "$fname"
-
-    In [7]: f.close()
-
-    In [8]: bid2 = id(__builtins__)
-
-    In [9]: print type(__builtins__)
-    <type 'module'>
-
-    In [10]: bid1 == bid2
-    Out[10]: True
-
-    In [12]: try:
-       ....:     os.unlink(fname)
-       ....: except:
-       ....:     pass
-       ....: 
-    """
-
-# For some tests, it will be handy to organize them in a class with a common
-# setup that makes a temp file
-
-class TestMagicRun(object):
-
-    def setup(self):
-        """Make a valid python temp file."""
-        fname = tempfile.mkstemp()[1]
-        f = open(fname,'w')
-        f.write('pass\n')
-        f.flush()
-        self.tmpfile = f
-        self.fname = fname
-
-    def run_tmpfile(self):
-        _ip = get_ipython()
-        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
-        # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run "%s"' % self.fname)
-
-    def test_builtins_id(self):
-        """Check that %run doesn't damage __builtins__ """
-        _ip = get_ipython()
-        # Test that the id of __builtins__ is not modified by %run
-        bid1 = id(_ip.user_ns['__builtins__'])
-        self.run_tmpfile()
-        bid2 = id(_ip.user_ns['__builtins__'])
-        tt.assert_equals(bid1, bid2)
-
-    def test_builtins_type(self):
-        """Check that the type of __builtins__ doesn't change with %run.
-        
-        However, the above could pass if __builtins__ was already modified to
-        be a dict (it should be a module) by a previous use of %run.  So we
-        also check explicitly that it really is a module:
-        """
-        _ip = get_ipython()
-        self.run_tmpfile()
-        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
-
-    def test_prompts(self):
-        """Test that prompts correctly generate after %run"""
-        self.run_tmpfile()
-        _ip = get_ipython()
-        p2 = str(_ip.outputcache.prompt2).strip()
-        nt.assert_equals(p2[:3], '...')
-
-    def teardown(self):
-        self.tmpfile.close()
-        try:
-            os.unlink(self.fname)
-        except:
-            # On Windows, even though we close the file, we still can't delete
-            # it.  I have no clue why
-            pass
-
 # Multiple tests for clipboard pasting
[e-mail]
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
@@ -286,11 +217,11 b' def test_paste():'
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
-        yield (nt.assert_equal, user_ns['x'], 1)
+        yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
-        yield (nt.assert_equal, user_ns['x'], 2)
+        yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
@@ -299,14 +230,14 b' def test_paste():'
         ...     y.append(i**2)
         ...
         """)
-        yield (nt.assert_equal, user_ns['x'], [1,2,3])
-        yield (nt.assert_equal, user_ns['y'], [1,4,9])
+        yield nt.assert_equal(user_ns['x'], [1,2,3])
+        yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
-        yield (nt.assert_false, 'x' in user_ns)
+        yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
-        yield (nt.assert_equal, user_ns['x'], [1,2,3])
+        yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
@@ -320,12 +251,29 b' def test_paste():'
             out = w.getvalue()
         finally:
             _ip.write = writer
-        yield (nt.assert_equal, user_ns['a'], 100)
-        yield (nt.assert_equal, user_ns['b'], 200)
-        yield (nt.assert_equal, out, code+"\n## -- End pasted text --\n")
+        yield nt.assert_equal(user_ns['a'], 100)
+        yield nt.assert_equal(user_ns['b'], 200)
+        yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
+
+def test_time():
+    _ip.magic('time None')
+
+
+def doctest_time():
+    """
+    In [10]: %time None
+    CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+    Wall time: 0.00 s
+    """
+
+def test_doctest_mode():
+    "Toggle doctest_mode twice, it should be a no-op and run without error"
+    _ip.magic('doctest_mode')
+    _ip.magic('doctest_mode')
+    

IPython/core/ultratb.py

@@ -88,7 +88,6 b' import types'
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
-
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython.utils import PyColorize
@@ -263,7 +262,7 b' def _fixed_getinnerframes(etb, context=1,tb_offset=0):'
 
 _parser = PyColorize.Parser()
     
-def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):
+def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
@@ -313,6 +312,15 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
 class TBTools:
     """Basic tools used by all traceback printer classes."""
 
+    #: Default output stream, can be overridden at call time.  A special value
+    #: of 'stdout' *as a string* can be given to force extraction of sys.stdout
+    #: at runtime.  This allows testing exception printing with doctests, that
+    #: swap sys.stdout just at execution time.
+    #: Warning: be VERY careful to set this to one of the Term streams, NEVER
+    #: directly to sys.stdout/err, because under win32 the Term streams come from
+    #: pyreadline and know how to handle color correctly, whie stdout/err don't.
+    out_stream = Term.cerr
+
     def __init__(self,color_scheme = 'NoColor',call_pdb=False):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
@@ -376,16 +384,31 b' class ListTB(TBTools):'
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
-        print >> Term.cerr, self.text(etype,value,elist)
-        Term.cerr.flush()
+        Term.cerr.writeln(self.text(etype,value,elist))
+
+    def text(self, etype, value, elist, context=5):
+        """Return a color formatted string with the traceback info.
+
+        Parameters
+        ----------
+        etype : exception type
+          Type of the exception raised.
 
-    def text(self,etype, value, elist,context=5):
-        """Return a color formatted string with the traceback info."""
+        value : object
+          Data stored in the exception
+
+        elist : list
+          List of frames, see class docstring for details.
+
+        Returns
+        -------
+        String with formatted exception.
+        """
 
         Colors = self.Colors
-        out_string = ['%s%s%s\n' % (Colors.topline,'-'*60,Colors.Normal)]
+        out_string = []
         if elist:
-            out_string.append('Traceback %s(most recent call last)%s:' % \
+            out_string.append('Traceback %s(most recent call last)%s:' %
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_string.extend(self._format_list(elist))
         lines = self._format_exception_only(etype, value)
@@ -492,15 +515,29 b' class ListTB(TBTools):'
             else:
                 list.append('%s\n' % str(stype))
 
-        # vds:>>
+        # sync with user hooks
         if have_filedata:
             ipinst = ipapi.get()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filename, lineno, 0)
-        # vds:<<
 
         return list
 
+    def show_exception_only(self, etype, value):
+        """Only print the exception type and message, without a traceback.
+        
+        Parameters
+        ----------
+        etype : exception type
+        value : exception value
+        """
+        # This method needs to use __call__ from *this* class, not the one from
+        # a subclass whose signature or behavior may be different
+        Term.cout.flush()
+        ostream = sys.stdout if self.out_stream == 'stdout' else Term.cerr
+        ostream.write(ListTB.text(self, etype, value, []))
+        ostream.flush()        
+
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
@@ -781,8 +818,8 b' class VerboseTB(TBTools):'
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
-                    _formatTracebackLines(lnum,index,lines,Colors,lvals,
-                                          col_scheme))))
+                    _format_traceback_lines(lnum,index,lines,Colors,lvals,
+                                            col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
@@ -854,11 +891,11 b' class VerboseTB(TBTools):'
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
-                if hasattr(self,'tb'):
+                if hasattr(self,'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
-                while self.tb.tb_next is not None:
+                while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
@@ -872,8 +909,7 b' class VerboseTB(TBTools):'
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
-        print >> Term.cerr, self.text(etype, evalue, etb)
-        Term.cerr.flush()
+        Term.cerr.writeln(self.text(etype, evalue, etb))
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
@@ -980,6 +1016,7 b' class AutoFormattedTB(FormattedTB):'
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
+    
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
@@ -990,16 +1027,18 b' class AutoFormattedTB(FormattedTB):'
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
-        
+
         if out is None:
-            out = Term.cerr
+            out = sys.stdout if self.out_stream=='stdout' else self.out_stream
         Term.cout.flush()
         if tb_offset is not None:
             tb_offset, self.tb_offset = self.tb_offset, tb_offset
-            print >> out, self.text(etype, evalue, etb)
+            out.write(self.text(etype, evalue, etb))
+            out.write('\n')
             self.tb_offset = tb_offset
         else:
-            print >> out, self.text(etype, evalue, etb)
+            out.write(self.text(etype, evalue, etb))
+            out.write('\n')
         out.flush()
         try:
             self.debugger()

IPython/core/usage.py

@@ -1,338 +1,47 b''
 # -*- coding: utf-8 -*-
-#*****************************************************************************
-#       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
+"""Usage information for the main IPython applications.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2010  The IPython Development Team
+#  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
 import sys
 from IPython.core import release
 
-__doc__ = """
-IPython -- An enhanced Interactive Python
-=========================================
+cl_usage = """\
+ipython [options] [files]
 
-A Python shell with automatic history (input and output), dynamic object
-introspection, easier configuration, command completion, access to the system
-shell and more.
-
-IPython can also be embedded in running programs. See EMBEDDING below.
-
-
-USAGE
-       ipython [options] files
-
-       If invoked with no options, it executes all the files listed in
-       sequence and drops you into the interpreter while still acknowledging
-       any options you may have set in your ipythonrc file. This behavior is
-       different from standard Python, which when called as python -i will
-       only execute one file and will ignore your configuration setup.
-
-       Please note that some of the configuration options are not available at
-       the command line, simply because they are not practical here. Look into
-       your ipythonrc configuration file for details on those. This file
-       typically installed in the $HOME/.ipython directory.
-
-       For Windows users, $HOME resolves to C:\\Documents and
-       Settings\\YourUserName in most instances, and _ipython is used instead
-       of .ipython, since some Win32 programs have problems with dotted names
-       in directories.
-
-       In the rest of this text, we will refer to this directory as
-       IPYTHONDIR.
-
-REGULAR OPTIONS
-       After the above threading options have been given, regular options  can
-       follow  in any order.  All options can be abbreviated to their shortest
-       non-ambiguous form and are case-sensitive.  One or two  dashes  can  be
-       used.   Some options have an alternate short form, indicated after a |.
-
-       Most options can also be set from your  ipythonrc  configuration  file.
-       See the provided examples for assistance.  Options given on the comman-
-       dline override the values set in the ipythonrc file.
-
-       All options with a [no] prepended can be specified in negated form
-       (using -nooption instead of -option) to turn the feature off.
-
-       -h, --help
-              Show summary of options.
-
-       -autocall <val>
-              Make IPython automatically call any callable object even if you
-              didn't type explicit parentheses. For example, 'str 43' becomes
-              'str(43)' automatically.  The value can be '0' to disable the
-              feature, '1' for 'smart' autocall, where it is not applied if
-              there are no more arguments on the line, and '2' for 'full'
-              autocall, where all callable objects are automatically called
-              (even if no arguments are present).  The default is '1'.
-
-       -[no]autoindent
-              Turn automatic indentation on/off.
-
-       -[no]automagic
-              Make magic commands automatic (without needing their first char-
-              acter to be %).  Type %magic at  the  IPython  prompt  for  more
-              information.
-
-       -[no]autoedit_syntax
-              When a syntax error occurs after editing a file, automatically
-              open the file to the trouble causing line for convenient fixing.
-
-       -[no]banner
-              Print the intial information banner (default on).
-
-       -c <command>
-              Execute  the  given  command  string, and set sys.argv to ['c'].
-              This is similar to the -c option in  the  normal  Python  inter-
-              preter.
-
-       -cache_size|cs <n>
-              Size  of  the output cache (maximum number of entries to hold in
-              memory).  The default is 1000, you can change it permanently  in
-              your  config  file.   Setting  it  to  0 completely disables the
-              caching system, and the minimum value accepted  is  20  (if  you
-              provide  a value less than 20, it is reset to 0 and a warning is
-              issued).  This limit is defined because otherwise  you'll  spend
-              more time re-flushing a too small cache than working.
-
-       -classic|cl
-              Gives IPython a similar feel to the classic Python prompt.
-
-       -colors <scheme>
-              Color  scheme  for  prompts  and exception reporting.  Currently
-              implemented: NoColor, Linux, and LightBG.
-
-       -[no]color_info
-              IPython can display information about objects via a set of func-
-              tions, and optionally can use colors for this, syntax highlight-
-              ing source code and various other  elements.   However,  because
-              this  information  is  passed  through a pager (like 'less') and
-              many pagers get confused with color codes, this option is off by
-              default.   You  can  test  it and turn it on permanently in your
-              ipythonrc file if it works for you.  As a reference, the  'less'
-              pager  supplied  with  Mandrake 8.2 works ok, but that in RedHat
-              7.2 doesn't.
-
-              Test it and turn it on permanently if it works with your system.
-              The  magic function @color_info allows you to toggle this inter-
-              actively for testing.
-
-       -[no]confirm_exit
-              Set to confirm when you try to exit IPython with  an  EOF  (Con-
-              trol-D in Unix, Control-Z/Enter in Windows). Note that using the
-              magic functions @Exit or @Quit you  can  force  a  direct  exit,
-              bypassing any confirmation.
-
-       -[no]debug
-              Show  information  about the loading process. Very useful to pin
-              down problems with your configuration files or  to  get  details
-              about session restores.
-
-       -[no]deep_reload
-              IPython  can use the deep_reload module which reloads changes in
-              modules recursively (it replaces the reload() function,  so  you
-              don't need to change anything to use it). deep_reload() forces a
-              full reload of modules whose code may have  changed,  which  the
-              default reload() function does not.
-
-              When  deep_reload  is off, IPython will use the normal reload(),
-              but deep_reload will still be available as dreload(). This  fea-
-              ture  is  off  by default [which means that you have both normal
-              reload() and dreload()].
-
-       -editor <name>
-              Which editor to use with the @edit command. By default,  IPython
-              will  honor  your EDITOR environment variable (if not set, vi is
-              the Unix default and notepad the Windows one). Since this editor
-              is  invoked on the fly by IPython and is meant for editing small
-              code snippets, you may want to use a small,  lightweight  editor
-              here (in case your default EDITOR is something like Emacs).
-
-       -ipythondir <name>
-              The  name  of  your  IPython configuration directory IPYTHONDIR.
-              This can also be  specified  through  the  environment  variable
-              IPYTHONDIR.
-
-       -log|l Generate  a log file of all input. The file is named
-              ipython_log.py in your current directory (which prevents logs
-              from multiple IPython sessions from trampling each other). You
-              can use this to later restore a session by loading your logfile
-              as a file to be executed with option -logplay (see below).
-
-       -logfile|lf
-              Specify the name of your logfile.
-
-       -logplay|lp
-              Replay  a previous log. For restoring a session as close as pos-
-              sible to the state you left it in, use this option  (don't  just
-              run the logfile). With -logplay, IPython will try to reconstruct
-              the previous working environment in full, not just  execute  the
-              commands in the logfile.
-              When  a  session is restored, logging is automatically turned on
-              again with the name of the logfile it was invoked  with  (it  is
-              read  from the log header). So once you've turned logging on for
-              a session, you can quit IPython and reload it as many  times  as
-              you  want  and  it  will continue to log its history and restore
-              from the beginning every time.
-
-              Caveats: there are limitations in this option. The history vari-
-              ables  _i*,_* and _dh don't get restored properly. In the future
-              we will try to implement full  session  saving  by  writing  and
-              retrieving  a failed because of inherent limitations of Python's
-              Pickle module, so this may have to wait.
-
-       -[no]messages
-              Print messages which IPython collects about its startup  process
-              (default on).
-
-       -[no]pdb
-              Automatically  call the pdb debugger after every uncaught excep-
-              tion. If you are used to debugging  using  pdb,  this  puts  you
-              automatically  inside of it after any call (either in IPython or
-              in code called by it) which triggers  an  exception  which  goes
-              uncaught.
-
-       -[no]pprint
-              IPython  can  optionally  use the pprint (pretty printer) module
-              for displaying results. pprint tends to give a nicer display  of
-              nested  data structures. If you like it, you can turn it on per-
-              manently in your config file (default off).
-
-       -profile|p <name>
-              Assume that your config file is ipythonrc-<name> (looks in  cur-
-              rent dir first, then in IPYTHONDIR). This is a quick way to keep
-              and load multiple config files for different  tasks,  especially
-              if  you  use  the include option of config files. You can keep a
-              basic IPYTHONDIR/ipythonrc file and then have  other  'profiles'
-              which  include  this  one  and  load extra things for particular
-              tasks. For example:
-
-              1) $HOME/.ipython/ipythonrc : load basic things you always want.
-              2)  $HOME/.ipython/ipythonrc-math  :  load  (1)  and basic math-
-              related modules.
-              3) $HOME/.ipython/ipythonrc-numeric : load (1) and  Numeric  and
-              plotting modules.
-
-              Since  it is possible to create an endless loop by having circu-
-              lar file inclusions, IPython will stop if it reaches  15  recur-
-              sive inclusions.
-
-       -prompt_in1|pi1 <string>
-              Specify  the string used for input prompts. Note that if you are
-              using numbered prompts, the number is represented with a '\#' in
-              the  string.  Don't forget to quote strings with spaces embedded
-              in them. Default: 'In [\#]: '.
-
-              Most bash-like  escapes  can  be  used  to  customize  IPython's
-              prompts, as well as a few additional ones which are IPython-spe-
-              cific.  All valid prompt escapes are described in detail in  the
-              Customization section of the IPython HTML/PDF manual.
-
-       -prompt_in2|pi2 <string>
-              Similar to the previous option, but used for the continuation
-              prompts. The special sequence '\D' is similar to '\#', but with
-              all digits replaced dots (so you can have your continuation
-              prompt aligned with your input prompt).  Default: ' .\D.: '
-              (note three spaces at the start for alignment with 'In [\#]').
-
-       -prompt_out|po <string>
-              String   used   for  output  prompts,  also  uses  numbers  like
-              prompt_in1.  Default: 'Out[\#]:'.
-
-       -quick Start in bare bones mode (no config file loaded).
-
-       -rcfile <name>
-              Name of your  IPython  resource  configuration  file.   normally
-              IPython    loads   ipythonrc   (from   current   directory)   or
-              IPYTHONDIR/ipythonrc.  If the loading of your config file fails,
-              IPython  starts  with  a  bare  bones  configuration (no modules
-              loaded at all).
-
-       -[no]readline
-              Use the readline library, which is needed to support  name  com-
-              pletion  and  command history, among other things. It is enabled
-              by default, but may cause  problems  for  users  of  X/Emacs  in
-              Python comint or shell buffers.
-
-              Note  that  emacs 'eterm' buffers (opened with M-x term) support
-              IPython's readline and syntax coloring fine, only  'emacs'  (M-x
-              shell and C-c !)  buffers do not.
-
-       -screen_length|sl <n>
-              Number  of lines of your screen.  This is used to control print-
-              ing of very long strings.  Strings longer than  this  number  of
-              lines  will be sent through a pager instead of directly printed.
-
-              The default value for this is 0, which means IPython will  auto-
-              detect  your  screen  size  every time it needs to print certain
-              potentially long strings (this doesn't change  the  behavior  of
-              the  'print'  keyword,  it's  only triggered internally). If for
-              some reason this isn't working well (it needs  curses  support),
-              specify it yourself. Otherwise don't change the default.
-
-       -separate_in|si <string>
-              Separator before input prompts.  Default '0.
-
-       -separate_out|so <string>
-              Separator before output prompts.  Default: 0 (nothing).
-
-       -separate_out2|so2 <string>
-              Separator after output prompts.  Default: 0 (nothing).
-
-       -nosep Shorthand for '-separate_in 0 -separate_out 0 -separate_out2 0'.
-              Simply removes all input/output separators.
-
-       -upgrade
-              Allows you to upgrade your  IPYTHONDIR  configuration  when  you
-              install  a  new  version  of  IPython.   Since  new versions may
-              include new command lines options or example files, this  copies
-              updated ipythonrc-type files.  However, it backs up (with a .old
-              extension) all files which it overwrites so that you  can  merge
-              back any custimizations you might have in your personal files.
-
-       -Version
-              Print version information and exit.
-
-       -wxversion <string>
-              Select a specific version of wxPython (used in conjunction with
-              -wthread). Requires the wxversion module, part of recent
-              wxPython distributions.
-
-       -xmode <modename>
-              Mode  for  exception reporting.  The valid modes are Plain, Con-
-              text, and Verbose.
-
-              - Plain: similar to python's normal traceback printing.
-
-              - Context: prints 5 lines of context  source  code  around  each
-              line in the traceback.
-
-              - Verbose: similar to Context, but additionally prints the vari-
-              ables currently visible where the exception happened (shortening
-              their  strings if too long).  This can potentially be very slow,
-              if you happen to have a huge data structure whose string  repre-
-              sentation  is  complex  to compute.  Your computer may appear to
-              freeze for a while with cpu usage at 100%.  If this occurs,  you
-              can cancel the traceback with Ctrl-C (maybe hitting it more than
-              once).
-
-
-EMBEDDING
-       It is possible to start an IPython instance inside your own Python pro-
-       grams.  In the documentation example files there are some illustrations
-       on how to do this.
-
-       This feature allows you to evalutate  dynamically  the  state  of  your
-       code,  operate  with  your  variables, analyze them, etc.  Note however
-       that any changes you make to values while in the shell do NOT propagate
-       back  to  the running code, so it is safe to modify your values because
-       you won't break your code in bizarre ways by doing so.
-"""
+IPython: an enhanced interactive Python shell.
+
+    A Python shell with automatic history (input and output), dynamic object
+    introspection, easier configuration, command completion, access to the
+    system shell and more.  IPython can also be embedded in running programs.
 
-cmd_line_usage = __doc__
+    If invoked with no options, it executes all the files listed in sequence
+    and exits, use -i to enter interactive mode after running the files.  Files
+    ending in .py will be treated as normal Python, but files ending in .ipy
+    can contain special IPython syntax (magic commands, shell expansions, etc.)
+
+    Please note that some of the configuration options are not available at the
+    command line, simply because they are not practical here. Look into your
+    ipython_config.py configuration file for details on those.
+
+    This file typically installed in the $HOME/.ipython directory.  For Windows
+    users, $HOME resolves to C:\\Documents and Settings\\YourUserName in most
+    instances.
+
+    In IPython's documentation, we will refer to this directory as IPYTHON_DIR,
+    you can change its default location by setting any path you want in this
+    environment variable.
+
+    For more information, see the manual available in HTML and PDF in your
+    installation, or online at http://ipython.scipy.org.
+"""
 
-#---------------------------------------------------------------------------
 interactive_usage = """
 IPython -- An enhanced Interactive Python
 =========================================

IPython/extensions/pretty.py

@@ -128,13 +128,15 b' class PrettyResultDisplay(Component):'
 #-----------------------------------------------------------------------------
 
 
-def load_ipython_extension(ip):
+def load_ipython_extension(ip=None):
     """Load the extension in IPython as a hook."""
+    if ip is None: ip = get_ipython()
     global _loaded
     if not _loaded:
         prd = PrettyResultDisplay(ip, name='pretty_result_display')
         ip.set_hook('result_display', prd, priority=99)
         _loaded = True
+        return prd
 
 def unload_ipython_extension(ip):
     """Unload the extension."""
@@ -163,60 +165,3 b' def dtype_pprinter(obj, p, cycle):'
                     p.breakable()
                 p.pretty(field)
             p.end_group(7, '])')
-
-
-#-----------------------------------------------------------------------------
-# Tests
-#-----------------------------------------------------------------------------
-
-
-def test_pretty():
-    """
-    In [1]: from IPython.extensions import ipy_pretty
-
-    In [2]: ipy_pretty.activate()
-
-    In [3]: class A(object):
-       ...:     def __repr__(self):
-       ...:         return 'A()'
-       ...:     
-       ...:     
-
-    In [4]: a = A()
-
-    In [5]: a
-    Out[5]: A()
-
-    In [6]: def a_pretty_printer(obj, p, cycle):
-       ...:     p.text('<A>')
-       ...:     
-       ...:     
-
-    In [7]: ipy_pretty.for_type(A, a_pretty_printer)
-
-    In [8]: a
-    Out[8]: <A>
-
-    In [9]: class B(object):
-       ...:     def __repr__(self):
-       ...:         return 'B()'
-       ...:     
-       ...:     
-
-    In [10]: B.__module__, B.__name__
-    Out[10]: ('__main__', 'B')
-
-    In [11]: def b_pretty_printer(obj, p, cycle):
-       ....:     p.text('<B>')
-       ....:     
-       ....:     
-
-    In [12]: ipy_pretty.for_type_by_name('__main__', 'B', b_pretty_printer)
-
-    In [13]: b = B()
-
-    In [14]: b
-    Out[14]: <B>
-    """
-    assert False, "This should only be doctested, not run."
-

IPython/extensions/tests/test_pretty.py

@@ -15,21 +15,20 b' Simple tests for :mod:`IPython.extensions.pretty`.'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
 from unittest import TestCase
 
 from IPython.core.component import Component, masquerade_as
 from IPython.core.iplib import InteractiveShell
 from IPython.extensions import pretty as pretty_ext
 from IPython.external import pretty
-
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
 from IPython.utils.traitlets import Bool
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class InteractiveShellStub(Component):
     pprint = Bool(True)
 
@@ -43,9 +42,11 b' class TestPrettyResultDisplay(TestCase):'
 
     def setUp(self):
         self.ip = InteractiveShellStub(None)
-        # This allows our stub to be retrieved instead of the real InteractiveShell
+        # This allows our stub to be retrieved instead of the real
+        # InteractiveShell
         masquerade_as(self.ip, InteractiveShell)
-        self.prd = pretty_ext.PrettyResultDisplay(self.ip, name='pretty_result_display')    
+        self.prd = pretty_ext.PrettyResultDisplay(self.ip,
+                                                  name='pretty_result_display')
 
     def test_for_type(self):
         self.prd.for_type(A, a_pprinter)
@@ -53,4 +54,48 b' class TestPrettyResultDisplay(TestCase):'
         result = pretty.pretty(a)
         self.assertEquals(result, "<A>")
 
+ipy_src = """
+class A(object):
+    def __repr__(self):
+        return 'A()'
+    
+class B(object):
+    def __repr__(self):
+        return 'B()'
+ 
+a = A()
+b = B()
+
+def a_pretty_printer(obj, p, cycle):
+    p.text('<A>')
+
+def b_pretty_printer(obj, p, cycle):
+    p.text('<B>')
+
+
+a
+b
+
+ip = get_ipython()
+prd = ip.load_extension('pretty')
+prd.for_type(A, a_pretty_printer)
+prd.for_type_by_name(B.__module__, B.__name__, b_pretty_printer)
+
+a
+b
+"""
+ipy_out = """
+A()
+B()
+<A>
+<B>
+"""
 
+class TestPrettyInteractively(tt.TempFileMixin):
+    
+    # XXX Unfortunately, ipexec_validate fails under win32.  If someone helps
+    # us write a win32-compatible version, we can reactivate this test.
+    @dec.skip_win32
+    def test_printers(self):
+        self.mktmp(ipy_src, '.ipy')
+        tt.ipexec_validate(self.fname, ipy_out)

IPython/external/argparse.py

@@ -2,25 +2,17 b''
 
 # Copyright © 2006-2009 Steven J. Bethard <[e-mail]>.
 #
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
 #
-#  * Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
 
 """Command-line parsing library
 
@@ -83,7 +75,7 b' considered public as object names -- the API of the formatter objects is'
 still considered an implementation detail.)
 """
 
-__version__ = '1.0.1'
+__version__ = '1.1a1'
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -92,7 +84,7 b' __all__ = ['
     'FileType',
     'HelpFormatter',
     'RawDescriptionHelpFormatter',
-    'RawTextHelpFormatter'
+    'RawTextHelpFormatter',
     'ArgumentDefaultsHelpFormatter',
 ]
 
@@ -126,6 +118,10 b' except NameError:'
             result.reverse()
         return result
 
+
+def _callable(obj):
+    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')
+
 # silence Python 2.6 buggy warnings about Exception.message
 if _sys.version_info[:2] == (2, 6):
     import warnings
@@ -141,7 +137,8 b" SUPPRESS = '==SUPPRESS=='"
 OPTIONAL = '?'
 ZERO_OR_MORE = '*'
 ONE_OR_MORE = '+'
-PARSER = '==PARSER=='
+PARSER = 'A...'
+REMAINDER = '...'
 
 # =============================
 # Utility functions and classes
@@ -508,6 +505,8 b' class HelpFormatter(object):'
         return text
 
     def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
         text_width = self._width - self._current_indent
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
@@ -608,7 +607,9 b' class HelpFormatter(object):'
             result = '[%s [%s ...]]' % get_metavar(2)
         elif action.nargs == ONE_OR_MORE:
             result = '%s [%s ...]' % get_metavar(2)
-        elif action.nargs is PARSER:
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
             result = '%s ...' % get_metavar(1)
         else:
             formats = ['%s' for _ in range(action.nargs)]
@@ -724,6 +725,12 b' class ArgumentError(Exception):'
         return format % dict(message=self.message,
                              argument_name=self.argument_name)
 
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
 # ==============
 # Action classes
 # ==============
@@ -1018,6 +1025,7 b' class _VersionAction(Action):'
 
     def __init__(self,
                  option_strings,
+                 version=None,
                  dest=SUPPRESS,
                  default=SUPPRESS,
                  help=None):
@@ -1027,10 +1035,15 b' class _VersionAction(Action):'
             default=default,
             nargs=0,
             help=help)
+        self.version = version
 
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.print_version()
-        parser.exit()
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
 
 
 class _SubParsersAction(Action):
@@ -1156,8 +1169,7 b' class Namespace(_AttributeHolder):'
     """
 
     def __init__(self, **kwargs):
-        for name in kwargs:
-            setattr(self, name, kwargs[name])
+        self.__dict__.update(**kwargs)
 
     def __eq__(self, other):
         return vars(self) == vars(other)
@@ -1165,6 +1177,9 b' class Namespace(_AttributeHolder):'
     def __ne__(self, other):
         return not (self == other)
 
+    def __contains__(self, key):
+        return key in self.__dict__
+
 
 class _ActionsContainer(object):
 
@@ -1211,7 +1226,7 b' class _ActionsContainer(object):'
         self._defaults = {}
 
         # determines whether an "option" looks like a negative number
-        self._negative_number_matcher = _re.compile(r'^-\d+|-\d*.\d+$')
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
 
         # whether or not there are any optionals that look like negative
         # numbers -- uses a list so it can be shared and edited
@@ -1228,7 +1243,7 b' class _ActionsContainer(object):'
         return self._registries[registry_name].get(value, default)
 
     # ==================================
-    # Namespace default settings methods
+    # Namespace default accessor methods
     # ==================================
     def set_defaults(self, **kwargs):
         self._defaults.update(kwargs)
@@ -1239,6 +1254,13 b' class _ActionsContainer(object):'
             if action.dest in kwargs:
                 action.default = kwargs[action.dest]
 
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
     # =======================
     # Adding argument actions
     # =======================
@@ -1253,6 +1275,8 b' class _ActionsContainer(object):'
         # argument
         chars = self.prefix_chars
         if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
             kwargs = self._get_positional_kwargs(*args, **kwargs)
 
         # otherwise, we're adding an optional argument
@@ -1269,6 +1293,8 b' class _ActionsContainer(object):'
 
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
+        if not _callable(action_class):
+            raise ValueError('unknown action "%s"' % action_class)
         action = action_class(**kwargs)
         return self._add_action(action)
 
@@ -1578,6 +1604,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
         if self.version:
             self.add_argument(
                 '-v', '--version', action='version', default=SUPPRESS,
+                version=self.version,
                 help=_("show program's version number and exit"))
 
         # add parent arguments and defaults
@@ -2011,6 +2038,13 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
             action = self._option_string_actions[arg_string]
             return action, arg_string, None
 
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
         # search through all possible prefixes of the option string
         # and all actions in the parser for possible interpretations
         option_tuples = self._get_option_tuples(arg_string)
@@ -2108,8 +2142,12 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
         elif nargs == ONE_OR_MORE:
             nargs_pattern = '(-*A[A-]*)'
 
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
         # allow one argument followed by any number of options or arguments
-        elif nargs is PARSER:
+        elif nargs == PARSER:
             nargs_pattern = '(-*A[-AO]*)'
 
         # all others should be integers
@@ -2129,7 +2167,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # ========================
     def _get_values(self, action, arg_strings):
         # for everything but PARSER args, strip out '--'
-        if action.nargs is not PARSER:
+        if action.nargs not in [PARSER, REMAINDER]:
             arg_strings = [s for s in arg_strings if s != '--']
 
         # optional argument produces a default when not present
@@ -2158,8 +2196,12 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
             value = self._get_value(action, arg_string)
             self._check_value(action, value)
 
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
         # PARSER arguments convert all values, but check only the first
-        elif action.nargs is PARSER:
+        elif action.nargs == PARSER:
             value = [self._get_value(action, v) for v in arg_strings]
             self._check_value(action, value[0])
 
@@ -2174,16 +2216,21 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
 
     def _get_value(self, action, arg_string):
         type_func = self._registry_get('type', action.type, action.type)
-        if not hasattr(type_func, '__call__'):
-            if not hasattr(type_func, '__bases__'): # classic classes
-                msg = _('%r is not callable')
-                raise ArgumentError(action, msg % type_func)
+        if not _callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
 
         # convert the value to the appropriate type
         try:
             result = type_func(arg_string)
 
-        # TypeErrors or ValueErrors indicate errors
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
         except (TypeError, ValueError):
             name = getattr(action.type, '__name__', repr(action.type))
             msg = _('invalid %s value: %r')
@@ -2243,9 +2290,13 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # Help-printing methods
     # =====================
     def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
         self._print_message(self.format_usage(), file)
 
     def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
         self._print_message(self.format_help(), file)
 
     def print_version(self, file=None):
@@ -2262,7 +2313,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # ===============
     def exit(self, status=0, message=None):
         if message:
-            _sys.stderr.write(message)
+            self._print_message(message, _sys.stderr)
         _sys.exit(status)
 
     def error(self, message):

IPython/frontend/prefilterfrontend.py

@@ -9,7 +9,6 b' functionnality is abstracted out of ipython0 in reusable functions and'
 is added on the interpreter. This class can be a used to guide this
 refactoring.
 """
-__docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
@@ -27,7 +26,7 b' import os'
 import re
 import __builtin__
 
-from IPython.core.ipmaker import make_IPython
+from IPython.core.ipapp import IPythonApp
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
@@ -36,6 +35,9 b' from IPython.utils.genutils import Term'
 
 from linefrontendbase import LineFrontEndBase, common_prefix
 
+#-----------------------------------------------------------------------------
+# Utility functions
+#-----------------------------------------------------------------------------
 
 def mk_system_call(system_call_function, command):
     """ given a os.system replacement, and a leading string command,
@@ -74,13 +76,7 b' class PrefilterFrontEnd(LineFrontEndBase):'
               Used as the instance's argv value.  If not given, [] is used.
         """
         if argv is None:
-            argv = []
-        # This is a hack to avoid the IPython exception hook to trigger
-        # on exceptions (https://bugs.launchpad.net/bugs/337105)
-        # XXX: This is horrible: module-leve monkey patching -> side
-        # effects.
-        from IPython.core import iplib
-        iplib.InteractiveShell.isthreaded = True
+            argv = ['--no-banner']
 
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
@@ -101,12 +97,15 b' class PrefilterFrontEnd(LineFrontEndBase):'
                 return '\n'
             old_rawinput = __builtin__.raw_input
             __builtin__.raw_input = my_rawinput
-            # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=argv, 
-                                    user_ns=self.shell.user_ns,
-                                    user_global_ns=self.shell.user_global_ns)
+            ipython0 = IPythonApp(argv=argv, 
+                                  user_ns=self.shell.user_ns,
+                                  user_global_ns=self.shell.user_global_ns)
+            ipython0.initialize()
             __builtin__.raw_input = old_rawinput
-        self.ipython0 = ipython0
+        # XXX This will need to be updated as we refactor things, but for now,
+        # the .shell attribute of the ipythonapp instance conforms to the old
+        # api.
+        self.ipython0 = ipython0.shell
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))
@@ -125,7 +124,7 b' class PrefilterFrontEnd(LineFrontEndBase):'
 
 
         if not 'banner' in kwargs and self.banner is None:
-            self.banner = self.ipython0.BANNER
+            self.banner = self.ipython0.banner
 
         # FIXME: __init__ and start should be two different steps
         self.start()
@@ -202,8 +201,7 b' class PrefilterFrontEnd(LineFrontEndBase):'
         if completions:
             prefix = common_prefix(completions) 
             line = line[:-len(word)] + prefix
-        return line, completions 
- 
+        return line, completions
     
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface 
@@ -220,23 +218,11 b' class PrefilterFrontEnd(LineFrontEndBase):'
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
         
-        ## try:
-        ##     for line in input_string.split('\n'):
-        ##         filtered_lines.append(
-        ##                 self.ipython0.prefilter(line, False).rstrip())
-        ## except:
-        ##     # XXX: probably not the right thing to do.
-        ##     self.ipython0.showsyntaxerror()
-        ##     self.after_execute()
-        ## finally:
-        ##     self.release_output()
-
-
         try:
             try:
                 for line in input_string.split('\n'):
-                    filtered_lines.append(
-                            self.ipython0.prefilter(line, False).rstrip())
+                    pf = self.ipython0.prefilter_manager.prefilter_lines
+                    filtered_lines.append(pf(line, False).rstrip())
             except:
                 # XXX: probably not the right thing to do.
                 self.ipython0.showsyntaxerror()
@@ -244,13 +230,10 b' class PrefilterFrontEnd(LineFrontEndBase):'
         finally:
             self.release_output()
 
-
-
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string
 
-
     #--------------------------------------------------------------------------
     # PrefilterFrontEnd interface 
     #--------------------------------------------------------------------------
@@ -261,13 +244,11 b' class PrefilterFrontEnd(LineFrontEndBase):'
         """
         return os.system(command_string)
 
-
     def do_exit(self):
         """ Exit the shell, cleanup and save the history.
         """
         self.ipython0.atexit_operations()
 
-
     def _get_completion_text(self, line):
         """ Returns the text to be completed by breaking the line at specified
         delimiters.
@@ -281,4 +262,3 b' class PrefilterFrontEnd(LineFrontEndBase):'
         complete_sep = re.compile(expression)
         text = complete_sep.split(line)[-1]
         return text
-

IPython/frontend/tests/test_prefilterfrontend.py

@@ -21,8 +21,11 b' from nose.tools import assert_equal'
 
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 from IPython.core.ipapi import get as get_ipython0
-from IPython.testing.plugin.ipdoctest import default_argv
+from IPython.testing.tools import default_argv
 
+#-----------------------------------------------------------------------------
+# Support utilities
+#-----------------------------------------------------------------------------
 
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
@@ -85,14 +88,14 b' def isolate_ipython0(func):'
                 del user_ns[k]
             for k in new_globals:
                 del user_global_ns[k]
-        # Undo the hack at creation of PrefilterFrontEnd
-        from IPython.core import iplib
-        iplib.InteractiveShell.isthreaded = False
         return out
 
     my_func.__name__ = func.__name__
     return my_func
 
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
 
 @isolate_ipython0
 def test_execution():
@@ -166,7 +169,7 b' def test_magic():'
     f.input_buffer += '%who'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert_equal(out_value, 'Interactive namespace is empty.\n')
+    assert_equal(out_value, 'In\tOut\tget_ipython\t\n')
 
 
 @isolate_ipython0

IPython/frontend/wx/ipythonx.py

@@ -6,11 +6,10 b' ipython.'
 try:
     import wx
 except ImportError, e:
-    e.message = """%s
+    e.args[0] = """%s
 ________________________________________________________________________________
 You need wxPython to run this application.
-""" % e.message
-    e.args = (e.message, ) + e.args[1:]
+""" % e.args[0]
     raise e
 
 from wx_frontend import WxController

IPython/gui/wx/ipshell_nonblocking.py

@@ -23,13 +23,9 b' import os'
 import locale
 from thread_ex import ThreadEx
 
-try:
-    import IPython
-    from IPython.utils import genutils
-    from IPython.core import iplib
-except Exception,e:
-    print "Error importing IPython (%s)" % str(e)
-    raise Exception, e
+import IPython
+from IPython.core import iplib, ipapp
+from IPython.utils import genutils
 
 ##############################################################################
 class _Helper(object):
@@ -155,12 +151,17 b' class NonBlockingIPShell(object):'
 
         #Hack to save sys.displayhook, because ipython seems to overwrite it...
         self.sys_displayhook_ori = sys.displayhook
+
+        ipython0 = ipapp.IPythonApp(argv,user_ns=user_ns,
+                                    user_global_ns=user_global_ns)
+        ipython0.initialize()
+        self._IP = ipython0.shell
         
-        self._IP = IPython.shell.make_IPython(
-                                    argv,user_ns=user_ns,
-                                    user_global_ns=user_global_ns,
-                                    embedded=True,
-                                    shell_class=IPython.shell.InteractiveShell)
+        ## self._IP = IPython.shell.make_IPython(
+        ##                             argv,user_ns=user_ns,
+        ##                             user_global_ns=user_global_ns,
+        ##                             embedded=True,
+        ##                             shell_class=IPython.shell.InteractiveShell)
 
         #we save ipython0 displayhook and we restore sys.displayhook
         self.displayhook = sys.displayhook
@@ -273,7 +274,7 b' class NonBlockingIPShell(object):'
         @return: The banner string.
         @rtype: string
         """
-        return self._IP.BANNER
+        return self._IP.banner
     
     def get_prompt_count(self):
         """
@@ -470,7 +471,7 b' class NonBlockingIPShell(object):'
         '''
 
         orig_stdout = sys.stdout
-        sys.stdout = IPython.shell.Term.cout
+        sys.stdout = genutils.Term.cout
         #self.sys_displayhook_ori = sys.displayhook
         #sys.displayhook = self.displayhook
         

IPython/gui/wx/wxIPython.py

@@ -109,7 +109,7 b' class MyFrame(wx.Frame):'
 
     def optionSave(self, name, value):
         ip = get()
-        path = ip.config.IPYTHONDIR
+        path = ip.ipython_dir
         opt = open(path + '/options.conf','w')
 
         try:
@@ -126,7 +126,7 b' class MyFrame(wx.Frame):'
     def optionLoad(self):
         try:
             ip = get()
-            path = ip.config.IPYTHONDIR
+            path = ip.ipython_dir
             opt = open(path + '/options.conf','r')
             lines = opt.readlines()
             opt.close()

IPython/kernel/asyncclient.py

@@ -1,3 +1,4 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """Asynchronous clients for the IPython controller.
@@ -9,32 +10,32 b' deferreds to the result.'
 
 The main methods are are `get_*_client` and `get_client`.
 """
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from IPython.kernel import codeutil
-from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.clientconnector import (
+    AsyncClientConnector,
+    AsyncCluster
+)
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask
 from IPython.kernel.error import CompositeError
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Code
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-_client_tub = ClientConnector()
+_client_tub = AsyncClientConnector()
 get_multiengine_client = _client_tub.get_multiengine_client
 get_task_client = _client_tub.get_task_client
 get_client = _client_tub.get_client

IPython/kernel/client.py

@@ -1,3 +1,4 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """This module contains blocking clients for the controller interfaces.
@@ -15,33 +16,36 b' The main classes in this module are:'
     * CompositeError
 """
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
+from cStringIO import StringIO
 import sys
+import warnings
 
 # from IPython.utils import growl
 # growl.start("IPython1 Client")
 
 
 from twisted.internet import reactor
-from IPython.kernel.clientconnector import ClientConnector
+from twisted.internet.error import PotentialZombieWarning
+from twisted.python import log
+
+from IPython.kernel.clientconnector import ClientConnector, Cluster
 from IPython.kernel.twistedutil import ReactorInThread
 from IPython.kernel.twistedutil import blockingCallFromThread
 
 # These enable various things 
 from IPython.kernel import codeutil
-import IPython.kernel.magic
+# import IPython.kernel.magic
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask
@@ -51,46 +55,34 b' from IPython.kernel.error import CompositeError'
 # Code
 #-------------------------------------------------------------------------------
 
-_client_tub = ClientConnector()
-
-
-def get_multiengine_client(furl_or_file=''):
-    """Get the blocking MultiEngine client.
-    
-    :Parameters:
-        furl_or_file : str
-            A furl or a filename containing a furl.  If empty, the
-            default furl_file will be used
-            
-    :Returns:
-        The connected MultiEngineClient instance
-    """
-    client = blockingCallFromThread(_client_tub.get_multiengine_client, 
-        furl_or_file)
-    return client.adapt_to_blocking_client()
-
-def get_task_client(furl_or_file=''):
-    """Get the blocking Task client.
-    
-    :Parameters:
-        furl_or_file : str
-            A furl or a filename containing a furl.  If empty, the
-            default furl_file will be used
-            
-    :Returns:
-        The connected TaskClient instance
-    """
-    client = blockingCallFromThread(_client_tub.get_task_client, 
-        furl_or_file)
-    return client.adapt_to_blocking_client()
+warnings.simplefilter('ignore', PotentialZombieWarning)
 
+_client_tub = ClientConnector()
 
+get_multiengine_client = _client_tub.get_multiengine_client
+get_task_client = _client_tub.get_task_client
 MultiEngineClient = get_multiengine_client
 TaskClient = get_task_client
 
-
+# This isn't great.  I should probably set this up in the ReactorInThread
+# class below.  But, it does work for now.
+log.startLogging(sys.stdout, setStdout=0)
 
 # Now we start the reactor in a thread
 rit = ReactorInThread()
 rit.setDaemon(True)
-rit.start()
  No newline at end of file
+rit.start()
+
+
+
+
+__all__ = [
+    'MapTask',
+    'StringTask',
+    'MultiEngineClient',
+    'TaskClient',
+    'CompositeError',
+    'get_task_client',
+    'get_multiengine_client',
+    'Cluster'
+]

IPython/kernel/clientconnector.py

@@ -1,142 +1,268 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
-"""A class for handling client connections to the controller."""
+"""Facilities for handling client connections to the controller."""
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
-
-from twisted.internet import defer
+#-----------------------------------------------------------------------------
 
-from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from __future__ import with_statement
+import os
 
-from IPython.kernel.config import config_manager as kernel_config_manager
+from IPython.kernel.fcutil import (
+    Tub,
+    find_furl,
+    is_valid_furl_or_file,
+    validate_furl_or_file,
+    FURLError
+)
+from IPython.kernel.clusterdir import ClusterDir, ClusterDirError
+from IPython.kernel.launcher import IPClusterLauncher
+from IPython.kernel.twistedutil import (
+    gatherBoth, 
+    make_deferred,
+    blockingCallFromThread, 
+    sleep_deferred
+)
 from IPython.utils.importstring import import_item
-from IPython.kernel.fcutil import find_furl
+from IPython.utils.genutils import get_ipython_dir
 
-co = kernel_config_manager.get_config_obj()
-client_co = co['client']
+from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python import failure, log
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The ClientConnector class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-class ClientConnector(object):
-    """
-    This class gets remote references from furls and returns the wrapped clients.
-    
-    This class is also used in `client.py` and `asyncclient.py` to create 
-    a single per client-process Tub.
+DELAY = 0.2
+MAX_TRIES = 9
+
+
+class ClientConnectorError(Exception):
+    pass
+
+
+class AsyncClientConnector(object):
+    """A class for getting remote references and clients from furls.
+
+    This start a single :class:`Tub` for all remote reference and caches
+    references.
     """
-    
+
     def __init__(self):
         self._remote_refs = {}
         self.tub = Tub()
         self.tub.startService()
-    
-    def get_reference(self, furl_or_file):
+
+    def _find_furl(self, profile='default', cluster_dir=None, 
+                   furl_or_file=None, furl_file_name=None,
+                   ipython_dir=None):
+        """Find a FURL file by profile+ipython_dir or cluster dir.
+
+        This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception 
+        if a FURL file can't be found.
         """
-        Get a remote reference using a furl or a file containing a furl.
-        
+        # Try by furl_or_file
+        if furl_or_file is not None:
+            validate_furl_or_file(furl_or_file)
+            return furl_or_file
+
+        if furl_file_name is None:
+            raise FURLError('A furl_file_name must be provided')
+
+        # Try by cluster_dir
+        if cluster_dir is not None:
+            cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+            sdir = cluster_dir_obj.security_dir
+            furl_file = os.path.join(sdir, furl_file_name)
+            validate_furl_or_file(furl_file)
+            return furl_file
+
+        # Try by profile
+        if ipython_dir is None:
+            ipython_dir = get_ipython_dir()
+        if profile is not None:
+            cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                ipython_dir, profile)
+            sdir = cluster_dir_obj.security_dir
+            furl_file = os.path.join(sdir, furl_file_name)
+            validate_furl_or_file(furl_file)
+            return furl_file
+
+        raise FURLError('Could not find a valid FURL file.')
+
+    def get_reference(self, furl_or_file):
+        """Get a remote reference using a furl or a file containing a furl.
+
         Remote references are cached locally so once a remote reference
         has been retrieved for a given furl, the cached version is 
         returned.
-        
-        :Parameters:
-            furl_or_file : str
-                A furl or a filename containing a furl
-        
-        :Returns:
-            A deferred to a remote reference
+
+        Parameters
+        ----------
+        furl_or_file : str
+            A furl or a filename containing a furl. This should already be
+            validated, but might not yet exist.
+
+        Returns
+        -------
+        A deferred to a remote reference
         """
-        furl = find_furl(furl_or_file)
+        furl = furl_or_file
         if furl in self._remote_refs:
             d = defer.succeed(self._remote_refs[furl])
         else:
             d = self.tub.getReference(furl)
-            d.addCallback(self.save_ref, furl)
+            d.addCallback(self._save_ref, furl)
         return d
         
-    def save_ref(self, ref, furl):
-        """
-        Cache a remote reference by its furl.
-        """
+    def _save_ref(self, ref, furl):
+        """Cache a remote reference by its furl."""
         self._remote_refs[furl] = ref
         return ref
         
-    def get_task_client(self, furl_or_file=''):
-        """
-        Get the task controller client.
+    def get_task_client(self, profile='default', cluster_dir=None,
+                        furl_or_file=None, ipython_dir=None,
+                        delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task controller client.
         
-        This method is a simple wrapper around `get_client` that allow
-        `furl_or_file` to be empty, in which case, the furls is taken
-        from the default furl file given in the configuration.
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Parameters:
-            furl_or_file : str
-                A furl or a filename containing a furl.  If empty, the
-                default furl_file will be used
-                
-        :Returns:
-            A deferred to the actual client class
-        """
-        task_co = client_co['client_interfaces']['task']
-        if furl_or_file:
-            ff = furl_or_file
-        else:
-            ff = task_co['furl_file']
-        return self.get_client(ff)
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
 
-    def get_multiengine_client(self, furl_or_file=''):
+        Returns
+        -------
+        A deferred to the actual client class.
         """
-        Get the multiengine controller client.
+        return self.get_client(
+            profile, cluster_dir, furl_or_file, 
+            'ipcontroller-tc.furl', ipython_dir,
+            delay, max_tries
+        )
+
+    def get_multiengine_client(self, profile='default', cluster_dir=None,
+                               furl_or_file=None, ipython_dir=None,
+                               delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine controller client.
         
-        This method is a simple wrapper around `get_client` that allow
-        `furl_or_file` to be empty, in which case, the furls is taken
-        from the default furl file given in the configuration.
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Parameters:
-            furl_or_file : str
-                A furl or a filename containing a furl.  If empty, the
-                default furl_file will be used
-                
-        :Returns:
-            A deferred to the actual client class
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        A deferred to the actual client class.
         """
-        task_co = client_co['client_interfaces']['multiengine']
-        if furl_or_file:
-            ff = furl_or_file
-        else:
-            ff = task_co['furl_file']
-        return self.get_client(ff)
+        return self.get_client(
+            profile, cluster_dir, furl_or_file, 
+            'ipcontroller-mec.furl', ipython_dir,
+            delay, max_tries
+        )
     
-    def get_client(self, furl_or_file):
-        """
-        Get a remote reference and wrap it in a client by furl.
-        
-        This method first gets a remote reference and then calls its 
-        `get_client_name` method to find the apprpriate client class
-        that should be used to wrap the remote reference.
-        
-        :Parameters:
-            furl_or_file : str
-                A furl or a filename containing a furl
+    def get_client(self, profile='default', cluster_dir=None,
+                   furl_or_file=None, furl_file_name=None, ipython_dir=None,
+                   delay=DELAY, max_tries=MAX_TRIES):
+        """Get a remote reference and wrap it in a client by furl.
+
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Returns:
-            A deferred to the actual client class
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURL. This is useful if you 
+            simply know the location of the FURL file.
+        furl_file_name : str
+            The filename (not the full path) of the FURL. This must be
+            provided if ``furl_or_file`` is not.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        A deferred to the actual client class. Or a failure to a 
+        :exc:`FURLError`.
         """
-        furl = find_furl(furl_or_file)
-        d = self.get_reference(furl)
-        def wrap_remote_reference(rr):
+        try:
+            furl_file = self._find_furl(
+                profile, cluster_dir, furl_or_file,
+                furl_file_name, ipython_dir
+            )
+        except FURLError:
+            return defer.fail(failure.Failure())
+
+        def _wrap_remote_reference(rr):
             d = rr.callRemote('get_client_name')
             d.addCallback(lambda name: import_item(name))
             def adapt(client_interface):
@@ -146,5 +272,502 b' class ClientConnector(object):'
             d.addCallback(adapt)
 
             return d
-        d.addCallback(wrap_remote_reference)
+
+        d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
+        d.addCallback(_wrap_remote_reference)
+        d.addErrback(self._handle_error, furl_file)
+        return d
+
+    def _handle_error(self, f, furl_file):
+        raise ClientConnectorError('Could not connect to the controller '
+            'using the FURL file. This usually means that i) the controller '
+            'was not started or ii) a firewall was blocking the client from '
+            'connecting to the controller: %s' % furl_file)
+
+    @inlineCallbacks
+    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
+        """Try to connect to the controller with retry logic."""
+        if attempt < max_tries:
+            log.msg("Connecting [%r]" % attempt)
+            try:
+                self.furl = find_furl(furl_or_file)
+                # Uncomment this to see the FURL being tried.
+                # log.msg("FURL: %s" % self.furl)
+                rr = yield self.get_reference(self.furl)
+                log.msg("Connected: %s" % furl_or_file)
+            except:
+                if attempt==max_tries-1:
+                    # This will propagate the exception all the way to the top
+                    # where it can be handled.
+                    raise
+                else:
+                    yield sleep_deferred(delay)
+                    rr = yield self._try_to_connect(
+                        furl_or_file, 1.5*delay, max_tries, attempt+1
+                    )
+                    returnValue(rr)
+            else:
+                returnValue(rr)
+        else:
+            raise ClientConnectorError(
+                'Could not connect to controller, max_tries (%r) exceeded. '
+                'This usually means that i) the controller was not started, '
+                'or ii) a firewall was blocking the client from connecting '
+                'to the controller.' % max_tries
+            )
+
+
+class ClientConnector(object):
+    """A blocking version of a client connector.
+
+    This class creates a single :class:`Tub` instance and allows remote
+    references and client to be retrieved by their FURLs.  Remote references
+    are cached locally and FURL files can be found using profiles and cluster
+    directories.
+    """
+
+    def __init__(self):
+        self.async_cc = AsyncClientConnector()
+
+    def get_task_client(self, profile='default', cluster_dir=None,
+                        furl_or_file=None, ipython_dir=None,
+                        delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client.
+        
+        Usually only the ``profile`` option will be needed. If a FURL file
+        can't be found by its profile, use ``cluster_dir`` or
+        ``furl_or_file``.
+        
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        The task client instance.
+        """
+        client = blockingCallFromThread(
+            self.async_cc.get_task_client, profile, cluster_dir,
+            furl_or_file, ipython_dir, delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+    def get_multiengine_client(self, profile='default', cluster_dir=None,
+                               furl_or_file=None, ipython_dir=None,
+                               delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client.
+        
+        Usually only the ``profile`` option will be needed. If a FURL file
+        can't be found by its profile, use ``cluster_dir`` or
+        ``furl_or_file``.
+        
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        The multiengine client instance.
+        """
+        client = blockingCallFromThread(
+            self.async_cc.get_multiengine_client, profile, cluster_dir,
+            furl_or_file, ipython_dir, delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+    def get_client(self, profile='default', cluster_dir=None,
+                   furl_or_file=None, ipython_dir=None,
+                   delay=DELAY, max_tries=MAX_TRIES):
+        client = blockingCallFromThread(
+            self.async_cc.get_client, profile, cluster_dir,
+            furl_or_file, ipython_dir,
+            delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+
+class ClusterStateError(Exception):
+    pass
+
+
+class AsyncCluster(object):
+    """An class that wraps the :command:`ipcluster` script."""
+
+    def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
+                 auto_create=False, auto_stop=True):
+        """Create a class to manage an IPython cluster.
+
+        This class calls the :command:`ipcluster` command with the right
+        options to start an IPython cluster.  Typically a cluster directory
+        must be created (:command:`ipcluster create`) and configured before 
+        using this class. Configuration is done by editing the 
+        configuration files in the top level of the cluster directory.
+
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        auto_create : bool
+            Automatically create the cluster directory it is dones't exist.
+            This will usually only make sense if using a local cluster 
+            (default=False).
+        auto_stop : bool
+            Automatically stop the cluster when this instance is garbage 
+            collected (default=True).  This is useful if you want the cluster
+            to live beyond your current process. There is also an instance
+            attribute ``auto_stop`` to change this behavior.
+        """
+        self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create)
+        self.state = 'before'
+        self.launcher = None
+        self.client_connector = None
+        self.auto_stop = auto_stop
+
+    def __del__(self):
+        if self.auto_stop and self.state=='running':
+            print "Auto stopping the cluster..."
+            self.stop()
+
+    @property
+    def location(self):
+        if hasattr(self, 'cluster_dir_obj'):
+            return self.cluster_dir_obj.location
+        else:
+            return ''
+
+    @property
+    def running(self):
+        if self.state=='running':
+            return True
+        else:
+            return False
+
+    def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create):
+        if ipython_dir is None:
+            ipython_dir = get_ipython_dir()
+        if cluster_dir is not None:
+            try:
+                self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+            except ClusterDirError:
+                pass
+        if profile is not None:
+            try:
+                self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                    ipython_dir, profile)
+            except ClusterDirError:
+                pass
+        if auto_create or profile=='default':
+            # This should call 'ipcluster create --profile default
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                ipython_dir, profile)
+        else:
+            raise ClusterDirError('Cluster dir not found.')
+
+    @make_deferred
+    def start(self, n=2):
+        """Start the IPython cluster with n engines.
+
+        Parameters
+        ----------
+        n : int
+            The number of engine to start.
+        """
+        # We might want to add logic to test if the cluster has started
+        # by another process....
+        if not self.state=='running':
+            self.launcher = IPClusterLauncher(os.getcwd())
+            self.launcher.ipcluster_n = n
+            self.launcher.ipcluster_subcommand = 'start'
+            d = self.launcher.start()
+            d.addCallback(self._handle_start)
+            return d
+        else:
+            raise ClusterStateError('Cluster is already running')
+
+    @make_deferred
+    def stop(self):
+        """Stop the IPython cluster if it is running."""
+        if self.state=='running':
+            d1 = self.launcher.observe_stop()
+            d1.addCallback(self._handle_stop)
+            d2 = self.launcher.stop()
+            return gatherBoth([d1, d2], consumeErrors=True)
+        else:
+            raise ClusterStateError("Cluster not running")
+
+    def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client for the running cluster.
+
+        If this fails, it means that the cluster has not finished starting.
+        Usually waiting a few seconds are re-trying will solve this.    
+        """
+        if self.client_connector is None:
+            self.client_connector = AsyncClientConnector()
+        return self.client_connector.get_multiengine_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client for the running cluster.
+
+        If this fails, it means that the cluster has not finished starting.
+        Usually waiting a few seconds are re-trying will solve this.    
+        """
+        if self.client_connector is None:
+            self.client_connector = AsyncClientConnector()
+        return self.client_connector.get_task_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_ipengine_logs(self):
+        return self.get_logs_by_name('ipengine')
+
+    def get_ipcontroller_logs(self):
+        return self.get_logs_by_name('ipcontroller')
+
+    def get_ipcluster_logs(self):
+        return self.get_logs_by_name('ipcluster')
+
+    def get_logs_by_name(self, name='ipcluster'):
+        log_dir = self.cluster_dir_obj.log_dir
+        logs = {}
+        for log in os.listdir(log_dir):
+            if log.startswith(name + '-') and log.endswith('.log'):
+                with open(os.path.join(log_dir, log), 'r') as f:
+                    logs[log] = f.read()
+        return logs
+
+    def get_logs(self):
+        d = self.get_ipcluster_logs()
+        d.update(self.get_ipengine_logs())
+        d.update(self.get_ipcontroller_logs())
         return d
+
+    def _handle_start(self, r):
+        self.state = 'running'
+
+    def _handle_stop(self, r):
+        self.state = 'after'
+
+
+class Cluster(object):
+
+
+    def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
+                 auto_create=False, auto_stop=True):
+        """Create a class to manage an IPython cluster.
+
+        This class calls the :command:`ipcluster` command with the right
+        options to start an IPython cluster.  Typically a cluster directory
+        must be created (:command:`ipcluster create`) and configured before 
+        using this class. Configuration is done by editing the 
+        configuration files in the top level of the cluster directory.
+
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        auto_create : bool
+            Automatically create the cluster directory it is dones't exist.
+            This will usually only make sense if using a local cluster 
+            (default=False).
+        auto_stop : bool
+            Automatically stop the cluster when this instance is garbage 
+            collected (default=True).  This is useful if you want the cluster
+            to live beyond your current process. There is also an instance
+            attribute ``auto_stop`` to change this behavior.
+        """
+        self.async_cluster = AsyncCluster(
+            profile, cluster_dir, ipython_dir, auto_create, auto_stop
+        )
+        self.cluster_dir_obj = self.async_cluster.cluster_dir_obj
+        self.client_connector = None
+
+    def _set_auto_stop(self, value):
+        self.async_cluster.auto_stop = value
+
+    def _get_auto_stop(self):
+        return self.async_cluster.auto_stop
+
+    auto_stop = property(_get_auto_stop, _set_auto_stop)
+
+    @property
+    def location(self):
+        return self.async_cluster.location
+
+    @property
+    def running(self):
+        return self.async_cluster.running
+
+    def start(self, n=2):
+        """Start the IPython cluster with n engines.
+
+        Parameters
+        ----------
+        n : int
+            The number of engine to start.
+        """
+        return blockingCallFromThread(self.async_cluster.start, n)
+
+    def stop(self):
+        """Stop the IPython cluster if it is running."""
+        return blockingCallFromThread(self.async_cluster.stop)
+
+    def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client for the running cluster.
+
+        This will try to attempt to the controller multiple times. If this 
+        fails altogether, try looking at the following:
+        * Make sure the controller is starting properly by looking at its
+          log files.
+        * Make sure the controller is writing its FURL file in the location
+          expected by the client.
+        * Make sure a firewall on the controller's host is not blocking the
+          client from connecting.
+
+        Parameters
+        ----------
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+        """
+        if self.client_connector is None:
+            self.client_connector = ClientConnector()
+        return self.client_connector.get_multiengine_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client for the running cluster.
+
+        This will try to attempt to the controller multiple times. If this 
+        fails altogether, try looking at the following:
+        * Make sure the controller is starting properly by looking at its
+          log files.
+        * Make sure the controller is writing its FURL file in the location
+          expected by the client.
+        * Make sure a firewall on the controller's host is not blocking the
+          client from connecting.
+
+        Parameters
+        ----------
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+        """
+        if self.client_connector is None:
+            self.client_connector = ClientConnector()
+        return self.client_connector.get_task_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def __repr__(self):
+        s = "<Cluster(running=%r, location=%s)" % (self.running, self.location)
+        return s
+
+    def get_logs_by_name(self, name='ipcluter'):
+        """Get a dict of logs by process name (ipcluster, ipengine, etc.)"""
+        return self.async_cluster.get_logs_by_name(name)
+
+    def get_ipengine_logs(self):
+        """Get a dict of logs for all engines in this cluster."""
+        return self.async_cluster.get_ipengine_logs()
+
+    def get_ipcontroller_logs(self):
+        """Get a dict of logs for the controller in this cluster."""
+        return self.async_cluster.get_ipcontroller_logs()
+
+    def get_ipcluster_logs(self):
+        """Get a dict of the ipcluster logs for this cluster."""
+        return self.async_cluster.get_ipcluster_logs()
+
+    def get_logs(self):
+        """Get a dict of all logs for this cluster."""
+        return self.async_cluster.get_logs()
+
+    def _print_logs(self, logs):
+        for k, v in logs.iteritems():
+            print "==================================="
+            print "Logfile: %s" % k
+            print "==================================="
+            print v
+            print
+
+    def print_ipengine_logs(self):
+        """Print the ipengine logs for this cluster to stdout."""
+        self._print_logs(self.get_ipengine_logs())
+
+    def print_ipcontroller_logs(self):
+        """Print the ipcontroller logs for this cluster to stdout."""
+        self._print_logs(self.get_ipcontroller_logs())
+
+    def print_ipcluster_logs(self):
+        """Print the ipcluster logs for this cluster to stdout."""
+        self._print_logs(self.get_ipcluster_logs())
+
+    def print_logs(self):
+        """Print all the logs for this cluster to stdout."""
+        self._print_logs(self.get_logs())
+

IPython/kernel/core/interpreter.py

@@ -211,7 +211,7 b' class Interpreter(object):'
 
     #### Public 'Interpreter' interface ########################################
 
-    def formatTraceback(self, et, ev, tb, message=''):
+    def format_traceback(self, et, ev, tb, message=''):
         """Put a formatted version of the traceback into value and reraise.
         
         When exceptions have to be sent over the network, the traceback 
@@ -375,7 +375,6 b' class Interpreter(object):'
             exec code in self.user_ns
             outflag = 0
         except SystemExit:
-            self.resetbuffer()
             self.traceback_trap.args = sys.exc_info()
         except:
             self.traceback_trap.args = sys.exc_info()
@@ -395,7 +394,7 b' class Interpreter(object):'
             python = self.translator(python)
         self.execute_python(python)
 
-    def getCommand(self, i=None):
+    def get_command(self, i=None):
         """Gets the ith message in the message_cache.
         
         This is implemented here for compatibility with the old ipython1 shell
@@ -492,7 +491,7 b' class Interpreter(object):'
             # somehow.  In the meantime, we'll just stop if there are two lines
             # of pure whitespace at the end.
             last_two = source.rsplit('\n',2)[-2:]
-            print 'last two:',last_two  # dbg
+            #print 'last two:',last_two  # dbg
             if len(last_two)==2 and all(s.isspace() for s in last_two):
                 return COMPLETE_INPUT,False
             else:

IPython/kernel/core/tests/test_redirectors.py

@@ -18,19 +18,19 b' __test__ = {}'
 
 from cStringIO import StringIO
 import os
+import sys
 
 from twisted.trial import unittest
 
-from IPython.testing import decorators_trial as dec
-
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class TestRedirector(unittest.TestCase):
 
-    @dec.skip_win32
+    if sys.platform == 'win32':
+        skip = True
+    
     def test_redirector(self):
         """Checks that the redirector can be used to do synchronous capture.
         """
@@ -51,7 +51,6 b' class TestRedirector(unittest.TestCase):'
         result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
         self.assertEquals(result1, result2)
 
-    @dec.skip_win32
     def test_redirector_output_trap(self):
         """Check the greedy trapping behavior of the traps.
         
@@ -59,7 +58,8 b' class TestRedirector(unittest.TestCase):'
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
         """
-        from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
+        from IPython.kernel.core.redirector_output_trap import \
+            RedirectorOutputTrap
         out = StringIO()
         trap = RedirectorOutputTrap(out.write, out.write)
         try:

IPython/kernel/engineconnector.py

@@ -1,32 +1,38 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """A class that manages the engines connection to the controller."""
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 import os
 import cPickle as pickle
 
 from twisted.python import log, failure
 from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
 
-from IPython.kernel.fcutil import find_furl
+from IPython.kernel.fcutil import find_furl, validate_furl_or_file
 from IPython.kernel.enginefc import IFCEngine
+from IPython.kernel.twistedutil import sleep_deferred, make_deferred
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The ClientConnector class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+
+class EngineConnectorError(Exception):
+    pass
+
 
 class EngineConnector(object):
     """Manage an engines connection to a controller.
@@ -38,8 +44,10 b' class EngineConnector(object):'
     
     def __init__(self, tub):
         self.tub = tub
-        
-    def connect_to_controller(self, engine_service, furl_or_file):
+
+    @make_deferred
+    def connect_to_controller(self, engine_service, furl_or_file,
+                              delay=0.1, max_tries=10):
         """
         Make a connection to a controller specified by a furl.
         
@@ -48,34 +56,73 b' class EngineConnector(object):'
         foolscap URL contains all the information needed to connect to the 
         controller, including the ip and port as well as any encryption and 
         authentication information needed for the connection.
-        
+
         After getting a reference to the controller, this method calls the 
         `register_engine` method of the controller to actually register the 
         engine.
-        
-        :Parameters:
-            engine_service : IEngineBase
-                An instance of an `IEngineBase` implementer
-            furl_or_file : str
-                A furl or a filename containing a furl
+
+        This method will try to connect to the controller multiple times with
+        a delay in between.  Each time the FURL file is read anew.
+
+        Parameters
+        __________
+        engine_service : IEngineBase
+            An instance of an `IEngineBase` implementer
+        furl_or_file : str
+            A furl or a filename containing a furl
+        delay : float
+            The intial time to wait between connection attempts.  Subsequent
+            attempts have increasing delays.
+        max_tries : int
+            The maximum number of connection attempts.
+
+        Returns
+        -------
+        A deferred to the registered client or a failure to an error
+        like :exc:`FURLError`.
         """
         if not self.tub.running:
             self.tub.startService()
         self.engine_service = engine_service
         self.engine_reference = IFCEngine(self.engine_service)
-        try:
-            self.furl = find_furl(furl_or_file)
-        except ValueError:
-            return defer.fail(failure.Failure())
+
+        validate_furl_or_file(furl_or_file)
+        d = self._try_to_connect(furl_or_file, delay, max_tries, attempt=0)
+        d.addCallback(self._register)
+        return d
+
+    @inlineCallbacks
+    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
+        """Try to connect to the controller with retry logic."""
+        if attempt < max_tries:
+            log.msg("Attempting to connect to controller [%r]: %s" % \
+                (attempt, furl_or_file))
+            try:
+                self.furl = find_furl(furl_or_file)
+                # Uncomment this to see the FURL being tried.
+                # log.msg("FURL: %s" % self.furl)
+                rr = yield self.tub.getReference(self.furl)
+            except:
+                if attempt==max_tries-1:
+                    # This will propagate the exception all the way to the top
+                    # where it can be handled.
+                    raise
+                else:
+                    yield sleep_deferred(delay)
+                    rr = yield self._try_to_connect(
+                        furl_or_file, 1.5*delay, max_tries, attempt+1
+                    )
+                    # rr becomes an int when there is a connection!!!
+                    returnValue(rr)
+            else:
+                returnValue(rr)
         else:
-            d = self.tub.getReference(self.furl)
-            d.addCallbacks(self._register, self._log_failure)
-            return d
-    
-    def _log_failure(self, reason):
-        log.err('EngineConnector: engine registration failed:')
-        log.err(reason)
-        return reason
+            raise EngineConnectorError(
+                'Could not connect to controller, max_tries (%r) exceeded. '
+                'This usually means that i) the controller was not started, '
+                'or ii) a firewall was blocking the engine from connecting '
+                'to the controller.' % max_tries
+            )
 
     def _register(self, rr):
         self.remote_ref = rr
@@ -83,7 +130,7 b' class EngineConnector(object):'
         desired_id = self.engine_service.id
         d = self.remote_ref.callRemote('register_engine', self.engine_reference, 
             desired_id, os.getpid(), pickle.dumps(self.engine_service.properties,2))
-        return d.addCallbacks(self._reference_sent, self._log_failure)
+        return d.addCallback(self._reference_sent)
 
     def _reference_sent(self, registration_dict):
         self.engine_service.id = registration_dict['id']

IPython/kernel/engineservice.py

@@ -387,7 +387,7 b' class EngineService(object, service.Service):'
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
-            et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
+            et,ev,tb = self.shell.format_traceback(et,ev,tb,msg)
             # Add another attribute
             ev._ipython_engine_info = msg
             f = failure.Failure(ev,et,None)
@@ -444,7 +444,7 b' class EngineService(object, service.Service):'
         msg = {'engineid':self.id,
                'method':'get_result',
                'args':[repr(i)]}
-        d = self.executeAndRaise(msg, self.shell.getCommand, i)
+        d = self.executeAndRaise(msg, self.shell.get_command, i)
         d.addCallback(self.addIDToResult)
         return d
     
@@ -877,7 +877,7 b' class ThreadedEngineService(EngineService):'
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
-            et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
+            et,ev,tb = self.shell.format_traceback(et,ev,tb,msg)
             # Add another attribute
             
             # Create a new exception with the new attributes

IPython/kernel/error.py

@@ -127,9 +127,11 b' class TaskRejectError(KernelError):'
 class CompositeError(KernelError):
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))
-        self.message = message
+        # Don't use pack_exception because it will conflict with the .message
+        # attribute that is being deprecated in 2.6 and beyond.
+        self.msg = message
         self.elist = elist
-  
+
     def _get_engine_str(self, ev):
         try:
             ei = ev._ipython_engine_info
@@ -137,7 +139,7 b' class CompositeError(KernelError):'
             return '[Engine Exception]'
         else:
             return '[%i:%s]: ' % (ei['engineid'], ei['method'])
-    
+
     def _get_traceback(self, ev):
         try:
             tb = ev._ipython_traceback_text
@@ -145,14 +147,14 b' class CompositeError(KernelError):'
             return 'No traceback available'
         else:
             return tb
-  
+
     def __str__(self):
-        s = str(self.message)
+        s = str(self.msg)
         for et, ev, etb in self.elist:
             engine_str = self._get_engine_str(ev)
             s = s + '\n' + engine_str + str(et.__name__) + ': ' + str(ev)
         return s
-    
+
     def print_tracebacks(self, excid=None):
         if excid is None:
             for (et,ev,etb) in self.elist:

IPython/kernel/fcutil.py

@@ -1,27 +1,62 b''
+#!/usr/bin/env python
 # encoding: utf-8
+"""
+Foolscap related utilities.
+"""
 
-"""Foolscap related utilities."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
 
 import os
+import tempfile
+
+from twisted.internet import reactor, defer
+from twisted.python import log
 
 from foolscap import Tub, UnauthenticatedTub
 
+from IPython.config.loader import Config
+
+from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
+
+from IPython.kernel.error import SecurityError
+
+from IPython.utils.traitlets import Int, Str, Bool, Instance
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+# We do this so if a user doesn't have OpenSSL installed, it will try to use
+# an UnauthenticatedTub.  But, they will still run into problems if they
+# try to use encrypted furls.
+try:
+    import OpenSSL
+except:
+    Tub = UnauthenticatedTub
+    have_crypto = False
+else:
+    have_crypto = True
+
+
+class FURLError(Exception):
+    pass
+
+
 def check_furl_file_security(furl_file, secure):
     """Remove the old furl_file if changing security modes."""
-    
     if os.path.isfile(furl_file):
         f = open(furl_file, 'r')
         oldfurl = f.read().strip()
@@ -29,41 +64,210 b' def check_furl_file_security(furl_file, secure):'
         if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
             os.remove(furl_file)
 
+
 def is_secure(furl):
+    """Is the given FURL secure or not."""
     if is_valid(furl):
         if furl.startswith("pb://"):
             return True
         elif furl.startswith("pbu://"):
             return False
     else:
-        raise ValueError("invalid furl: %s" % furl)
+        raise FURLError("invalid FURL: %s" % furl)
+
 
 def is_valid(furl):
+    """Is the str a valid FURL or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
     else:
         return False
 
+
 def find_furl(furl_or_file):
+    """Find, validate and return a FURL in a string or file."""
     if isinstance(furl_or_file, str):
         if is_valid(furl_or_file):
             return furl_or_file
     if os.path.isfile(furl_or_file):
-        furl = open(furl_or_file, 'r').read().strip()
+        with open(furl_or_file, 'r') as f:
+            furl = f.read().strip()
         if is_valid(furl):
             return furl
-    raise ValueError("not a furl or a file containing a furl: %s" % furl_or_file)
+    raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
 
-# We do this so if a user doesn't have OpenSSL installed, it will try to use
-# an UnauthenticatedTub.  But, they will still run into problems if they
-# try to use encrypted furls.
-try:
-    import OpenSSL
-except:
-    Tub = UnauthenticatedTub
-    have_crypto = False
-else:
-    have_crypto = True
 
+def is_valid_furl_or_file(furl_or_file):
+    """Validate a FURL or a FURL file.
+
+    If ``furl_or_file`` looks like a file, we simply make sure its directory
+    exists and that it has a ``.furl`` file extension.  We don't try to see
+    if the FURL file exists or to read its contents. This is useful for
+    cases where auto re-connection is being used.
+    """
+    if isinstance(furl_or_file, str):
+        if is_valid(furl_or_file):
+            return True
+    if isinstance(furl_or_file, (str, unicode)):
+        path, furl_filename = os.path.split(furl_or_file)
+        if os.path.isdir(path) and furl_filename.endswith('.furl'):
+            return True
+    return False
+
+
+def validate_furl_or_file(furl_or_file):
+    if not is_valid_furl_or_file(furl_or_file):
+        raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
+
+
+def get_temp_furlfile(filename):
+    """Return a temporary FURL file."""
+    return tempfile.mktemp(dir=os.path.dirname(filename),
+                           prefix=os.path.basename(filename))
+
+
+def make_tub(ip, port, secure, cert_file):
+    """Create a listening tub given an ip, port, and cert_file location.
+    
+    Parameters
+    ----------
+    ip : str
+        The ip address or hostname that the tub should listen on.  
+        Empty means all interfaces.
+    port : int
+        The port that the tub should listen on.  A value of 0 means
+        pick a random port
+    secure: bool
+        Will the connection be secure (in the Foolscap sense).
+    cert_file: str
+        A filename of a file to be used for theSSL certificate.
+
+    Returns
+    -------
+    A tub, listener tuple.
+    """
+    if secure:
+        if have_crypto:
+            tub = Tub(certFile=cert_file)
+        else:
+            raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
+                                "can't run in secure mode.  Try running without "
+                                "security using 'ipcontroller -xy'.")
+    else:
+        tub = UnauthenticatedTub()
+    
+    # Set the strport based on the ip and port and start listening
+    if ip == '':
+        strport = "tcp:%i" % port
+    else:
+        strport = "tcp:%i:interface=%s" % (port, ip)
+    log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
+    listener = tub.listenOn(strport)
+    
+    return tub, listener
+
+
+class FCServiceFactory(AdaptedConfiguredObjectFactory):
+    """This class creates a tub with various services running in it.
+
+    The basic idea is that :meth:`create` returns a running :class:`Tub`
+    instance that has a number of Foolscap references registered in it.
+    This class is a subclass of :class:`IPython.core.component.Component`
+    so the IPython configuration and component system are used.
+
+    Attributes
+    ----------
+    interfaces : Config
+        A Config instance whose values are sub-Config objects having two
+        keys: furl_file and interface_chain.
+
+    The other attributes are the standard ones for Foolscap.
+    """
+
+    ip = Str('', config=True)
+    port = Int(0, config=True)
+    secure = Bool(True, config=True)
+    cert_file = Str('', config=True)
+    location = Str('', config=True)
+    reuse_furls = Bool(False, config=True)
+    interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
+
+    def __init__(self, config, adaptee):
+        super(FCServiceFactory, self).__init__(config, adaptee)
+        self._check_reuse_furls()
+
+    def _ip_changed(self, name, old, new):
+        if new == 'localhost' or new == '127.0.0.1':
+            self.location = '127.0.0.1'
+
+    def _check_reuse_furls(self):
+        furl_files = [i.furl_file for i in self.interfaces.values()]
+        for ff in furl_files:
+            fullfile = self._get_security_file(ff)
+            if self.reuse_furls:
+                if self.port==0:
+                    raise FURLError("You are trying to reuse the FURL file "
+                        "for this connection, but the port for this connection "
+                        "is set to 0 (autoselect). To reuse the FURL file "
+                        "you need to specify specific port to listen on."
+                    )
+                else:
+                    log.msg("Reusing FURL file: %s" % fullfile)
+            else:
+                if os.path.isfile(fullfile):
+                    log.msg("Removing old FURL file: %s" % fullfile)
+                    os.remove(fullfile)
+
+    def _get_security_file(self, filename):
+        return os.path.join(self.config.Global.security_dir, filename)
+
+    def create(self):
+        """Create and return the Foolscap tub with everything running."""
+
+        self.tub, self.listener = make_tub(
+            self.ip, self.port, self.secure, 
+            self._get_security_file(self.cert_file)
+        )
+        # log.msg("Interfaces to register [%r]: %r" % \
+        #     (self.__class__, self.interfaces))
+        if not self.secure:
+            log.msg("WARNING: running with no security: %s" % \
+                self.__class__.__name__)
+        reactor.callWhenRunning(self.set_location_and_register)
+        return self.tub
+
+    def set_location_and_register(self):
+        """Set the location for the tub and return a deferred."""
+
+        if self.location == '':
+            d = self.tub.setLocationAutomatically()
+        else:
+            d = defer.maybeDeferred(self.tub.setLocation,
+                "%s:%i" % (self.location, self.listener.getPortnum()))
+        self.adapt_to_interfaces(d)
+
+    def adapt_to_interfaces(self, d):
+        """Run through the interfaces, adapt and register."""
+
+        for ifname, ifconfig in self.interfaces.iteritems():
+            ff = self._get_security_file(ifconfig.furl_file)
+            log.msg("Adapting [%s] to interface: %s" % \
+                (self.adaptee.__class__.__name__, ifname))
+            log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
+            check_furl_file_security(ff, self.secure)
+            adaptee = self.adaptee
+            for i in ifconfig.interface_chain:
+                adaptee = import_item(i)(adaptee)
+            d.addCallback(self.register, adaptee, furl_file=ff)
+
+    def register(self, empty, ref, furl_file):
+        """Register the reference with the FURL file.
+
+        The FURL file is created and then moved to make sure that when the
+        file appears, the buffer has been flushed and the file closed.
+        """
+        temp_furl_file = get_temp_furlfile(furl_file)
+        self.tub.registerReference(ref, furlFile=temp_furl_file)
+        os.rename(temp_furl_file, furl_file)
 

IPython/kernel/multiengine.py

@@ -262,9 +262,8 b' class MultiEngine(ControllerAdapterBase):'
         elif targets == 'all':
             eList = self.engines.values()
             if len(eList) == 0:
-                msg = """There are no engines registered.
-                     Check the logs in ~/.ipython/log if you think there should have been."""
-                raise error.NoEnginesRegistered(msg)
+                raise error.NoEnginesRegistered("There are no engines registered. "
+                    "Check the logs if you think there should have been.")
             else:
                 return eList
         else:

IPython/kernel/multiengineclient.py

@@ -263,10 +263,18 b' class InteractiveMultiEngineClient(object):'
         """
         
         try:
-            __IPYTHON__.activeController = self
+            # This is injected into __builtins__.
+            ip = get_ipython()
         except NameError:
-            print "The IPython Controller magics only work within IPython."
-                    
+            print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
+        else:
+            pmagic = ip.get_component('parallel_magic')
+            if pmagic is not None:
+                pmagic.active_multiengine_client = self
+            else:
+                print "You must first load the parallelmagic extension " \
+                      "by doing '%load_ext parallelmagic'"
+
     def __setitem__(self, key, value):
         """Add a dictionary interface for pushing/pulling.
         

IPython/kernel/scripts/ipcluster

@@ -1,22 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-"""ipcluster script"""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipcluster
-	ipcluster.main()
+from IPython.kernel.ipclusterapp import launch_new_instance
 
+launch_new_instance()

IPython/kernel/scripts/ipcontroller

@@ -1,20 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipcontroller
-	ipcontroller.main()
+from IPython.kernel.ipcontrollerapp import launch_new_instance
 
+launch_new_instance()

IPython/kernel/scripts/ipengine

@@ -1,20 +1,20 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+
+from IPython.kernel.ipengineapp import launch_new_instance
+
+launch_new_instance()
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipengine
-	ipengine.main()
 

IPython/kernel/tests/test_multienginefc.py

@@ -42,6 +42,14 b' def _raise_it(f):'
 
 class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase):
 
+    # XXX (fperez) this is awful: I'm fully disabling this entire test class.
+    # Right now it's blocking the tests from running at all, and I don't know
+    # how to fix it.  I hope Brian can have a stab at it, but at least by doing
+    # this we can run the entire suite to completion.
+    # Once the problem is cleared, remove this skip method.
+    skip = True
+    # END XXX
+    
     def setUp(self):
     
         self.engines = []
@@ -141,4 +149,4 b' class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMulti'
         def f(x): return 1/0
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-        return d
  No newline at end of file
+        return d

IPython/kernel/tests/test_taskfc.py

@@ -48,6 +48,14 b' def _raise_it(f):'
 
 class TaskTest(DeferredTestCase, ITaskControllerTestCase):
 
+    # XXX (fperez) this is awful: I'm fully disabling this entire test class.
+    # Right now it's blocking the tests from running at all, and I don't know
+    # how to fix it.  I hope Brian can have a stab at it, but at least by doing
+    # this we can run the entire suite to completion.
+    # Once the problem is cleared, remove this skip method.
+    skip = True
+    # END XXX
+ 
     def setUp(self):
     
         self.engines = []
@@ -158,4 +166,4 b' class TaskTest(DeferredTestCase, ITaskControllerTestCase):'
         def f(x): return 1/0
         d = f(range(10))
         d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
-        return d
  No newline at end of file
+        return d

IPython/kernel/twistedutil.py

@@ -3,18 +3,16 b''
 
 """Things directly related to all of twisted."""
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 import os, sys
 import threading, Queue, atexit
@@ -25,9 +23,9 b' from twisted.python import log, failure'
 
 from IPython.kernel.error import FileTimeoutError
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Classes related to twisted and threads
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 
 class ReactorInThread(threading.Thread):
@@ -42,6 +40,15 b' class ReactorInThread(threading.Thread):'
     """
     
     def run(self):
+        """Run the twisted reactor in a thread.
+
+        This runs the reactor with installSignalHandlers=0, which prevents
+        twisted from installing any of its own signal handlers. This needs to
+        be disabled because signal.signal can't be called in a thread. The
+        only problem with this is that SIGCHLD events won't be detected so
+        spawnProcess won't detect that its processes have been killed by
+        an external factor.
+        """
         reactor.run(installSignalHandlers=0)
         # self.join()
         
@@ -247,3 +254,21 b' def wait_for_file(filename, delay=0.1, max_tries=10):'
     
     _test_for_file(filename)
     return d
+
+
+def sleep_deferred(seconds):
+    """Sleep without blocking the event loop."""
+    d = defer.Deferred()
+    reactor.callLater(seconds, d.callback, seconds)
+    return d
+
+
+def make_deferred(func):
+    """A decorator that calls a function with :func`maybeDeferred`."""
+
+    def _wrapper(*args, **kwargs):
+        return defer.maybeDeferred(func, *args, **kwargs)
+
+    return _wrapper
+
+

IPython/lib/inputhook.py

@@ -1,5 +1,5 b''
 #!/usr/bin/env python
-# encoding: utf-8
+# coding: utf-8
 """
 Inputhook management for GUI event loop integration.
 """
@@ -24,6 +24,7 b' import sys'
 
 # Constants for identifying the GUI toolkits.
 GUI_WX = 'wx'
+GUI_QT = 'qt'
 GUI_QT4 = 'qt4'
 GUI_GTK = 'gtk'
 GUI_TK = 'tk'
@@ -326,8 +327,17 b' class InputHookManager(object):'
         self._installed = True
         return original
 
-    def clear_inputhook(self):
-        """Set PyOS_InputHook to NULL and return the previous one."""
+    def clear_inputhook(self, app=None):
+        """Set PyOS_InputHook to NULL and return the previous one.
+
+        Parameters
+        ----------
+        app : optional, ignored
+          This parameter is allowed only so that clear_inputhook() can be
+          called with a similar interface as all the ``enable_*`` methods.  But
+          the actual value of the parameter is ignored.  This uniform interface
+          makes it easier to have user-level entry points in the main IPython
+          app like :meth:`enable_gui`."""
         pyos_inputhook_ptr = self.get_pyos_inputhook()
         original = self.get_pyos_inputhook_as_func()
         pyos_inputhook_ptr.value = ctypes.c_void_p(None).value
@@ -523,3 +533,39 b' set_inputhook = inputhook_manager.set_inputhook'
 current_gui = inputhook_manager.current_gui
 clear_app_refs = inputhook_manager.clear_app_refs
 spin = inputhook_manager.spin
+
+
+# Convenience function to switch amongst them
+def enable_gui(gui=None, app=True):
+    """Switch amongst GUI input hooks by name.
+
+    This is just a utility wrapper around the methods of the InputHookManager
+    object.
+
+    Parameters
+    ----------
+    gui : optional, string or None
+      If None, clears input hook, otherwise it must be one of the recognized
+      GUI names (see ``GUI_*`` constants in module).
+
+    app : optional, bool
+      If true, create an app object and return it.
+
+    Returns
+    -------
+    The output of the underlying gui switch routine, typically the actual
+    PyOS_InputHook wrapper object or the GUI toolkit app created, if there was
+    one.
+    """
+    guis = {None: clear_inputhook,
+            GUI_TK: enable_tk,
+            GUI_GTK: enable_gtk,
+            GUI_WX: enable_wx,
+            GUI_QT: enable_qt4, # qt3 not supported
+            GUI_QT4: enable_qt4 }
+    try:
+        gui_hook = guis[gui]
+    except KeyError:
+        e="Invalid GUI request %r, valid ones are:%s" % (gui, guis.keys())
+        raise ValueError(e)
+    return gui_hook(app)

IPython/lib/inputhookwx.py

@@ -51,19 +51,22 b' def inputhook_wx1():'
     This approach seems to work, but its performance is not great as it 
     relies on having PyOS_InputHook called regularly.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-
-        # Make a temporary event loop and process system events until
-        # there are no more waiting, then allow idle events (which
-        # will also deal with pending or posted wx events.)
-        evtloop = wx.EventLoop()
-        ea = wx.EventLoopActivator(evtloop)
-        while evtloop.Pending():
-            evtloop.Dispatch()
-        app.ProcessIdle()
-        del ea
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # Make a temporary event loop and process system events until
+            # there are no more waiting, then allow idle events (which
+            # will also deal with pending or posted wx events.)
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            while evtloop.Pending():
+                evtloop.Dispatch()
+            app.ProcessIdle()
+            del ea
+    except KeyboardInterrupt:
+        pass
     return 0
 
 class EventLoopTimer(wx.Timer):
@@ -102,13 +105,16 b' def inputhook_wx2():'
     but eventually performance would suffer from calling select/kbhit too 
     often.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-        elr = EventLoopRunner()
-        # As this time is made shorter, keyboard response improves, but idle
-        # CPU load goes up.  10 ms seems like a good compromise.
-        elr.Run(time=10)  # CHANGE time here to control polling interval
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+            elr = EventLoopRunner()
+            # As this time is made shorter, keyboard response improves, but idle
+            # CPU load goes up.  10 ms seems like a good compromise.
+            elr.Run(time=10)  # CHANGE time here to control polling interval
+    except KeyboardInterrupt:
+        pass
     return 0
 
 def inputhook_wx3():
@@ -119,49 +125,54 b' def inputhook_wx3():'
     time.sleep is inserted.  This is needed, otherwise, CPU usage is at 100%.
     This sleep time should be tuned though for best performance.
     """
-    app = wx.GetApp()
-    if app is not None:
-        assert wx.Thread_IsMain()
-
-        # The import of wx on Linux sets the handler for signal.SIGINT
-        # to 0.  This is a bug in wx or gtk.  We fix by just setting it
-        # back to the Python default.
-        if not callable(signal.getsignal(signal.SIGINT)):
-            signal.signal(signal.SIGINT, signal.default_int_handler)
-
-        evtloop = wx.EventLoop()
-        ea = wx.EventLoopActivator(evtloop)
-        t = clock()
-        while not stdin_ready():
-            while evtloop.Pending():
-                t = clock()
-                evtloop.Dispatch()
-            app.ProcessIdle()
-            # We need to sleep at this point to keep the idle CPU load
-            # low.  However, if sleep to long, GUI response is poor.  As 
-            # a compromise, we watch how often GUI events are being processed
-            # and switch between a short and long sleep time.  Here are some
-            # stats useful in helping to tune this.
-            # time    CPU load
-            # 0.001   13%
-            # 0.005   3%
-            # 0.01    1.5%
-            # 0.05    0.5%
-            used_time = clock() - t
-            if used_time > 5*60.0:
-                # print 'Sleep for 5 s'  # dbg
-                time.sleep(5.0)
-            elif used_time > 10.0:
-                # print 'Sleep for 1 s'  # dbg
-                time.sleep(1.0)
-            elif used_time > 0.1:
-                # Few GUI events coming in, so we can sleep longer
-                # print 'Sleep for 0.05 s'  # dbg
-                time.sleep(0.05)
-            else:
-                # Many GUI events coming in, so sleep only very little
-                time.sleep(0.001)
-        del ea
+    # We need to protect against a user pressing Control-C when IPython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # The import of wx on Linux sets the handler for signal.SIGINT
+            # to 0.  This is a bug in wx or gtk.  We fix by just setting it
+            # back to the Python default.
+            if not callable(signal.getsignal(signal.SIGINT)):
+                signal.signal(signal.SIGINT, signal.default_int_handler)
+
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            t = clock()
+            while not stdin_ready():
+                while evtloop.Pending():
+                    t = clock()
+                    evtloop.Dispatch()
+                app.ProcessIdle()
+                # We need to sleep at this point to keep the idle CPU load
+                # low.  However, if sleep to long, GUI response is poor.  As 
+                # a compromise, we watch how often GUI events are being processed
+                # and switch between a short and long sleep time.  Here are some
+                # stats useful in helping to tune this.
+                # time    CPU load
+                # 0.001   13%
+                # 0.005   3%
+                # 0.01    1.5%
+                # 0.05    0.5%
+                used_time = clock() - t
+                if used_time > 5*60.0:
+                    # print 'Sleep for 5 s'  # dbg
+                    time.sleep(5.0)
+                elif used_time > 10.0:
+                    # print 'Sleep for 1 s'  # dbg
+                    time.sleep(1.0)
+                elif used_time > 0.1:
+                    # Few GUI events coming in, so we can sleep longer
+                    # print 'Sleep for 0.05 s'  # dbg
+                    time.sleep(0.05)
+                else:
+                    # Many GUI events coming in, so sleep only very little
+                    time.sleep(0.001)
+            del ea
+    except KeyboardInterrupt:
+        pass
     return 0
 
 # This is our default implementation

IPython/lib/irunner.py

@@ -303,11 +303,11 b' class IPythonRunner(InteractiveRunner):'
     def __init__(self,program = 'ipython',args=None,out=sys.stdout,echo=True):
         """New runner, optionally passing the ipython command to use."""
         
-        args0 = ['-colors','NoColor',
+        args0 = ['--colors','NoColor',
                  '-pi1','In [\\#]: ',
                  '-pi2','   .\\D.: ',
-                 '-noterm_title',
-                 '-noautoindent']
+                 '--noterm-title',
+                 '--no-auto-indent']
         if args is None: args = args0
         else: args = args0 + args
         prompts = [r'In \[\d+\]: ',r'   \.*: ']

IPython/quarantine/InterpreterExec.py

@@ -14,7 +14,7 b' executed.'
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-# TODO: deprecated
+
 def prefilter_shell(self,line,continuation):
     """Alternate prefilter, modified for shell-like functionality.
 

IPython/quarantine/ipy_fsops.py

@@ -141,7 +141,7 b' def collect(ip,arg):'
     Without args, try to open ~/_ipython/collect dir (in win32 at least).
     """
     from IPython.external.path import path
-    basedir = path(ip.options.IPYTHONDIR + '/collect')
+    basedir = path(ip.ipython_dir + '/collect')
     try:    
         fs = mglob.expand(arg.split(None,1)[1])
     except IndexError:
@@ -170,7 +170,7 b' def inote(ip,arg):'
     Without args, opens notes.txt for editing.
     """
     import time
-    fname = ip.options.IPYTHONDIR + '/notes.txt'
+    fname = ip.ipython_dir + '/notes.txt'
     
     try:
         entry = " === " + time.asctime() + ': ===\n' + arg.split(None,1)[1] + '\n'

IPython/testing/__init__.py

@@ -0,0 +1,18 b''
+"""Testing support (tools to test IPython itself).
+"""
+
+# User-level entry point for testing
+def test():
+    """Run the entire IPython test suite.
+
+    For fine-grained control, you should use the :file:`iptest` script supplied
+    with the IPython installation."""
+
+    # Do the import internally, so that this function doesn't increase total
+    # import time
+    from iptest import run_iptestall
+    run_iptestall()
+
+# So nose doesn't try to run this as a test itself and we end up with an
+# infinite test loop
+test.__test__ = False

IPython/testing/decorators.py

@@ -10,27 +10,79 b' This module provides a set of useful decorators meant to be ready to use in'
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
+Included decorators:
+
+
+Lightweight testing that remains unittest-compatible.
+
+- @parametric, for parametric test support that is vastly easier to use than
+  nose's for debugging. With ours, if a test fails, the stack under inspection
+  is that of the test and not that of the test framework.
+
+- An @as_unittest decorator can be used to tag any normal parameter-less
+  function as a unittest TestCase.  Then, both nose and normal unittest will
+  recognize it as such.  This will make it easier to migrate away from Nose if
+  we ever need/want to while maintaining very lightweight tests.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009-2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 # Stdlib imports
 import inspect
 import sys
+import unittest
 
 # Third-party imports
 
-# This is Michele Simionato's decorator module, also kept verbatim.
+# This is Michele Simionato's decorator module, kept verbatim.
 from IPython.external.decorator import decorator, update_wrapper
 
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from _paramtestpy2 import parametric, ParametricTestCase
+else:
+    from _paramtestpy3 import parametric, ParametricTestCase
+
+# Expose the unittest-driven decorators
+from ipunittest import ipdoctest, ipdocstring
+
 # Grab the numpy-specific decorators which we keep in a file that we
-# occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
-# of numpy.testing.decorators.
-from decorators_numpy import *
+# occasionally update from upstream: decorators.py is a copy of
+# numpy.testing.decorators, we expose all of it here.
+from IPython.external.decorators import *
 
-##############################################################################
-# Local code begins
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+# Simple example of the basic idea
+def as_unittest(func):
+    """Decorator to make a simple function into a normal test via unittest."""
+    class Tester(unittest.TestCase):
+        def test(self):
+            func()
+
+    Tester.__name__ = func.__name__
+
+    return Tester
 
 # Utility functions
 
@@ -51,21 +103,23 b' def apply_wrapper(wrapper,func):'
 def make_label_dec(label,ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
-    :Parameters:
+    Parameters
+    ----------
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
-    :Keywords:
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
-    :Returns:
+    Returns
+    -------
       A decorator.
 
-    :Examples:
+    Examples
+    --------
 
     A simple labeling decorator:
     >>> slow = make_label_dec('slow')
@@ -151,7 +205,7 b' def skipif(skip_condition, msg=None):'
 
         # Allow for both boolean or callable skip conditions.
         if callable(skip_condition):
-            skip_val = lambda : skip_condition()
+            skip_val = skip_condition
         else:
             skip_val = lambda : skip_condition
 
@@ -193,11 +247,13 b' def skipif(skip_condition, msg=None):'
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
-    :Parameters:
+    Parameters
+    ----------
       msg : string
         Optional message to be added.
 
-    :Returns:
+    Returns
+    -------
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
@@ -206,6 +262,16 b' def skip(msg=None):'
     return skipif(True,msg)
 
 
+def onlyif(condition, msg):
+    """The reverse from skipif, see skipif for details."""
+
+    if callable(condition):
+        skip_condition = lambda : not condition()
+    else:
+        skip_condition = lambda : not condition
+
+    return skipif(skip_condition, msg)
+
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
 def numpy_not_available():
@@ -252,3 +318,7 b" skip_if_not_osx = skipif(sys.platform != 'darwin',"
 skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
 
 skipknownfailure = skip('This test is known to fail')
+
+# A null 'decorator', useful to make more readable code that needs to pick
+# between different decorators based on OS or other conditions
+null_deco = lambda f: f

IPython/testing/iptest.py

@@ -16,27 +16,53 b' For now, this script requires that both nose and twisted are installed. This'
 will change in the future.
 """
 
+from __future__ import absolute_import
+
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
+# Stdlib
 import os
 import os.path as path
+import signal
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
+# Note: monkeypatch!
+# We need to monkeypatch a small problem in nose itself first, before importing
+# it for actual use.  This should get into nose upstream, but its release cycle
+# is slow and we need it for our parametric tests to work correctly.
+from . import nosepatch
+# Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.core import TestProgram
 
-from IPython.utils.platutils import find_cmd
-# from IPython.testing.plugin.ipdoctest import IPythonDoctest
+# Our own imports
+from IPython.utils import genutils
+from IPython.utils.platutils import find_cmd, FindCmdError
+from . import globalipapp
+from . import tools
+from .plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
 #-----------------------------------------------------------------------------
+# Warnings control
+#-----------------------------------------------------------------------------
+# Twisted generates annoying warnings with Python 2.6, as will do other code
+# that imports 'sets' as of today
+warnings.filterwarnings('ignore', 'the sets module is deprecated',
+                        DeprecationWarning )
+
+# This one also comes from Twisted
+warnings.filterwarnings('ignore', 'the sha module is deprecated',
+                        DeprecationWarning)
+
+#-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
@@ -49,6 +75,7 b' def test_for(mod):'
     else:
         return True
 
+    
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
 have_wx_aui = test_for('wx.aui')
@@ -62,83 +89,211 b" have_gobject = test_for('gobject')"
 
 
 def make_exclude():
-
-    # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
-    # testing problems.  We should strive to minimize the number of skipped
-    # modules, since this means untested code.  As the testing machinery
-    # solidifies, this list should eventually become empty.
-    EXCLUDE = [pjoin('IPython', 'external'),
-               pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
-               pjoin('IPython_doctest_plugin'),
-               pjoin('IPython', 'quarantine'),
-               pjoin('IPython', 'deathrow'),
-               pjoin('IPython', 'testing', 'attic'),
-               pjoin('IPython', 'testing', 'tools'),
-               pjoin('IPython', 'testing', 'mkdoctests'),
-               pjoin('IPython', 'lib', 'inputhook')
-               ]
+    """Make patterns of modules and packages to exclude from testing.
+    
+    For the IPythonDoctest plugin, we need to exclude certain patterns that
+    cause testing problems.  We should strive to minimize the number of
+    skipped modules, since this means untested code.  As the testing
+    machinery solidifies, this list should eventually become empty.
+    These modules and packages will NOT get scanned by nose at all for tests.
+    """
+    # Simple utility to make IPython paths more readably, we need a lot of
+    # these below
+    ipjoin = lambda *paths: pjoin('IPython', *paths)
+    
+    exclusions = [ipjoin('external'),
+                  ipjoin('frontend', 'process', 'winprocess.py'),
+                  pjoin('IPython_doctest_plugin'),
+                  ipjoin('quarantine'),
+                  ipjoin('deathrow'),
+                  ipjoin('testing', 'attic'),
+                  # This guy is probably attic material
+                  ipjoin('testing', 'mkdoctests'),
+                  # Testing inputhook will need a lot of thought, to figure out
+                  # how to have tests that don't lock up with the gui event
+                  # loops in the picture
+                  ipjoin('lib', 'inputhook'),
+                  # Config files aren't really importable stand-alone
+                  ipjoin('config', 'default'),
+                  ipjoin('config', 'profile'),
+                  ]
 
     if not have_wx:
-        EXCLUDE.append(pjoin('IPython', 'gui'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookwx'))
+        exclusions.append(ipjoin('gui'))
+        exclusions.append(ipjoin('frontend', 'wx'))
+        exclusions.append(ipjoin('lib', 'inputhookwx'))
 
     if not have_gtk or not have_gobject:
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookgtk'))
+        exclusions.append(ipjoin('lib', 'inputhookgtk'))
 
     if not have_wx_aui:
-        EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+        exclusions.append(ipjoin('gui', 'wx', 'wxIPython'))
 
     if not have_objc:
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
+        exclusions.append(ipjoin('frontend', 'cocoa'))
 
     if not sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
+        exclusions.append(ipjoin('utils', 'platutils_win32'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
+        exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
+        exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
     if not os.name == 'posix':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
+        exclusions.append(ipjoin('utils', 'platutils_posix'))
 
     if not have_pexpect:
-        EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
+        exclusions.extend([ipjoin('scripts', 'irunner'),
+                           ipjoin('lib', 'irunner')])
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have_twisted and have_zi and have_foolscap):
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
-                             'test_linefrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 
-                             'test_frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
-                             'test_prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
-                             'test_asyncfrontendbase')),
-        EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 
-                             'test_decorators_trial'))
+        exclusions.extend(
+            [ipjoin('frontend', 'asyncfrontendbase'),
+             ipjoin('frontend', 'prefilterfrontend'),
+             ipjoin('frontend', 'frontendbase'),
+             ipjoin('frontend', 'linefrontendbase'),
+             ipjoin('frontend', 'tests', 'test_linefrontend'),
+             ipjoin('frontend', 'tests', 'test_frontendbase'),
+             ipjoin('frontend', 'tests', 'test_prefilterfrontend'),
+             ipjoin('frontend', 'tests', 'test_asyncfrontendbase'),
+             ipjoin('testing', 'parametric'),
+             ipjoin('testing', 'util'),
+             ipjoin('testing', 'tests', 'test_decorators_trial'),
+             ] )
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
-        EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+        exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
-    return EXCLUDE
+    return exclusions
 
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
+class IPTester(object):
+    """Call that calls iptest or trial in a subprocess.
+    """
+    #: string, name of test runner that will be called
+    runner = None
+    #: list, parameters for test runner
+    params = None
+    #: list, arguments of system call to be made to call test runner
+    call_args = None
+    #: list, process ids of subprocesses we start (for cleanup)
+    pids = None
+    
+    def __init__(self,runner='iptest',params=None):
+        """Create new test runner."""
+        if runner == 'iptest':
+            # Find our own 'iptest' script OS-level entry point
+            try:
+                iptest_path = os.path.abspath(find_cmd('iptest'))
+            except FindCmdError:
+                # Script not installed (may be the case for testing situations
+                # that are running from a source tree only), pull from internal
+                # path:
+                pak_dir = os.path.abspath(genutils.get_ipython_package_dir())
+                iptest_path = pjoin(pak_dir, 'scripts', 'iptest')
+            self.runner = tools.cmd2argv(iptest_path) + ['-v']
+        else:
+            self.runner = tools.cmd2argv(os.path.abspath(find_cmd('trial')))
+        if params is None:
+            params = []
+        if isinstance(params,str):
+            params = [params]
+        self.params = params
+
+        # Assemble call
+        self.call_args = self.runner+self.params
+
+        # Store pids of anything we start to clean up on deletion, if possible
+        # (on posix only, since win32 has no os.kill)
+        self.pids = []
+
+    if sys.platform == 'win32':
+        def _run_cmd(self):
+            # On Windows, use os.system instead of subprocess.call, because I
+            # was having problems with subprocess and I just don't know enough
+            # about win32 to debug this reliably.  Os.system may be the 'old
+            # fashioned' way to do it, but it works just fine.  If someone
+            # later can clean this up that's fine, as long as the tests run
+            # reliably in win32.
+            return os.system(' '.join(self.call_args))
+    else:
+        def _run_cmd(self):
+            subp = subprocess.Popen(self.call_args)
+            self.pids.append(subp.pid)
+            # If this fails, the pid will be left in self.pids and cleaned up
+            # later, but if the wait call succeeds, then we can clear the
+            # stored pid.
+            retcode = subp.wait()
+            self.pids.pop()
+            return retcode
+        
+    def run(self):
+        """Run the stored commands"""
+        try:
+            return self._run_cmd()
+        except:
+            import traceback
+            traceback.print_exc()
+            return 1  # signal failure
+
+    def __del__(self):
+        """Cleanup on exit by killing any leftover processes."""
+
+        if not hasattr(os, 'kill'):
+            return
+        
+        for pid in self.pids:
+            try:
+                print 'Cleaning stale PID:', pid
+                os.kill(pid, signal.SIGKILL)
+            except OSError:
+                # This is just a best effort, if we fail or the process was
+                # really gone, ignore it.
+                pass     
+
+
+def make_runners():
+    """Define the top-level packages that need to be tested.
+    """
+
+    nose_packages = ['config', 'core', 'extensions', 'frontend', 'lib',
+                     'scripts', 'testing', 'utils',
+                     # Note that we list the kernel here, though the bulk of it
+                     # is twisted-based, because nose picks up doctests that
+                     # twisted doesn't.
+                     'kernel']
+    trial_packages = ['kernel']
+
+    if have_wx:
+        nose_packages.append('gui')
+
+    #nose_packages = ['core']  # dbg
+    #trial_packages = []  # dbg 
+
+    nose_packages = ['IPython.%s' % m for m in nose_packages ]
+    trial_packages = ['IPython.%s' % m for m in trial_packages ]
+
+    # Make runners, most with nose
+    nose_testers = [IPTester(params=v) for v in nose_packages]
+    runners = dict(zip(nose_packages, nose_testers))
+    # And add twisted ones if conditions are met
+    if have_zi and have_twisted and have_foolscap:
+        trial_testers = [IPTester('trial',params=v) for v in trial_packages]
+        runners.update(dict(zip(trial_packages,trial_testers)))
+                                 
+    return runners
+
+
 def run_iptest():
     """Run the IPython test suite using nose.
     
@@ -150,16 +305,15 b' def run_iptest():'
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
-    argv = sys.argv + [ 
-                        # Loading ipdoctest causes problems with Twisted.
-                        # I am removing this as a temporary fix to get the 
-                        # test suite back into working shape.  Our nose
-                        # plugin needs to be gone through with a fine
-                        # toothed comb to find what is causing the problem.
-                        # '--with-ipdoctest',
-                        # '--ipdoctest-tests','--ipdoctest-extension=txt',
-                        # '--detailed-errors',
-                       
+    argv = sys.argv + [ '--detailed-errors',
+                        # Loading ipdoctest causes problems with Twisted, but
+                        # our test suite runner now separates things and runs
+                        # all Twisted tests with trial.
+                        '--with-ipdoctest',
+                        '--ipdoctest-tests','--ipdoctest-extension=txt',
+                        
+                        #'-x','-s',  # dbg
+                        
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
@@ -182,90 +336,21 b' def run_iptest():'
     if not has_tests:
         argv.append('IPython')
 
-    # Construct list of plugins, omitting the existing doctest plugin, which
-    # ours replaces (and extends).
-    EXCLUDE = make_exclude()
-    plugins = []
-    # plugins = [IPythonDoctest(EXCLUDE)]
+    ## # Construct list of plugins, omitting the existing doctest plugin, which
+    ## # ours replaces (and extends).
+    plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
+    # We need a global ipython running in this process
+    globalipapp.start_ipython()
+    # Now nose can run
     TestProgram(argv=argv,plugins=plugins)
 
 
-class IPTester(object):
-    """Call that calls iptest or trial in a subprocess.
-    """
-    def __init__(self,runner='iptest',params=None):
-        """ """
-        if runner == 'iptest':
-            self.runner = ['iptest','-v']
-        else:
-            self.runner = [find_cmd('trial')]
-        if params is None:
-            params = []
-        if isinstance(params,str):
-            params = [params]
-        self.params = params
-
-        # Assemble call
-        self.call_args = self.runner+self.params
-
-    if sys.platform == 'win32':
-        def run(self):
-            """Run the stored commands"""
-            # On Windows, cd to temporary directory to run tests.  Otherwise,
-            # Twisted's trial may not be able to execute 'trial IPython', since
-            # it will confuse the IPython module name with the ipython
-            # execution scripts, because the windows file system isn't case
-            # sensitive.
-            # We also use os.system instead of subprocess.call, because I was
-            # having problems with subprocess and I just don't know enough
-            # about win32 to debug this reliably.  Os.system may be the 'old
-            # fashioned' way to do it, but it works just fine.  If someone
-            # later can clean this up that's fine, as long as the tests run
-            # reliably in win32.
-            curdir = os.getcwd()
-            os.chdir(tempfile.gettempdir())
-            stat = os.system(' '.join(self.call_args))
-            os.chdir(curdir)
-            return stat
-    else:
-        def run(self):
-            """Run the stored commands"""
-            return subprocess.call(self.call_args)
-
-
-def make_runners():
-    """Define the top-level packages that need to be tested.
-    """
-
-    nose_packages = ['config', 'core', 'extensions',
-                     'frontend', 'lib',
-                     'scripts', 'testing', 'utils']
-    trial_packages = ['kernel']
-
-    if have_wx:
-        nose_packages.append('gui')
-
-    nose_packages = ['IPython.%s' % m for m in nose_packages ]
-    trial_packages = ['IPython.%s' % m for m in trial_packages ]
-
-    # Make runners
-    runners = dict()
-    
-    nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages]))
-    if have_zi and have_twisted and have_foolscap:
-        trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages]))
-    runners.update(nose_runners)
-    runners.update(trial_runners)
-
-    return runners
-
-
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
@@ -277,15 +362,26 b' def run_iptestall():'
 
     runners = make_runners()
 
+    # Run the test runners in a temporary dir so we can nuke it when finished
+    # to clean up any junk files left over by accident.  This also makes it
+    # robust against being run in non-writeable directories by mistake, as the
+    # temp dir will always be user-writeable.
+    curdir = os.getcwd()
+    testdir = tempfile.gettempdir()
+    os.chdir(testdir)
+
     # Run all test runners, tracking execution time
     failed = {}
     t_start = time.time()
-    for name,runner in runners.iteritems():
-        print '*'*77
-        print 'IPython test group:',name
-        res = runner.run()
-        if res:
-            failed[name] = res
+    try:
+        for name,runner in runners.iteritems():
+            print '*'*77
+            print 'IPython test group:',name
+            res = runner.run()
+            if res:
+                failed[name] = res
+    finally:
+        os.chdir(curdir)
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)

IPython/testing/parametric.py

@@ -1,5 +1,8 b''
 """Parametric testing on top of twisted.trial.unittest.
 
+XXX - It may be possbile to deprecate this in favor of the new, cleaner
+parametric code.  We just need to double-check that the new code doesn't clash
+with Twisted (we know it works with nose and unittest).
 """
 
 __all__ = ['parametric','Parametric']

IPython/testing/plugin/ipdoctest.py

@@ -49,182 +49,14 b' from nose.util import anyp, getpackage, test_address, resolve_name, tolist'
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
+#-----------------------------------------------------------------------------
 
 log = logging.getLogger(__name__)
 
-###########################################################################
-# *** HACK ***
-# We must start our own ipython object and heavily muck with it so that all the
-# modifications IPython makes to system behavior don't send the doctest
-# machinery into a fit.  This code should be considered a gross hack, but it
-# gets the job done.
-
-def default_argv():
-    """Return a valid default argv for creating testing instances of ipython"""
-
-    # Get the install directory for the user configuration and tell ipython to
-    # use the default profile from there.
-    from IPython.config import userconfig
-    ipcdir = os.path.dirname(userconfig.__file__)
-    #ipconf = os.path.join(ipcdir,'ipy_user_conf.py')
-    ipconf = os.path.join(ipcdir,'ipythonrc')
-    #print 'conf:',ipconf # dbg
-    
-    return ['--colors=NoColor','--noterm_title','-rcfile=%s' % ipconf]
-
-
-# Hack to modify the %run command so we can sync the user's namespace with the
-# test globals.  Once we move over to a clean magic system, this will be done
-# with much less ugliness.
-
-class py_file_finder(object):
-    def __init__(self,test_filename):
-        self.test_filename = test_filename
-        
-    def __call__(self,name):
-        from IPython.utils.genutils import get_py_filename
-        try:
-            return get_py_filename(name)
-        except IOError:
-            test_dir = os.path.dirname(self.test_filename)
-            new_path = os.path.join(test_dir,name)
-            return get_py_filename(new_path)
-    
-
-def _run_ns_sync(self,arg_s,runner=None):
-    """Modified version of %run that syncs testing namespaces.
-
-    This is strictly needed for running doctests that call %run.
-    """
-
-    # When tests call %run directly (not via doctest) these function attributes
-    # are not set
-    try:
-        fname = _run_ns_sync.test_filename
-    except AttributeError:
-        fname = arg_s
-
-    finder = py_file_finder(fname)
-    out = _ip.magic_run_ori(arg_s,runner,finder)
-    
-    # Simliarly, there is no test_globs when a test is NOT a doctest
-    if hasattr(_run_ns_sync,'test_globs'):
-        _run_ns_sync.test_globs.update(_ip.user_ns)
-    return out
-
-
-class ipnsdict(dict):
-    """A special subclass of dict for use as an IPython namespace in doctests.
-
-    This subclass adds a simple checkpointing capability so that when testing
-    machinery clears it (we use it as the test execution context), it doesn't
-    get completely destroyed.
-    """
-    
-    def __init__(self,*a):
-        dict.__init__(self,*a)
-        self._savedict = {}
-        
-    def clear(self):
-        dict.clear(self)
-        self.update(self._savedict)
-
-    def _checkpoint(self):
-        self._savedict.clear()
-        self._savedict.update(self)
-
-    def update(self,other):
-        self._checkpoint()
-        dict.update(self,other)
-
-        # If '_' is in the namespace, python won't set it when executing code,
-        # and we have examples that test it.  So we ensure that the namespace
-        # is always 'clean' of it before it's used for test code execution.
-        self.pop('_',None)
-
-        # The builtins namespace must *always* be the real __builtin__ module,
-        # else weird stuff happens.  The main ipython code does have provisions
-        # to ensure this after %run, but since in this class we do some
-        # aggressive low-level cleaning of the execution namespace, we need to
-        # correct for that ourselves, to ensure consitency with the 'real'
-        # ipython.
-        self['__builtins__'] = __builtin__
-        
-
-def start_ipython():
-    """Start a global IPython shell, which we need for IPython-specific syntax.
-    """
-
-    # This function should only ever run once!
-    if hasattr(start_ipython,'already_called'):
-        return
-    start_ipython.already_called = True
-
-    # Ok,  first time we're called, go ahead
-    import new
-
-    import IPython
-    from IPython.core import ipapi
-    
-    def xsys(cmd):
-        """Execute a command and print its output.
-
-        This is just a convenience function to replace the IPython system call
-        with one that is more doctest-friendly.
-        """
-        cmd = _ip.var_expand(cmd,depth=1)
-        sys.stdout.write(commands.getoutput(cmd))
-        sys.stdout.flush()
-
-    # Store certain global objects that IPython modifies
-    _displayhook = sys.displayhook
-    _excepthook = sys.excepthook
-    _main = sys.modules.get('__main__')
-
-    argv = default_argv()
-    
-    # Start IPython instance.  We customize it to start with minimal frills.
-    IPython.shell.IPShell(argv,ipnsdict(),global_ns)
-
-    # Deactivate the various python system hooks added by ipython for
-    # interactive convenience so we don't confuse the doctest system
-    sys.modules['__main__'] = _main
-    sys.displayhook = _displayhook
-    sys.excepthook = _excepthook
-
-    # So that ipython magics and aliases can be doctested (they work by making
-    # a call into a global _ip object)
-    _ip = ipapi.get()
-    __builtin__._ip = _ip
-
-    # Modify the IPython system call with one that uses getoutput, so that we
-    # can capture subcommands and print them to Python's stdout, otherwise the
-    # doctest machinery would miss them.
-    _ip.system = xsys
-
-    # Also patch our %run function in.
-    im = new.instancemethod(_run_ns_sync,_ip, _ip.__class__)
-    _ip.magic_run_ori = _ip.magic_run
-    _ip.magic_run = im
-
-    # XXX - For some very bizarre reason, the loading of %history by default is
-    # failing.  This needs to be fixed later, but for now at least this ensures
-    # that tests that use %hist run to completion.
-    from IPython.core import history
-    history.init_ipython(_ip)
-    if not hasattr(_ip,'magic_history'):
-        raise RuntimeError("Can't load magics, aborting")
-
-
-# The start call MUST be made here.  I'm not sure yet why it doesn't work if
-# it is made later, at plugin initialization time, but in all my tests, that's
-# the case.
-start_ipython()
-
-# *** END HACK ***
-###########################################################################
 
+#-----------------------------------------------------------------------------
 # Classes and functions
+#-----------------------------------------------------------------------------
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
@@ -287,7 +119,7 b' class DocTestFinder(doctest.DocTestFinder):'
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
-
+        #print '_find for:', obj, name, module  # dbg
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = DocTestSkip(obj)
@@ -386,6 +218,7 b' class DocTestCase(doctests.DocTestCase):'
         self._dt_optionflags = optionflags
         self._dt_checker = checker
         self._dt_test = test
+        self._dt_test_globs_ori = test.globs
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
 
@@ -395,8 +228,9 b' class DocTestCase(doctests.DocTestCase):'
         self._dt_runner = runner
 
 
-        # Each doctest should remember what directory it was loaded from...
-        self._ori_dir = os.getcwd()
+        # Each doctest should remember the directory it was loaded from, so
+        # things like %run work without too many contortions
+        self._ori_dir = os.path.dirname(test.filename)
 
     # Modified runTest from the default stdlib
     def runTest(self):
@@ -417,6 +251,7 b' class DocTestCase(doctests.DocTestCase):'
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
             curdir = os.getcwd()
+            #print 'runTest in dir:', self._ori_dir  # dbg
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
@@ -431,7 +266,7 b' class DocTestCase(doctests.DocTestCase):'
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
-        
+        #print "setUp test", self._dt_test.examples # dbg
         if isinstance(self._dt_test.examples[0],IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
@@ -442,6 +277,12 b' class DocTestCase(doctests.DocTestCase):'
         super(DocTestCase, self).setUp()
 
     def tearDown(self):
+
+        # Undo the test.globs reassignment we made, so that the parent class
+        # teardown doesn't destroy the ipython namespace
+        if isinstance(self._dt_test.examples[0],IPExample):
+            self._dt_test.globs = self._dt_test_globs_ori
+            
         # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
         # it does look like one to me: its tearDown method tries to run
         #
@@ -730,8 +571,10 b' class IPDocTestRunner(doctest.DocTestRunner,object):'
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
-        _run_ns_sync.test_globs = test.globs
-        _run_ns_sync.test_filename = test.filename
+        #_ip._ipdoctest_test_globs = test.globs
+        #_ip._ipdoctest_test_filename = test.filename
+
+        test.globs.update(_ip.user_ns)
         
         return super(IPDocTestRunner,self).run(test,
                                                compileflags,out,clear_globs)
@@ -845,6 +688,7 b' class ExtensionDoctest(doctests.Doctest):'
 
 
     def loadTestsFromFile(self, filename):
+        #print "ipdoctest - from file", filename # dbg
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
@@ -871,7 +715,7 b' class ExtensionDoctest(doctests.Doctest):'
         Modified version that accepts extension modules as valid containers for
         doctests.
         """
-        # print '*** ipdoctest- wantFile:',filename  # dbg
+        #print '*** ipdoctest- wantFile:',filename  # dbg
 
         for pat in self.exclude_patterns:
             if pat.search(filename):
@@ -889,11 +733,12 b' class IPythonDoctest(ExtensionDoctest):'
     """
     name = 'ipdoctest'   # call nosetests with --with-ipdoctest
     enabled = True
-    
+
     def makeTest(self, obj, parent):
         """Look for doctests in the given object, which will be a
         function, method or class.
         """
+        #print 'Plugin analyzing:', obj, parent  # dbg
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
@@ -908,6 +753,7 b' class IPythonDoctest(ExtensionDoctest):'
                                   checker=self.checker)
 
     def options(self, parser, env=os.environ):
+        #print "Options for nose plugin:", self.name # dbg
         Plugin.options(self, parser, env)
         parser.add_option('--ipdoctest-tests', action='store_true',
                           dest='ipdoctest_tests',
@@ -928,6 +774,7 b' class IPythonDoctest(ExtensionDoctest):'
             parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
     def configure(self, options, config):
+        #print "Configuring nose plugin:", self.name # dbg
         Plugin.configure(self, options, config)
         self.doctest_tests = options.ipdoctest_tests
         self.extension = tolist(options.ipdoctest_extension)

IPython/testing/plugin/test_ipdoctest.py

@@ -18,25 +18,6 b' def doctest_simple():'
     """
 
 
-def doctest_run_builtins():
-    """Check that %run doesn't damage __builtins__ via a doctest.
-
-    This is similar to the test_run_builtins, but I want *both* forms of the
-    test to catch any possible glitches in our testing machinery, since that
-    modifies %run somewhat.  So for this, we have both a normal test (below)
-    and a doctest (this one).
-
-    In [1]: import tempfile
-
-    In [3]: f = tempfile.NamedTemporaryFile()
-
-    In [4]: f.write('pass\\n')
-
-    In [5]: f.flush()
-
-    In [7]: %run $f.name
-    """
-
 def doctest_multiline1():
     """The ipdoctest machinery must handle multiline examples gracefully.
 

IPython/testing/tests/test_decorators.py

@@ -5,13 +5,14 b''
 # Std lib
 import inspect
 import sys
+import unittest
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.testing import decorators as dec
-
+from IPython.testing.ipunittest import ParametricTestCase
 
 #-----------------------------------------------------------------------------
 # Utilities
@@ -41,6 +42,30 b' def getargspec(obj):'
 #-----------------------------------------------------------------------------
 # Testing functions
 
[e-mail]_unittest
+def trivial():
+    """A trivial test"""
+    pass
+
+# Some examples of parametric tests.
+
+def is_smaller(i,j):
+    assert i<j,"%s !< %s" % (i,j)
+
+class Tester(ParametricTestCase):
+
+    def test_parametric(self):
+        yield is_smaller(3, 4)
+        x, y = 1, 2
+        yield is_smaller(x, y)
+
[e-mail]
+def test_par_standalone():
+    yield is_smaller(3, 4)
+    x, y = 1, 2
+    yield is_smaller(x, y)
+
+
 @dec.skip
 def test_deliberately_broken():
     """A deliberately broken test - we want to skip this one."""
@@ -159,3 +184,36 b' def test_win32():'
 @dec.skip_osx
 def test_osx():
     nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")
+
+
+# Verify that the same decorators work for methods.
+# Note: this code is identical to that in test_decorators_trial, but that one
+# uses twisted's unittest, not the one from the stdlib, which we are using
+# here. While somewhat redundant, we want to check both with the stdlib and
+# with twisted, so the duplication is OK.
+class TestDecoratorsTrial(unittest.TestCase):
+    
+    @dec.skip()
+    def test_deliberately_broken(self):
+        """A deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip('Testing the skip decorator')
+    def test_deliberately_broken2(self):
+        """Another deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip_linux
+    def test_linux(self):
+        self.assertNotEquals(sys.platform, 'linux2',
+                             "This test can't run under linux")
+
+    @dec.skip_win32
+    def test_win32(self):
+        self.assertNotEquals(sys.platform, 'win32',
+                             "This test can't run under windows")
+
+    @dec.skip_osx
+    def test_osx(self):
+        self.assertNotEquals(sys.platform, 'darwin',
+                             "This test can't run under osx")

IPython/testing/tests/test_decorators_trial.py

@@ -1,6 +1,6 b''
 # encoding: utf-8
 """
-Tests for decorators_trial.py
+Tests for decorators.py compatibility with Twisted.trial
 """
 
 #-----------------------------------------------------------------------------
@@ -14,19 +14,24 b' Tests for decorators_trial.py'
 # Imports
 #-----------------------------------------------------------------------------
 
-# Tell nose to skip this module
+# Tell nose to skip this module, since this is for twisted only
 __test__ = {}
 
 import os
 import sys
 
 from twisted.trial import unittest
-import IPython.testing.decorators_trial as dec
+import IPython.testing.decorators as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
+# Note: this code is identical to that in test_decorators, but that one uses
+# stdlib unittest, not the one from twisted, which we are using here. While
+# somewhat redundant, we want to check both with the stdlib and with twisted,
+# so the duplication is OK.
+
 class TestDecoratorsTrial(unittest.TestCase):
     
     @dec.skip()
@@ -49,4 +54,4 b' class TestDecoratorsTrial(unittest.TestCase):'
 
     @dec.skip_osx
     def test_osx(self):
-        self.assertNotEquals(sys.platform,'darwin',"This test can't run under osx")
  No newline at end of file
+        self.assertNotEquals(sys.platform,'darwin',"This test can't run under osx")

IPython/testing/tests/test_tools.py

@@ -14,6 +14,7 b' Tests for testing.tools'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import with_statement
 
 import os
 import sys
@@ -21,32 +22,53 b' import sys'
 import nose.tools as nt
 
 from IPython.testing import decorators as dec
-from IPython.testing.tools import full_path
+from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 @dec.skip_win32
 def test_full_path_posix():
     spath = '/foo/bar.py'
-    result = full_path(spath,['a.txt','b.txt'])
+    result = tt.full_path(spath,['a.txt','b.txt'])
     nt.assert_equal(result, ['/foo/a.txt', '/foo/b.txt'])
     spath = '/foo'
-    result = full_path(spath,['a.txt','b.txt'])
+    result = tt.full_path(spath,['a.txt','b.txt'])
     nt.assert_equal(result, ['/a.txt', '/b.txt'])
-    result = full_path(spath,'a.txt')
+    result = tt.full_path(spath,'a.txt')
     nt.assert_equal(result, ['/a.txt'])
 
 
 @dec.skip_if_not_win32
 def test_full_path_win32():
     spath = 'c:\\foo\\bar.py'
-    result = full_path(spath,['a.txt','b.txt'])
+    result = tt.full_path(spath,['a.txt','b.txt'])
     nt.assert_equal(result, ['c:\\foo\\a.txt', 'c:\\foo\\b.txt'])
     spath = 'c:\\foo'
-    result = full_path(spath,['a.txt','b.txt'])
+    result = tt.full_path(spath,['a.txt','b.txt'])
     nt.assert_equal(result, ['c:\\a.txt', 'c:\\b.txt'])
-    result = full_path(spath,'a.txt')
-    nt.assert_equal(result, ['c:\\a.txt'])
  No newline at end of file
+    result = tt.full_path(spath,'a.txt')
+    nt.assert_equal(result, ['c:\\a.txt'])
+
+    
[e-mail]
+def test_parser():
+    err = ("FAILED (errors=1)", 1, 0)
+    fail = ("FAILED (failures=1)", 0, 1)
+    both = ("FAILED (errors=1, failures=1)", 1, 1)
+    for txt, nerr, nfail in [err, fail, both]:
+        nerr1, nfail1 = tt.parse_test_output(txt)
+        yield nt.assert_equal(nerr, nerr1)
+        yield nt.assert_equal(nfail, nfail1)
+
+        
[e-mail]
+def test_temp_pyfile():
+    src = 'pass\n'
+    fname, fh = tt.temp_pyfile(src)
+    yield nt.assert_true(os.path.isfile(fname))
+    fh.close()
+    with open(fname) as fh2:
+        src2 = fh2.read()
+    yield nt.assert_equal(src2, src)

IPython/testing/tools.py

@@ -25,14 +25,25 b' Authors'
 #-----------------------------------------------------------------------------
 # Required modules and packages
 #-----------------------------------------------------------------------------
+from __future__ import absolute_import
 
 import os
+import re
 import sys
+import tempfile
 
-import nose.tools as nt
+try:
+    # These tools are used by parts of the runtime, so we make the nose
+    # dependency optional at this point.  Nose is a hard dependency to run the
+    # test suite, but NOT to use ipython itself.
+    import nose.tools as nt
+    has_nose = True
+except ImportError:
+    has_nose = False
 
-from IPython.utils import genutils
-from IPython.testing import decorators as dec
+from IPython.utils import genutils, platutils
+
+from . import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -46,14 +57,19 b' def %(name)s(*a,**kw):'
     return nt.%(name)s(*a,**kw)
 """
 
-for _x in [a for a in dir(nt) if a.startswith('assert')]:
-    exec _tpl % dict(name=_x)
+if has_nose:
+    for _x in [a for a in dir(nt) if a.startswith('assert')]:
+        exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
+# The docstring for full_path doctests differently on win32 (different path
+# separator) so just skip the doctest there.  The example remains informative.
+doctest_deco = dec.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
+@doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
@@ -87,3 +103,213 b' def full_path(startPath,files):'
     files = genutils.list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
+
+
+def parse_test_output(txt):
+    """Parse the output of a test run and return errors, failures.
+
+    Parameters
+    ----------
+    txt : str
+      Text output of a test run, assumed to contain a line of one of the
+      following forms::
+        'FAILED (errors=1)'
+        'FAILED (failures=1)'
+        'FAILED (errors=1, failures=1)'
+
+    Returns
+    -------
+    nerr, nfail: number of errors and failures.
+    """
+
+    err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
+    if err_m:
+        nerr = int(err_m.group(1))
+        nfail = 0
+        return  nerr, nfail
+    
+    fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
+    if fail_m:
+        nerr = 0
+        nfail = int(fail_m.group(1))
+        return  nerr, nfail
+
+    both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
+                       re.MULTILINE)
+    if both_m:
+        nerr = int(both_m.group(1))
+        nfail = int(both_m.group(2))
+        return  nerr, nfail
+    
+    # If the input didn't match any of these forms, assume no error/failures
+    return 0, 0
+
+
+# So nose doesn't think this is a test
+parse_test_output.__test__ = False
+
+
+def cmd2argv(cmd):
+    r"""Take the path of a command and return a list (argv-style).
+
+    For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
+    .com or .bat, and ['python', cmd] otherwise.
+
+    This is mostly a Windows utility, to deal with the fact that the scripts in
+    Windows get wrapped in .exe entry points, so we have to call them
+    differently.
+
+    Parameters
+    ----------
+    cmd : string
+      The path of the command.
+
+    Returns
+    -------
+    argv-style list.
+
+    Examples
+    --------
+    In [2]: cmd2argv('/usr/bin/ipython')
+    Out[2]: ['python', '/usr/bin/ipython']
+
+    In [3]: cmd2argv(r'C:\Python26\Scripts\ipython.exe')
+    Out[3]: ['C:\\Python26\\Scripts\\ipython.exe']
+    """
+    ext = os.path.splitext(cmd)[1]
+    if ext in ['.exe', '.com', '.bat']:
+        return [cmd]
+    else:
+        return ['python', cmd]
+    
+
+def temp_pyfile(src, ext='.py'):
+    """Make a temporary python file, return filename and filehandle.
+
+    Parameters
+    ----------
+    src : string or list of strings (no need for ending newlines if list)
+      Source code to be written to the file.
+
+    ext : optional, string
+      Extension for the generated file.
+
+    Returns
+    -------
+    (filename, open filehandle)
+      It is the caller's responsibility to close the open file and unlink it.
+    """
+    fname = tempfile.mkstemp(ext)[1]
+    f = open(fname,'w')
+    f.write(src)
+    f.flush()
+    return fname, f
+
+
+def default_argv():
+    """Return a valid default argv for creating testing instances of ipython"""
+
+    # Get the install directory for the user configuration and tell ipython to
+    # use the default profile from there.
+    from IPython.config import default
+    ipcdir = os.path.dirname(default.__file__)
+    ipconf = os.path.join(ipcdir,'ipython_config.py')
+    return ['--colors=NoColor', '--no-term-title','--no-banner',
+            '--config-file="%s"' % ipconf, '--autocall=0',
+            '--prompt-out=""']
+    
+
+def ipexec(fname, options=None):
+    """Utility to call 'ipython filename'.
+
+    Starts IPython witha minimal and safe configuration to make startup as fast
+    as possible.
+
+    Note that this starts IPython in a subprocess!
+
+    Parameters
+    ----------
+    fname : str
+      Name of file to be executed (should have .py or .ipy extension).
+
+    options : optional, list
+      Extra command-line flags to be passed to IPython.
+
+    Returns
+    -------
+    (stdout, stderr) of ipython subprocess.
+    """
+    if options is None: options = []
+    cmdargs = ' '.join(default_argv() + options)
+    
+    _ip = get_ipython()
+    test_dir = os.path.dirname(__file__)
+    # Find the ipython script from the package we're using, so that the test
+    # suite can be run from the source tree without an installed IPython
+    ipython_package_dir = genutils.get_ipython_package_dir()
+    ipython_script = os.path.join(ipython_package_dir,'scripts','ipython')
+    ipython_cmd = 'python "%s"' % ipython_script
+    # Absolute path for filename
+    full_fname = os.path.join(test_dir, fname)
+    full_cmd = '%s %s "%s"' % (ipython_cmd, cmdargs, full_fname)
+    return genutils.getoutputerror(full_cmd)
+
+
+def ipexec_validate(fname, expected_out, expected_err=None,
+                    options=None):
+    """Utility to call 'ipython filename' and validate output/error.
+
+    This function raises an AssertionError if the validation fails.
+
+    Note that this starts IPython in a subprocess!
+
+    Parameters
+    ----------
+    fname : str
+      Name of the file to be executed (should have .py or .ipy extension).
+
+    expected_out : str
+      Expected stdout of the process.
+
+    expected_err : optional, str
+      Expected stderr of the process.
+
+    options : optional, list
+      Extra command-line flags to be passed to IPython.
+
+    Returns
+    -------
+    None
+    """
+
+    import nose.tools as nt
+    
+    out, err = ipexec(fname)
+    nt.assert_equals(out.strip(), expected_out.strip())
+    if expected_err:
+        nt.assert_equals(err.strip(), expected_err.strip())
+
+
+class TempFileMixin(object):
+    """Utility class to create temporary Python/IPython files.
+
+    Meant as a mixin class for test cases."""
+    
+    def mktmp(self, src, ext='.py'):
+        """Make a valid python temp file."""
+        fname, f = temp_pyfile(src, ext)
+        self.tmpfile = f
+        self.fname = fname
+
+    def teardown(self):
+        if hasattr(self, 'tmpfile'):
+            # If the tmpfile wasn't made because of skipped tests, like in
+            # win32, there's nothing to cleanup.
+            self.tmpfile.close()
+            try:
+                os.unlink(self.fname)
+            except:
+                # On Windows, even though we close the file, we still can't
+                # delete it.  I have no clue why
+                pass
+

IPython/utils/genutils.py

@@ -46,11 +46,6 b' from IPython.utils import platutils'
 from IPython.utils.generics import result_display
 from IPython.external.path import path
 
-try:
-    set
-except:
-    from sets import Set as set
-
 
 #****************************************************************************
 # Exceptions
@@ -81,6 +76,10 b' class IOStream:'
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
+    def writeln(self, data):
+        self.write(data)
+        self.write('\n')        
+                
     def close(self):
         pass
 
@@ -110,6 +109,65 b" if sys.platform == 'win32' and readline.have_readline:"
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
 
 
+class Tee(object):
+    """A class to duplicate an output stream to stdout/err.
+
+    This works in a manner very similar to the Unix 'tee' command.
+
+    When the object is closed or deleted, it closes the original file given to
+    it for duplication.
+    """
+    # Inspired by:
+    # http://mail.python.org/pipermail/python-list/2007-May/442737.html
+
+    def __init__(self, file, mode=None, channel='stdout'):
+        """Construct a new Tee object.
+
+        Parameters
+        ----------
+        file : filename or open filehandle (writable)
+          File that will be duplicated
+
+        mode : optional, valid mode for open().
+          If a filename was give, open with this mode.
+
+        channel : str, one of ['stdout', 'stderr']  
+        """
+        if channel not in ['stdout', 'stderr']:
+            raise ValueError('Invalid channel spec %s' % channel)
+        
+        if hasattr(file, 'write') and hasattr(file, 'seek'):
+            self.file = file
+        else:
+            self.file = open(name, mode)
+        self.channel = channel
+        self.ostream = getattr(sys, channel)
+        setattr(sys, channel, self)
+        self._closed = False
+
+    def close(self):
+        """Close the file and restore the channel."""
+        self.flush()
+        setattr(sys, self.channel, self.ostream)
+        self.file.close()
+        self._closed = True
+
+    def write(self, data):
+        """Write data to both channels."""
+        self.file.write(data)
+        self.ostream.write(data)
+        self.ostream.flush()
+
+    def flush(self):
+        """Flush both channels."""
+        self.file.flush()
+        self.ostream.flush()
+
+    def __del__(self):
+        if not self._closed:
+            self.close()
+        
+
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
@@ -400,7 +458,8 b" def getoutput(cmd,verbose=0,debug=0,header='',split=0):"
 
     if verbose or debug: print header+cmd
     if not debug:
-        output = os.popen(cmd).read()
+        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
+        output = pipe.read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
@@ -427,7 +486,13 b" def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):"
         else:
             return '',''
     if not debug:
-        pin,pout,perr = os.popen3(cmd)
+        p = subprocess.Popen(cmd, shell=True,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             close_fds=True)
+        pin, pout, perr = (p.stdin, p.stdout, p.stderr)
+
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
@@ -553,15 +618,21 b' def filefind(filename, path_dirs=None):'
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
+    
+    # If paths are quoted, abspath gets confused, strip them...
+    filename = filename.strip('"').strip("'")
+    # If the input is an absolute path, just check it exists
+    if os.path.isabs(filename) and os.path.isfile(filename):
+        return filename
+        
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, basestring):
         path_dirs = (path_dirs,)
+        
     for path in path_dirs:
         if path == '.': path = os.getcwd()
-        testname = os.path.expandvars(
-                       os.path.expanduser(
-                           os.path.join(path, filename)))
+        testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
     raise IOError("File does not exist in any "
@@ -717,10 +788,18 b' class HomeDirError(Error):'
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
+    * On POSIX, we try $HOME.
+    * On Windows we try:
+      - %HOME%: rare, but some people with unix-like setups may have defined it
+      - %HOMESHARE%
+      - %HOMEDRIVE\%HOMEPATH%
+      - %USERPROFILE%
+      - Registry hack
+    * On Dos C:\
+ 
     Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
+    raised for all other OSes.
+    """
 
     isdir = os.path.isdir
     env = os.environ
@@ -736,93 +815,100 b' def get_home_dir():'
         root=os.path.abspath(root).rstrip('\\')
         if isdir(os.path.join(root, '_ipython')):
             os.environ["IPYKITROOT"] = root
-        return root
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except KeyError:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
+        return root.decode(sys.getfilesystemencoding())
+
+    if os.name == 'posix':
+        # Linux, Unix, AIX, OS X
+        try:
+            homedir = env['HOME']
+        except KeyError:
+            raise HomeDirError('Undefined $HOME, IPython cannot proceed.')
         else:
-            raise HomeDirError,'support for your operating system not implemented.'
+            return homedir.decode(sys.getfilesystemencoding())
+    elif os.name == 'nt':
+        # Now for win9x, XP, Vista, 7?
+        # For some strange reason all of these return 'nt' for os.name.
+        # First look for a network home directory. This will return the UNC
+        # path (\\server\\Users\%username%) not the mapped path (Z:\). This
+        # is needed when running IPython on cluster where all paths have to 
+        # be UNC.
+        try:
+            # A user with a lot of unix tools in win32 may have defined $HOME,
+            # honor it if it exists, but otherwise let the more typical
+            # %HOMESHARE% variable be used.
+            homedir = env.get('HOME')
+            if homedir is None:
+                homedir = env['HOMESHARE']
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now look for a local home directory
+        try:
+            homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Now the users profile directory
+        try:
+            homedir = os.path.join(env['USERPROFILE'])
+        except KeyError:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # Use the registry to get the 'My Documents' folder.
+        try:
+            import _winreg as wreg
+            key = wreg.OpenKey(
+                wreg.HKEY_CURRENT_USER,
+                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+            )
+            homedir = wreg.QueryValueEx(key,'Personal')[0]
+            key.Close()
+        except:
+            pass
+        else:
+            if isdir(homedir):
+                return homedir.decode(sys.getfilesystemencoding())
+
+        # If all else fails, raise HomeDirError
+        raise HomeDirError('No valid home directory could be found')
+    elif os.name == 'dos':
+        # Desperate, may do absurd things in classic MacOS. May work under DOS.
+        return 'C:\\'.decode(sys.getfilesystemencoding())
+    else:
+        raise HomeDirError('No valid home directory could be found for your OS')
 
 
 def get_ipython_dir():
     """Get the IPython directory for this platform and user.
     
     This uses the logic in `get_home_dir` to find the home directory
-    and the adds either .ipython or _ipython to the end of the path.
+    and the adds .ipython to the end of the path.
     """
-    if os.name == 'posix':
-         ipdir_def = '.ipython'
-    else:
-         ipdir_def = '_ipython'
+    ipdir_def = '.ipython'
     home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir, ipdir_def)))
+    #import pdb; pdb.set_trace()  # dbg
+    ipdir = os.environ.get(
+        'IPYTHON_DIR', os.environ.get(
+            'IPYTHONDIR', os.path.join(home_dir, ipdir_def)
+        )
+    )
     return ipdir.decode(sys.getfilesystemencoding())
 
-def get_security_dir():
-    """Get the IPython security directory.
-    
-    This directory is the default location for all security related files,
-    including SSL/TLS certificates and FURL files.
-    
-    If the directory does not exist, it is created with 0700 permissions.
-    If it exists, permissions are set to 0700.
-    """
-    security_dir = os.path.join(get_ipython_dir(), 'security')
-    if not os.path.isdir(security_dir):
-        os.mkdir(security_dir, 0700)
-    else:
-        os.chmod(security_dir, 0700)
-    return security_dir
 
-def get_log_dir():
-    """Get the IPython log directory.
-    
-    If the log directory does not exist, it is created.
-    """
-    log_dir = os.path.join(get_ipython_dir(), 'log')
-    if not os.path.isdir(log_dir):
-        os.mkdir(log_dir, 0777)
-    return log_dir
+def get_ipython_package_dir():
+    """Get the base directory where IPython itself is installed."""
+    ipdir = os.path.dirname(IPython.__file__)
+    return ipdir.decode(sys.getfilesystemencoding())
+
 
 #****************************************************************************
 # strings and text
@@ -1738,18 +1824,27 b' def extract_vars_above(*names):'
     callerNS = sys._getframe(2).f_locals
     return dict((k,callerNS[k]) for k in names)
 
-def shexp(s):
+def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
     
        In [2]: os.environ['FOO']='test'
 
-       In [3]: shexp('variable FOO is $FOO')
+       In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
-    return os.path.expandvars(os.path.expanduser(s))
-    
+    # This is a pretty subtle hack. When expand user is given a UNC path
+    # on Windows (\\server\share$\%username%), os.path.expandvars, removes
+    # the $ to get (\\server\share\%username%). I think it considered $
+    # alone an empty var. But, we need the $ to remains there (it indicates
+    # a hidden share).
+    if os.name=='nt':
+        s = s.replace('$\\', 'IPYTHON_TEMP')
+    s = os.path.expandvars(os.path.expanduser(s))
+    if os.name=='nt':
+        s = s.replace('IPYTHON_TEMP', '$\\')
+    return s
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings

IPython/utils/notification.py

@@ -1,119 +1,137 b''
+#!/usr/bin/env python
 # encoding: utf-8
-
-"""The IPython Core Notification Center.
+"""
+The IPython Core Notification Center.
 
 See docs/source/development/notification_blueprint.txt for an overview of the
 notification module.
+
+Authors:
+
+* Barry Wark
+* Brian Granger
 """
 
-__docformat__ = "restructuredtext en"
-                                                                              
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                           
-#                                                                             
-#  Distributed under the terms of the BSD License.  The full license is in    
-#  the file COPYING, distributed as part of this software.                    
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
 #-----------------------------------------------------------------------------
 
-# Tell nose to skip the testing of this module
-__test__ = {}
+
+class NotificationError(Exception):
+    pass
+
 
 class NotificationCenter(object):
-    """Synchronous notification center
+    """Synchronous notification center.
     
     Examples
     --------
-    >>> import IPython.kernel.core.notification as notification
-    >>> def callback(theType, theSender, args={}):
-    ...     print theType,theSender,args
-    ...     
-    >>> notification.sharedCenter.add_observer(callback, 'NOTIFICATION_TYPE', None)
-    >>> notification.sharedCenter.post_notification('NOTIFICATION_TYPE', object()) # doctest:+ELLIPSIS
-    NOTIFICATION_TYPE ...
-        
+    Here is a simple example of how to use this::
+
+        import IPython.kernel.core.notification as notification
+        def callback(ntype, theSender, args={}):
+            print ntype,theSender,args
+
+        notification.sharedCenter.add_observer(callback, 'NOTIFICATION_TYPE', None)
+        notification.sharedCenter.post_notification('NOTIFICATION_TYPE', object()) # doctest:+ELLIPSIS
+        NOTIFICATION_TYPE ...
     """
     def __init__(self):
         super(NotificationCenter, self).__init__()
         self._init_observers()
-    
-    
+
     def _init_observers(self):
         """Initialize observer storage"""
-        
+
         self.registered_types = set() #set of types that are observed
         self.registered_senders = set() #set of senders that are observed
         self.observers = {} #map (type,sender) => callback (callable)
-    
-    
-    def post_notification(self, theType, sender, **kwargs):
-        """Post notification (type,sender,**kwargs) to all registered
-        observers.
 
-        Implementation notes:
+    def post_notification(self, ntype, sender, *args, **kwargs):
+        """Post notification to all registered observers.
+
+        The registered callback will be called as::
+
+            callback(ntype, sender, *args, **kwargs)
+
+        Parameters
+        ----------
+        ntype : hashable
+            The notification type.
+        sender : hashable
+            The object sending the notification.
+        *args : tuple
+            The positional arguments to be passed to the callback.
+        **kwargs : dict
+            The keyword argument to be passed to the callback.
 
+        Notes
+        -----
         * If no registered observers, performance is O(1).
         * Notificaiton order is undefined.
         * Notifications are posted synchronously.
         """
-        
-        if(theType==None or sender==None):
-            raise Exception("NotificationCenter.post_notification requires \
-                type and sender.")
-        
+
+        if(ntype==None or sender==None):
+            raise NotificationError(
+                "Notification type and sender are required.")
+
         # If there are no registered observers for the type/sender pair
-        if((theType not in self.registered_types and 
+        if((ntype not in self.registered_types and 
                 None not in self.registered_types) or
             (sender not in self.registered_senders and 
                 None not in self.registered_senders)):
             return
-        
-        for o in self._observers_for_notification(theType, sender):
-            o(theType, sender, args=kwargs)
-    
-        
-    def _observers_for_notification(self, theType, sender):
+
+        for o in self._observers_for_notification(ntype, sender):
+            o(ntype, sender, *args, **kwargs)
+
+    def _observers_for_notification(self, ntype, sender):
         """Find all registered observers that should recieve notification"""
-        
+
         keys = (
-                    (theType,sender),
-                    (theType, None),
-                    (None, sender),  
-                    (None,None)
-                )
-        
-        
+                   (ntype,sender),
+                   (ntype, None),
+                   (None, sender),
+                   (None,None)
+               )
+
         obs = set()
         for k in keys:
             obs.update(self.observers.get(k, set()))
-        
+
         return obs
-    
-    
-    def add_observer(self, callback, theType, sender):
+
+    def add_observer(self, callback, ntype, sender):
         """Add an observer callback to this notification center.
         
         The given callback will be called upon posting of notifications of
-        the given type/sender and will receive any additional kwargs passed
+        the given type/sender and will receive any additional arguments passed
         to post_notification.
         
         Parameters
         ----------
-        observerCallback : callable
-            Callable. Must take at least two arguments::
-                observerCallback(type, sender, args={})
-        
-        theType : hashable
+        callback : callable
+            The callable that will be called by :meth:`post_notification`
+            as ``callback(ntype, sender, *args, **kwargs)
+        ntype : hashable
             The notification type. If None, all notifications from sender
             will be posted.
-        
         sender : hashable
-            The notification sender. If None, all notifications of theType
+            The notification sender. If None, all notifications of ntype
             will be posted.
         """
         assert(callback != None)
-        self.registered_types.add(theType)
+        self.registered_types.add(ntype)
         self.registered_senders.add(sender)
-        self.observers.setdefault((theType,sender), set()).add(callback)
+        self.observers.setdefault((ntype,sender), set()).add(callback)
     
     def remove_all_observers(self):
         """Removes all observers from this notification center"""
@@ -122,4 +140,4 b' class NotificationCenter(object):'
     
 
 
-sharedCenter = NotificationCenter()
+shared_center = NotificationCenter()

IPython/utils/platutils.py

@@ -79,7 +79,7 b' def find_cmd(cmd):'
         return sys.executable
     try:
         path = _platutils.find_cmd(cmd)
-    except:
+    except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':

IPython/utils/tests/test_genutils.py

@@ -20,7 +20,9 b' import os'
 import shutil
 import sys
 import tempfile
+import unittest
 
+from cStringIO import StringIO
 from os.path import join, abspath, split
 
 # third-party
@@ -32,6 +34,7 b' from nose.tools import raises'
 # Our own
 import IPython
 from IPython.utils import genutils
+from IPython.testing import decorators as dec
 from IPython.testing.decorators import skipif, skip_if_not_win32
 
 # Platform-dependent imports
@@ -52,7 +55,7 b' env = os.environ'
 TEST_FILE_PATH = split(abspath(__file__))[0]
 TMP_TEST_DIR = tempfile.mkdtemp()
 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
-IP_TEST_DIR = join(HOME_TEST_DIR,'_ipython')
+IP_TEST_DIR = join(HOME_TEST_DIR,'.ipython')
 #
 # Setup/teardown functions/decorators
 #
@@ -85,18 +88,17 b' def setup_environment():'
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
-    oldstuff = (env.copy(), os.name, genutils.get_home_dir, IPython.__file__,)
+    oldstuff = (env.copy(), os.name, genutils.get_home_dir, IPython.__file__)
 
     if os.name == 'nt':
         platformstuff = (wreg.OpenKey, wreg.QueryValueEx,)
 
-    if 'IPYTHONDIR' in env:
-        del env['IPYTHONDIR']
-
+ 
 def teardown_environment():
     """Restore things that were remebered by the setup_environment function
     """
     (oldenv, os.name, genutils.get_home_dir, IPython.__file__,) = oldstuff
+        
     for key in env.keys():
         if key not in oldenv:
             del env[key]
@@ -107,7 +109,7 b' def teardown_environment():'
         (wreg.OpenKey, wreg.QueryValueEx,) = platformstuff
 
 # Build decorator that uses the setup_environment/setup_environment
-with_enivronment = with_setup(setup_environment, teardown_environment)
+with_environment = with_setup(setup_environment, teardown_environment)
 
 
 #
@@ -115,7 +117,7 b' with_enivronment = with_setup(setup_environment, teardown_environment)'
 #
 
 @skip_if_not_win32
-@with_enivronment
+@with_environment
 def test_get_home_dir_1():
     """Testcase for py2exe logic, un-compressed lib
     """
@@ -128,7 +130,7 b' def test_get_home_dir_1():'
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
     
 @skip_if_not_win32
-@with_enivronment
+@with_environment
 def test_get_home_dir_2():
     """Testcase for py2exe logic, compressed lib
     """
@@ -139,14 +141,14 b' def test_get_home_dir_2():'
     home_dir = genutils.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR).lower())
 
-@with_enivronment
+@with_environment
 def test_get_home_dir_3():
     """Testcase $HOME is set, then use its value as home directory."""
     env["HOME"] = HOME_TEST_DIR
     home_dir = genutils.get_home_dir()
     nt.assert_equal(home_dir, env["HOME"])
 
-@with_enivronment
+@with_environment
 def test_get_home_dir_4():
     """Testcase $HOME is not set, os=='poix'. 
     This should fail with HomeDirError"""
@@ -156,7 +158,7 b' def test_get_home_dir_4():'
     nt.assert_raises(genutils.HomeDirError, genutils.get_home_dir)
         
 @skip_if_not_win32
-@with_enivronment
+@with_environment
 def test_get_home_dir_5():
     """Testcase $HOME is not set, os=='nt' 
     env['HOMEDRIVE'],env['HOMEPATH'] points to path."""
@@ -169,7 +171,7 b' def test_get_home_dir_5():'
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 @skip_if_not_win32
-@with_enivronment
+@with_environment
 def test_get_home_dir_6():
     """Testcase $HOME is not set, os=='nt' 
     env['HOMEDRIVE'],env['HOMEPATH'] do not point to path.
@@ -186,14 +188,16 b' def test_get_home_dir_6():'
 
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
-@with_enivronment
+@with_environment
 def test_get_home_dir_7():
-    """Testcase $HOME is not set, os=='nt' 
-    env['HOMEDRIVE'],env['HOMEPATH'], env['USERPROFILE'] missing
+    """Testcase $HOME is not set, os=='nt'
+    
+    env['HOMEDRIVE'],env['HOMEPATH'], env['USERPROFILE'] and others missing
     """
     os.name = 'nt'
-    if 'HOME' in env: del env['HOME']
-    if 'HOMEDRIVE' in env: del env['HOMEDRIVE']
+    # Remove from stub environment all keys that may be set
+    for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
+        env.pop(key, None)
 
     #Stub windows registry functions
     def OpenKey(x, y):
@@ -214,47 +218,23 b' def test_get_home_dir_7():'
 # Tests for get_ipython_dir
 #
 
-@with_enivronment
+@with_environment
 def test_get_ipython_dir_1():
     """test_get_ipython_dir_1, Testcase to see if we can call get_ipython_dir without Exceptions."""
-    env['IPYTHONDIR'] = "someplace/.ipython"
+    env['IPYTHON_DIR'] = "someplace/.ipython"
     ipdir = genutils.get_ipython_dir()
-    nt.assert_equal(ipdir, os.path.abspath("someplace/.ipython"))
+    nt.assert_equal(ipdir, "someplace/.ipython")
 
 
-@with_enivronment
+@with_environment
 def test_get_ipython_dir_2():
     """test_get_ipython_dir_2, Testcase to see if we can call get_ipython_dir without Exceptions."""
     genutils.get_home_dir = lambda : "someplace"
     os.name = "posix"
+    env.pop('IPYTHON_DIR', None)
+    env.pop('IPYTHONDIR', None)
     ipdir = genutils.get_ipython_dir()
-    nt.assert_equal(ipdir, os.path.abspath(os.path.join("someplace", ".ipython")))
-
-@with_enivronment
-def test_get_ipython_dir_3():
-    """test_get_ipython_dir_3, Testcase to see if we can call get_ipython_dir without Exceptions."""
-    genutils.get_home_dir = lambda : "someplace"
-    os.name = "nt"
-    ipdir = genutils.get_ipython_dir()
-    nt.assert_equal(ipdir, os.path.abspath(os.path.join("someplace", "_ipython")))
-
-#
-# Tests for get_security_dir
-#
-
-@with_enivronment
-def test_get_security_dir():
-    """Testcase to see if we can call get_security_dir without Exceptions."""
-    sdir = genutils.get_security_dir()
-
-#
-# Tests for get_log_dir
-#
-
-@with_enivronment
-def test_get_log_dir():
-    """Testcase to see if we can call get_log_dir without Exceptions."""
-    sdir = genutils.get_log_dir()
+    nt.assert_equal(ipdir, os.path.join("someplace", ".ipython"))
 
 #
 # Tests for popkey
@@ -304,3 +284,43 b' def test_filefind():'
     alt_dirs = genutils.get_ipython_dir()
     t = genutils.filefind(f.name,alt_dirs)
     print 'found:',t
+
+
+def test_get_ipython_package_dir():
+    ipdir = genutils.get_ipython_package_dir()
+    nt.assert_true(os.path.isdir(ipdir))
+
+
+def test_tee_simple():
+    "Very simple check with stdout only"
+    chan = StringIO()
+    text = 'Hello'
+    tee = genutils.Tee(chan, channel='stdout')
+    print >> chan, text,
+    nt.assert_equal(chan.getvalue(), text)
+
+
+class TeeTestCase(dec.ParametricTestCase):
+
+    def tchan(self, channel, check='close'):
+        trap = StringIO()
+        chan = StringIO()
+        text = 'Hello'
+        
+        std_ori = getattr(sys, channel)
+        setattr(sys, channel, trap)
+
+        tee = genutils.Tee(chan, channel=channel)
+        print >> chan, text,
+        setattr(sys, channel, std_ori)
+        trap_val = trap.getvalue()
+        nt.assert_equals(chan.getvalue(), text)
+        if check=='close':
+            tee.close()
+        else:
+            del tee
+
+    def test(self):
+        for chan in ['stdout', 'stderr']:
+            for check in ['close', 'del']:
+                yield self.tchan(chan, check)

IPython/utils/tests/test_notification.py

@@ -13,135 +13,129 b''
 # Imports
 #-----------------------------------------------------------------------------
 
-# Tell nose to skip this module
-__test__ = {}
+import unittest
 
-from twisted.trial import unittest
-import IPython.kernel.core.notification as notification
+from IPython.utils.notification import (
+    NotificationCenter,
+    NotificationError,
+    shared_center
+)
 
 #-----------------------------------------------------------------------------
 # Support Classes
 #-----------------------------------------------------------------------------
 
+
 class Observer(object):
-    """docstring for Observer"""
-    def __init__(self, expectedType, expectedSender, 
-                    center=notification.sharedCenter, **kwargs):
+
+    def __init__(self, expected_ntype, expected_sender, 
+                    center=shared_center, *args, **kwargs):
         super(Observer, self).__init__()
-        self.expectedType = expectedType
-        self.expectedSender = expectedSender
-        self.expectedKwArgs = kwargs
+        self.expected_ntype = expected_ntype
+        self.expected_sender = expected_sender
+        self.expected_args = args
+        self.expected_kwargs = kwargs
         self.recieved = False
         center.add_observer(self.callback, 
-                            self.expectedType, 
-                            self.expectedSender)
+                            self.expected_ntype, 
+                            self.expected_sender)
     
-    def callback(self, theType, sender, args={}):
-        """callback"""
-        
-        assert(theType == self.expectedType or
-                self.expectedType == None)
-        assert(sender == self.expectedSender or
-                self.expectedSender == None)
-        assert(args == self.expectedKwArgs)
+    def callback(self, ntype, sender, *args, **kwargs):
+        assert(ntype == self.expected_ntype or
+                self.expected_ntype == None)
+        assert(sender == self.expected_sender or
+                self.expected_sender == None)
+        assert(args == self.expected_args)
+        assert(kwargs == self.expected_kwargs)
         self.recieved = True
     
     def verify(self):
-        """verify"""
-        
         assert(self.recieved)
     
     def reset(self):
-        """reset"""
-        
         self.recieved = False
 
 
 class Notifier(object):
-    """docstring for Notifier"""
-    def __init__(self, theType, **kwargs):
+
+    def __init__(self, ntype, **kwargs):
         super(Notifier, self).__init__()
-        self.theType = theType
+        self.ntype = ntype
         self.kwargs = kwargs
-    
-    def post(self, center=notification.sharedCenter):
-        """fire"""
-        
-        center.post_notification(self.theType, self,
+
+    def post(self, center=shared_center):
+
+        center.post_notification(self.ntype, self,
             **self.kwargs)
 
+
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
+
 class NotificationTests(unittest.TestCase):
-    """docstring for NotificationTests"""
-    
+
     def tearDown(self):
-        notification.sharedCenter.remove_all_observers()
+        shared_center.remove_all_observers()
     
     def test_notification_delivered(self):
         """Test that notifications are delivered"""
-        expectedType = 'EXPECTED_TYPE'
-        sender = Notifier(expectedType)
-        observer = Observer(expectedType, sender)
-        
+
+        expected_ntype = 'EXPECTED_TYPE'
+        sender = Notifier(expected_ntype)
+        observer = Observer(expected_ntype, sender)
+
         sender.post()
-        
         observer.verify()
-    
+
     def test_type_specificity(self):
         """Test that observers are registered by type"""
-        
-        expectedType = 1
-        unexpectedType = "UNEXPECTED_TYPE"
-        sender = Notifier(expectedType)
-        unexpectedSender = Notifier(unexpectedType)
-        observer = Observer(expectedType, sender)
-    
+
+        expected_ntype = 1
+        unexpected_ntype = "UNEXPECTED_TYPE"
+        sender = Notifier(expected_ntype)
+        unexpected_sender = Notifier(unexpected_ntype)
+        observer = Observer(expected_ntype, sender)
+
         sender.post()
-        unexpectedSender.post()
-    
+        unexpected_sender.post()
         observer.verify()
-    
+
     def test_sender_specificity(self):
         """Test that observers are registered by sender"""
         
-        expectedType = "EXPECTED_TYPE"
-        sender1 = Notifier(expectedType)
-        sender2 = Notifier(expectedType)
-        observer = Observer(expectedType, sender1)
-        
+        expected_ntype = "EXPECTED_TYPE"
+        sender1 = Notifier(expected_ntype)
+        sender2 = Notifier(expected_ntype)
+        observer = Observer(expected_ntype, sender1)
+
         sender1.post()
         sender2.post()
-        
+
         observer.verify()
-    
+
     def test_remove_all_observers(self):
         """White-box test for remove_all_observers"""
-        
+
         for i in xrange(10):
-            Observer('TYPE', None, center=notification.sharedCenter)
-        
-        self.assert_(len(notification.sharedCenter.observers[('TYPE',None)]) >= 10, 
+            Observer('TYPE', None, center=shared_center)
+
+        self.assert_(len(shared_center.observers[('TYPE',None)]) >= 10, 
             "observers registered")
-        
-        notification.sharedCenter.remove_all_observers()
-        
-        self.assert_(len(notification.sharedCenter.observers) == 0, "observers removed")
+
+        shared_center.remove_all_observers()
+        self.assert_(len(shared_center.observers) == 0, "observers removed")
 
     def test_any_sender(self):
-        """test_any_sender"""
-        
-        expectedType = "EXPECTED_TYPE"
-        sender1 = Notifier(expectedType)
-        sender2 = Notifier(expectedType)
-        observer = Observer(expectedType, None)
-        
-        
+        expected_ntype = "EXPECTED_TYPE"
+        sender1 = Notifier(expected_ntype)
+        sender2 = Notifier(expected_ntype)
+        observer = Observer(expected_ntype, None)
+
         sender1.post()
         observer.verify()
-        
+
         observer.reset()
         sender2.post()
         observer.verify()
@@ -152,10 +146,9 b' class NotificationTests(unittest.TestCase):'
         
         for i in xrange(10):
             Observer("UNRELATED_TYPE", None)
-        
+
         o = Observer('EXPECTED_TYPE', None)
-        
-        notification.sharedCenter.post_notification('EXPECTED_TYPE', self)
-        
+        shared_center.post_notification('EXPECTED_TYPE', self)
         o.verify()
 
+

IPython/utils/tests/test_platutils.py

@@ -1,4 +1,3 b''
-#!/usr/bin/env python
 # encoding: utf-8
 """
 Tests for platutils.py
@@ -31,29 +30,43 b' def test_find_cmd_python():'
     """Make sure we find sys.exectable for python."""
     nt.assert_equals(find_cmd('python'), sys.executable)
 
+    
 @dec.skip_win32
-def test_find_cmd():
+def test_find_cmd_ls():
     """Make sure we can find the full path to ls."""
     path = find_cmd('ls')
     nt.assert_true(path.endswith('ls'))
 
[e-mail]_if_not_win32
-def test_find_cmd():
+    
+def has_pywin32():
+    try:
+        import win32api
+    except ImportError:
+        return False
+    return True
+
+
[e-mail](has_pywin32, "This test requires win32api to run")
+def test_find_cmd_pythonw():
     """Try to find pythonw on Windows."""
     path = find_cmd('pythonw')
     nt.assert_true(path.endswith('pythonw.exe'))
 
+
[e-mail](lambda : sys.platform != 'win32' or has_pywin32(),
+            "This test runs on posix or in win32 with win32api installed")
 def test_find_cmd_fail():
     """Make sure that FindCmdError is raised if we can't find the cmd."""
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
+    
 @dec.skip_if_not_win32
 def test_get_long_path_name_win32():
     p = get_long_path_name('c:\\docume~1')
     nt.assert_equals(p,u'c:\\Documents and Settings') 
-   
+
+    
 @dec.skip_win32
 def test_get_long_path_name():
     p = get_long_path_name('/usr/local')
     nt.assert_equals(p,'/usr/local')
-

IPython/utils/traitlets.py

@@ -160,6 +160,26 b' class _SimpleTest:'
         return self.__repr__()
 
 
+def getmembers(object, predicate=None):
+    """A safe version of inspect.getmembers that handles missing attributes.
+
+    This is useful when there are descriptor based attributes that for 
+    some reason raise AttributeError even though they exist.  This happens
+    in zope.inteface with the __provides__ attribute.
+    """
+    results = []
+    for key in dir(object):
+        try:
+            value = getattr(object, key)
+        except AttributeError:
+            pass
+        else:
+            if not predicate or predicate(value):
+                results.append((key, value))
+    results.sort()
+    return results
+
+
 #-----------------------------------------------------------------------------
 # Base TraitType for all traits
 #-----------------------------------------------------------------------------
@@ -325,6 +345,9 b' class MetaHasTraits(type):'
         This instantiates all TraitTypes in the class dict and sets their
         :attr:`name` attribute.
         """
+        # print "MetaHasTraitlets (mcls, name): ", mcls, name
+        # print "MetaHasTraitlets (bases): ", bases
+        # print "MetaHasTraitlets (classdict): ", classdict
         for k,v in classdict.iteritems():
             if isinstance(v, TraitType):
                 v.name = k
@@ -363,9 +386,17 b' class HasTraits(object):'
         # Here we tell all the TraitType instances to set their default
         # values on the instance. 
         for key in dir(cls):
-            value = getattr(cls, key)
-            if isinstance(value, TraitType):
-                value.instance_init(inst)
+            # Some descriptors raise AttributeError like zope.interface's
+            # __provides__ attributes even though they exist.  This causes
+            # AttributeErrors even though they are listed in dir(cls).
+            try:
+                value = getattr(cls, key)
+            except AttributeError:
+                pass
+            else:
+                if isinstance(value, TraitType):
+                    value.instance_init(inst)
+
         return inst
 
     # def __init__(self):
@@ -484,7 +515,7 b' class HasTraits(object):'
         exists, but has any value.  This is because get_metadata returns
         None if a metadata key doesn't exist.
         """
-        traits = dict([memb for memb in inspect.getmembers(self.__class__) if \
+        traits = dict([memb for memb in getmembers(self.__class__) if \
                      isinstance(memb[1], TraitType)])
 
         if len(metadata) == 0:

docs/Makefile

@@ -14,6 +14,8 b' ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SRCDIR)'
 
 .PHONY: help clean html web pickle htmlhelp latex changes linkcheck api
 
+default: html
+
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html      to make standalone HTML files"

docs/autogen_api.py

@@ -29,11 +29,34 b" if __name__ == '__main__':"
                                         r'\.frontend',
                                         r'\.gui'
                                         ]
+
     docwriter.module_skip_patterns += [ r'\.core\.fakemodule',
+                                        
+                                        # XXX These need fixing, disabling for
+                                        # now but we need to figure out why
+                                        # they are breaking.  Error from sphinx
+                                        # for each group copied below
+                                        
+                                        # AttributeError: __abstractmethods__
+                                        r'\.core\.component',
+                                        r'\.utils\.traitlets',
+        
+                                        # AttributeError: __provides__
+                                        r'\.kernel\.clusterdir',
+                                        r'\.kernel\.configobjfactory',
+                                        r'\.kernel\.fcutil',
+                                        r'\.kernel\.ipcontrollerapp',
+                                        r'\.kernel\.launcher',
+                                        r'\.kernel\.task',
+                                        r'\.kernel\.winhpcjob',
+                                        r'\.testing\.util',
+    
+                                        # Keeping these disabled is OK
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',
                                         r'\.frontend\.process\.winprocess',
+                                        r'\.Shell',
                                         ]
     docwriter.write_api_docs(outdir)
     docwriter.write_index(outdir, 'gen',

docs/emacs/ipython.el

@@ -311,7 +311,7 b' gets converted to:'
         (replace-match "" t nil)))))
 
 (defvar ipython-completion-command-string
-  "print ';'.join(__IP.Completer.all_completions('%s')) #PYTHON-MODE SILENT\n"
+  "print(';'.join(__IP.Completer.all_completions('%s'))) #PYTHON-MODE SILENT\n"
   "The string send to ipython to query for all possible completions")
 
 

docs/examples/kernel/mcdriver.py

@@ -1,71 +1,71 b''
 #!/usr/bin/env python
-# encoding: utf-8
 """Run a Monte-Carlo options pricer in parallel."""
 
 from IPython.kernel import client
-import numpy as N
-from mcpricer import MCOptionPricer
+import numpy as np
+from mcpricer import price_options
 
+# The MultiEngineClient is used to setup the calculation and works with all
+# engine.
+mec = client.MultiEngineClient(profile='mycluster')
 
-tc = client.TaskClient()
-rc = client.MultiEngineClient()
+# The TaskClient is an interface to the engines that provides dynamic load 
+# balancing at the expense of not knowing which engine will execute the code.
+tc = client.TaskClient(profile='mycluster')
 
-# Initialize the common code on the engines
-rc.run('mcpricer.py')
+# Initialize the common code on the engines. This Python module has the
+# price_options function that prices the options.
+mec.run('mcpricer.py')
 
-# Push the variables that won't change 
-#(stock print, interest rate, days and MC paths)
-rc.push(dict(S=100.0, r=0.05, days=260, paths=10000))
-
-task_string = """\
-op = MCOptionPricer(S,K,sigma,r,days,paths)
-op.run()
-vp, ap, vc, ac = op.vanilla_put, op.asian_put, op.vanilla_call, op.asian_call
-"""
+# Define the function that will make up our tasks. We basically want to
+# call the price_options function with all but two arguments (K, sigma)
+# fixed.
+def my_prices(K, sigma):
+    S = 100.0
+    r = 0.05
+    days = 260
+    paths = 100000
+    return price_options(S, K, sigma, r, days, paths)
 
 # Create arrays of strike prices and volatilities
-K_vals = N.linspace(90.0,100.0,5)
-sigma_vals = N.linspace(0.0, 0.2,5)
+nK = 10
+nsigma = 10
+K_vals = np.linspace(90.0, 100.0, nK)
+sigma_vals = np.linspace(0.1, 0.4, nsigma)
 
-# Submit tasks
+# Submit tasks to the TaskClient for each (K, sigma) pair as a MapTask.
+# The MapTask simply applies a function (my_prices) to the arguments:
+# my_prices(K, sigma) and returns the result.
 taskids = []
 for K in K_vals:
     for sigma in sigma_vals:
-        t = client.StringTask(task_string, 
-            push=dict(sigma=sigma,K=K),
-            pull=('vp','ap','vc','ac','sigma','K'))
+        t = client.MapTask(my_prices, args=(K, sigma))
         taskids.append(tc.run(t))
 
-print "Submitted tasks: ", taskids
+print "Submitted tasks: ", len(taskids)
 
-# Block until tasks are completed
+# Block until all tasks are completed.
 tc.barrier(taskids)
 
-# Get the results
+# Get the results using TaskClient.get_task_result.
 results = [tc.get_task_result(tid) for tid in taskids]
 
-# Assemble the result
-vc = N.empty(K_vals.shape[0]*sigma_vals.shape[0],dtype='float64')
-vp = N.empty(K_vals.shape[0]*sigma_vals.shape[0],dtype='float64')
-ac = N.empty(K_vals.shape[0]*sigma_vals.shape[0],dtype='float64')
-ap = N.empty(K_vals.shape[0]*sigma_vals.shape[0],dtype='float64')
-for i, tr in enumerate(results):
-    ns = tr.ns
-    vc[i] = ns.vc
-    vp[i] = ns.vp
-    ac[i] = ns.ac
-    ap[i] = ns.ap
-vc.shape = (K_vals.shape[0],sigma_vals.shape[0])
-vp.shape = (K_vals.shape[0],sigma_vals.shape[0])
-ac.shape = (K_vals.shape[0],sigma_vals.shape[0])
-ap.shape = (K_vals.shape[0],sigma_vals.shape[0])
-
+# Assemble the result into a structured NumPy array.
+prices = np.empty(nK*nsigma,
+    dtype=[('ecall',float),('eput',float),('acall',float),('aput',float)]
+)
+for i, price_tuple in enumerate(results):
+    prices[i] = price_tuple
+prices.shape = (nK, nsigma)
+K_vals, sigma_vals = np.meshgrid(K_vals, sigma_vals)
 
-def plot_options(K_vals, sigma_vals, prices):
-    """Make a contour plot of the option prices."""
-    import pylab
-    pylab.contourf(sigma_vals, K_vals, prices)
-    pylab.colorbar()
-    pylab.title("Option Price")
-    pylab.xlabel("Volatility")
-    pylab.ylabel("Strike Price")
+def plot_options(sigma_vals, K_vals, prices):
+    """
+    Make a contour plot of the option price in (sigma, K) space.
+    """
+    from matplotlib import pyplot as plt
+    plt.contourf(sigma_vals, K_vals, prices)
+    plt.colorbar()
+    plt.title("Option Price")
+    plt.xlabel("Volatility")
+    plt.ylabel("Strike Price")

docs/examples/kernel/mcpricer.py

@@ -1,43 +1,45 b''
-import numpy as N
+import numpy as np
 from math import *
 
-class MCOptionPricer(object):
-    def __init__(self, S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
-        self.S = S
-        self.K = K
-        self.sigma = sigma
-        self.r = r
-        self.days = days
-        self.paths = paths
-        self.h = 1.0/self.days
-        self.const1 = exp((self.r-0.5*self.sigma**2)*self.h)
-        self.const2 = self.sigma*sqrt(self.h)
-        
-    def run(self):
-        stock_price = self.S*N.ones(self.paths, dtype='float64')
-        stock_price_sum = N.zeros(self.paths, dtype='float64')
-        for j in range(self.days):
-            growth_factor = self.const1*N.exp(self.const2*N.random.standard_normal(self.paths))
-            stock_price = stock_price*growth_factor
-            stock_price_sum = stock_price_sum + stock_price
-        stock_price_avg = stock_price_sum/self.days
-        zeros = N.zeros(self.paths, dtype='float64')
-        r_factor = exp(-self.r*self.h*self.days)
-        self.vanilla_put = r_factor*N.mean(N.maximum(zeros,self.K-stock_price))
-        self.asian_put = r_factor*N.mean(N.maximum(zeros,self.K-stock_price_avg))
-        self.vanilla_call = r_factor*N.mean(N.maximum(zeros,stock_price-self.K))
-        self.asian_call = r_factor*N.mean(N.maximum(zeros,stock_price_avg-self.K))
 
+def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
+    """
+    Price European and Asian options using a Monte Carlo method.
 
-def main():
-    op = MCOptionPricer()
-    op.run()
-    print "Vanilla Put Price = ", op.vanilla_put
-    print "Asian Put Price = ", op.asian_put
-    print "Vanilla Call Price = ", op.vanilla_call
-    print "Asian Call Price = ", op.asian_call
+    Parameters
+    ----------
+    S : float
+        The initial price of the stock.
+    K : float
+        The strike price of the option.
+    sigma : float
+        The volatility of the stock.
+    r : float
+        The risk free interest rate.
+    days : int
+        The number of days until the option expires.
+    paths : int
+        The number of Monte Carlo paths used to price the option.
 
-
-if __name__ == '__main__':
-    main()
+    Returns
+    -------
+    A tuple of (E. call, E. put, A. call, A. put) option prices.
+    """
+    h = 1.0/days
+    const1 = exp((r-0.5*sigma**2)*h)
+    const2 = sigma*sqrt(h)
+    stock_price = S*np.ones(paths, dtype='float64')
+    stock_price_sum = np.zeros(paths, dtype='float64')
+    for j in range(days):
+        growth_factor = const1*np.exp(const2*np.random.standard_normal(paths))
+        stock_price = stock_price*growth_factor
+        stock_price_sum = stock_price_sum + stock_price
+    stock_price_avg = stock_price_sum/days
+    zeros = np.zeros(paths, dtype='float64')
+    r_factor = exp(-r*h*days)
+    euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))
+    asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))
+    euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))
+    asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))
+    return (euro_call, euro_put, asian_call, asian_put)
 

docs/examples/kernel/wordfreq.py

@@ -1,13 +1,20 b''
 """Count the frequencies of words in a string"""
 
+from __future__ import division
+
+import cmath as math
+
+
 def wordfreq(text):
     """Return a dictionary of words and word counts in a string."""
 
     freqs = {}
     for word in text.split():
-        freqs[word] = freqs.get(word, 0) + 1
+        lword = word.lower()
+        freqs[lword] = freqs.get(lword, 0) + 1
     return freqs
 
+
 def print_wordfreq(freqs, n=10):
     """Print the n most common words and counts in the freqs dict."""
     
@@ -17,7 +24,43 b' def print_wordfreq(freqs, n=10):'
     for (count, word) in items[:n]:
         print word, count
 
-if __name__ == '__main__':
-    import gzip
-    text = gzip.open('HISTORY.gz').read()
-    freqs = wordfreq(text)
  No newline at end of file
+
+def wordfreq_to_weightsize(worddict, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0):
+    mincount = min(worddict.itervalues())
+    maxcount = max(worddict.itervalues())
+    weights = {}
+    for k, v in worddict.iteritems():
+        w = (v-mincount)/(maxcount-mincount)
+        alpha = minalpha + (maxalpha-minalpha)*w
+        size = minsize + (maxsize-minsize)*w
+        weights[k] = (alpha, size)
+    return weights
+
+
+def tagcloud(worddict, n=10, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0):
+    from matplotlib import pyplot as plt
+    import random
+
+    worddict = wordfreq_to_weightsize(worddict, minsize, maxsize, minalpha, maxalpha)
+
+    fig = plt.figure()
+    ax = fig.add_subplot(111)
+    ax.set_position([0.0,0.0,1.0,1.0])
+    plt.xticks([])
+    plt.yticks([])
+
+    words = worddict.keys()
+    alphas = [v[0] for v in worddict.values()]
+    sizes = [v[1] for v in worddict.values()]
+    items = zip(alphas, sizes, words)
+    items.sort(reverse=True)
+    for alpha, size, word in items[:n]:
+        # xpos = random.normalvariate(0.5, 0.3)
+        # ypos = random.normalvariate(0.5, 0.3)
+        xpos = random.uniform(0.0,1.0)
+        ypos = random.uniform(0.0,1.0)
+        ax.text(xpos, ypos, word.lower(), alpha=alpha, fontsize=size)
+    ax.autoscale_view()
+    return ax
+    
+    
  No newline at end of file

docs/man/ipcluster.1

@@ -11,7 +11,7 b''
 ipcluster is a control tool for IPython's parallel computing functions.
 
 IPython cluster startup. This starts a controller and engines using various
-approaches. Use the IPYTHONDIR environment variable to change your IPython
+approaches. Use the IPYTHON_DIR environment variable to change your IPython
 directory from the default of .ipython or _ipython. The log and security
 subdirectories of your IPython directory will be used by this script for log
 files and security files.

docs/man/ipython.1

@@ -141,8 +141,8 b' may want to use a small, lightweight editor here (in case your default'
 EDITOR is something like Emacs).
 .TP
 .B \-ipythondir <name>
-The name of your IPython configuration directory IPYTHONDIR.  This can
-also be specified through the environment variable IPYTHONDIR.
+The name of your IPython configuration directory IPYTHON_DIR.  This can
+also be specified through the environment variable IPYTHON_DIR.
 .TP
 .B \-log|l
 Generate a log file of all input. The file is named ipython_log.py in your
@@ -197,10 +197,10 b' your config file (default off).'
 .TP
 .B \-profile|p <name>
 Assume that your config file is ipythonrc-<name> (looks in current dir
-first, then in IPYTHONDIR). This is a quick way to keep and load
+first, then in IPYTHON_DIR). This is a quick way to keep and load
 multiple config files for different tasks, especially if you use the
 include option of config files. You can keep a basic
-IPYTHONDIR/ipythonrc file and then have other 'profiles' which include
+IPYTHON_DIR/ipythonrc file and then have other 'profiles' which include
 this one and load extra things for particular tasks. For example:
 .br
 .sp 1
@@ -244,7 +244,7 b' Start in bare bones mode (no config file loaded).'
 .TP
 .B \-rcfile <name>
 Name of your IPython resource configuration file.  normally IPython
-loads ipythonrc (from current directory) or IPYTHONDIR/ipythonrc.  If
+loads ipythonrc (from current directory) or IPYTHON_DIR/ipythonrc.  If
 the loading of your config file fails, IPython starts with a bare
 bones configuration (no modules loaded at all).
 .TP
@@ -286,7 +286,7 b" Shorthand for '\\-separate_in 0 \\-separate_out 0 \\-separate_out2 0'."
 Simply removes all input/output separators.
 .TP
 .B \-upgrade
-Allows you to upgrade your IPYTHONDIR configuration when you install a
+Allows you to upgrade your IPYTHON_DIR configuration when you install a
 new version of IPython.  Since new versions may include new command
 lines options or example files, this copies updated ipythonrc-type
 files.  However, it backs up (with a .old extension) all files which

docs/source/conf.py

@@ -162,10 +162,13 b" latex_font_size = '11pt'"
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 
-latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
-                     ur"""The IPython Development Team""",
-                     'manual', True),
-                    ]
+latex_documents = [
+    ('index', 'ipython.tex', 'IPython Documentation',
+     ur"""The IPython Development Team""", 'manual', True),
+    ('parallel/winhpc_index', 'winhpc_whitepaper.tex',
+     'Using IPython on Windows HPC Server 2008',
+     ur"Brian E. Granger", 'manual', True)
+]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
@@ -182,7 +185,7 b" latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',"
 #latex_appendices = []
 
 # If false, no module index is generated.
-#latex_use_modindex = True
+latex_use_modindex = True
 
 
 # Cleanup

docs/source/config/overview.txt

@@ -243,15 +243,15 b' So where should you put your configuration files? By default, all IPython'
 applications look in the so called "IPython directory".  The location of
 this directory is determined by the following algorithm:
 
-* If the ``-ipythondir`` command line flag is given, its value is used.
+* If the ``--ipython-dir`` command line flag is given, its value is used.
 
 * If not, the value returned by :func:`IPython.utils.genutils.get_ipython_dir`
-  is used. This function will first look at the :envvar:`IPYTHONDIR`
+  is used. This function will first look at the :envvar:`IPYTHON_DIR`
   environment variable and then default to the directory
-  :file:`$HOME/.ipythondir`.
+  :file:`$HOME/.ipython`.
 
 For most users, the default value will simply be something like 
-:file:`$HOME/.ipythondir`.
+:file:`$HOME/.ipython`.
 
 Once the location of the IPython directory has been determined, you need to
 know what filename to use for the configuration file. The basic idea is that

docs/source/development/coding_guide.txt

@@ -76,3 +76,37 b' specific ``IPY`` or ``ipy`` are preferred.'
 
 .. [PEP8] Python Enhancement Proposal 8.  http://www.python.org/peps/pep-0008.html
 
+Attribute declarations for objects
+==================================
+
+In general, objects should declare in their *class* all attributes the object
+is meant to hold throughout its life.  While Python allows you to add an
+attribute to an instance at any point in time, this makes the code harder to
+read and requires methods to constantly use checks with hasattr() or try/except
+calls.  By declaring all attributes of the object in the class header, there is
+a single place one can refer to for understanding the object's data interface,
+where comments can explain the role of each variable and when possible,
+sensible deafaults can be assigned.
+
+.. Warning::
+    
+    If an attribute is meant to contain a mutable object, it should be set to
+    ``None`` in the class and its mutable value should be set in the object's
+    constructor.  Since class attributes are shared by all instances, failure
+    to do this can lead to difficult to track bugs.  But you should still set
+    it in the class declaration so the interface specification is complete and
+    documdented in one place.
+
+A simple example::
+
+    class foo:
+        # X does..., sensible default given:
+        x = 1
+        # y does..., default will be set by constructor
+        y = None
+        # z starts as an empty list, must be set in constructor
+        z = None
+        
+        def __init__(self, y):
+            self.y = y
+            self.z = []

docs/source/development/index.txt

@@ -14,5 +14,7 b" IPython developer's guide"
    release.txt
    roadmap.txt
    reorg.txt
+   magic_blueprint.txt
    notification_blueprint.txt
+   ipgraph.txt
 

docs/source/development/testing.txt

@@ -1,8 +1,8 b''
 .. _testing:
 
-=========================
-Writing and running tests
-=========================
+==========================================
+ Testing IPython for users and developers
+==========================================
 
 Overview
 ========
@@ -14,41 +14,342 b' IPython test system can detect. See below for more details on this.'
 Each subpackage in IPython should have its own :file:`tests` directory that
 contains all of the tests for that subpackage. All of the files in the
 :file:`tests` directory should have the word "tests" in them to enable
-the testing framework to find them.  
+the testing framework to find them.
+
+In docstrings, examples (either using IPython prompts like ``In [1]:`` or
+'classic' python ``>>>`` ones) can and should be included.  The testing system
+will detect them as doctests and will run them; it offers control to skip parts
+or all of a specific doctest if the example is meant to be informative but
+shows non-reproducible information (like filesystem data).
 
 If a subpackage has any dependencies beyond the Python standard library, the
 tests for that subpackage should be skipped if the dependencies are not found.
 This is very important so users don't get tests failing simply because they
-don't have dependencies.  We are still figuring out the best way for this
-to be handled.
+don't have dependencies.
+
+The testing system we use is a hybrid of nose_ and Twisted's trial_ test runner.
+We use both because nose detects more things than Twisted and allows for more
+flexible (and lighter-weight) ways of writing tests; in particular we've
+developed a nose plugin that allows us to paste verbatim IPython sessions and
+test them as doctests, which is extremely important for us.  But the parts of
+IPython that depend on Twisted must be tested using trial, because only trial
+manages the Twisted reactor correctly.
+
+.. _nose: http://code.google.com/p/python-nose
+.. _trial: http://twistedmatrix.com/trac/wiki/TwistedTrial
+
+
+For the impatient: running the tests
+====================================
+
+The simplest way to test IPython is to type at the command line:
+
+.. code-block:: bash
+
+  python -c "import IPython; IPython.test()"
+
+This should work as long as IPython can be imported, even if you haven't fully
+installed the user-facing scripts yet (common in a development environment).
+After a lot of output, you should see something like:
+
+.. code-block:: bash
+  
+    ************************************************************************
+    Ran 10 test groups in 35.228s
+
+    OK
+
+If not, there will be a message indicating which test group failed and how to
+rerun that group individually.
+
+But IPython ships with an entry point script called :file:`iptest` that offers
+fine-grain control over the test process and is particularly useful for
+developers; this script also manages intelligently both nose and trial,
+choosing the correct test system for each of IPython's components.  Running
+:file:`iptest` without arguments gives output identical to that above, but with
+it, you can also run specific tests with fine control.  The :file:`iptest`
+script is installed with IPython, but if you are running from a source tree,
+you can find it in the :file:`IPython/scripts` directory and you can run
+directly from there.
+
+For example, this tests the :mod:`IPython.utils` subpackage, the :option:`-v`
+option shows progress indicators:
+
+.. code-block:: bash
 
-Status
-======
+    maqroll[ipython]> cd IPython/scripts/
+    maqroll[scripts]> ./iptest -v IPython.utils
+    ..........................SS..SSS............................S.S.........
+    ...................................................
+    ----------------------------------------------------------------------
+    Ran 125 tests in 0.070s
 
-Currently IPython's testing system is being reworked.  In the meantime,
-we recommend the following testing practices:
+    OK (SKIP=7)
 
-* To run regular tests, use the :command:`nosetests` command that Nose [Nose]_ 
-  provides on a per file basis:
+Because :file:`iptest` is based on nose, you can use all nose options and
+syntax, typing ``iptest -h`` shows all available options.  For example, this
+lets you run the specific test :func:`test_rehashx` inside the
+:mod:`test_magic` module:
 
 .. code-block:: bash
 
-    nosetests -vvs IPython.core.tests.test_component
+    maqroll[scripts]> ./iptest -vv IPython.core.tests.test_magic:test_rehashx
+    IPython.core.tests.test_magic.test_rehashx(True,) ... ok
+    IPython.core.tests.test_magic.test_rehashx(True,) ... ok
 
-* To run Twisted-using tests, use the :command:`trial` command on a per file 
-  basis:
+    ----------------------------------------------------------------------
+    Ran 2 tests in 0.101s
+
+    OK
+
+When developing, the :option:`--pdb` and :option:`--pdb-failures` of nose are
+particularly useful, these drop you into an interactive pdb session at the
+point of the error or failure respectively.
+
+To run Twisted-using tests, use the :command:`trial` command on a per file or
+package basis:
 
 .. code-block:: bash
 
     trial IPython.kernel
 
-* For now, regular tests (of non-Twisted using code) should be written as
-  unit tests.  They should be subclasses of :class:`unittest.TestCase`.
 
-* Tests of Twisted [Twisted]_ using code should be written by subclassing the
-  ``TestCase`` class that comes with ``twisted.trial.unittest``. Furthermore,
-  all :class:`Deferred` instances that are created in the test must be
-  properly chained and the final one *must* be the return value of the test
-  method.
+For developers: writing tests
+=============================
+
+By now IPython has a reasonable test suite, so the best way to see what's
+available is to look at the :file:`tests` directory in most subpackages.  But
+here are a few pointers to make the process easier.
+
+
+Main tools: :mod:`IPython.testing`
+----------------------------------
+
+The :mod:`IPython.testing` package is where all of the machinery to test
+IPython (rather than the tests for its various parts) lives.  In particular,
+the :mod:`iptest` module in there has all the smarts to control the test
+process.  In there, the :func:`make_exclude` function is used to build a
+blacklist of exclusions, these are modules that do not get even imported for
+tests.  This is important so that things that would fail to even import because
+of missing dependencies don't give errors to end users, as we stated above.
+
+The :mod:`decorators` module contains a lot of useful decorators, especially
+useful to mark individual tests that should be skipped under certain conditions
+(rather than blacklisting the package altogether because of a missing major
+dependency).
+
+Our nose plugin for doctests
+----------------------------
+
+The :mod:`plugin` subpackage in testing contains a nose plugin called
+:mod:`ipdoctest` that teaches nose about IPython syntax, so you can write
+doctests with IPython prompts.  You can also mark doctest output with ``#
+random`` for the output corresponding to a single input to be ignored (stronger
+than using ellipsis and useful to keep it as an example).  If you want the
+entire docstring to be executed but none of the output from any input to be
+checked, you can use the ``# all-random`` marker.  The
+:mod:`IPython.testing.plugin.dtexample` module contains examples of how to use
+these; for reference here is how to use ``# random``::
+
+    def ranfunc():
+	"""A function with some random output.
+
+	   Normal examples are verified as usual:
+	   >>> 1+3
+	   4
+
+	   But if you put '# random' in the output, it is ignored:
+	   >>> 1+3
+	   junk goes here...  # random
+
+	   >>> 1+2
+	   again,  anything goes #random
+	   if multiline, the random mark is only needed once.
+
+	   >>> 1+2
+	   You can also put the random marker at the end:
+	   # random
+
+	   >>> 1+2
+	   # random
+	   .. or at the beginning.
+
+	   More correct input is properly verified:
+	   >>> ranfunc()
+	   'ranfunc'
+	"""
+	return 'ranfunc'
+
+and an example of ``# all-random``::
+	
+    def random_all():
+	"""A function where we ignore the output of ALL examples.
+
+	Examples:
+
+	  # all-random
+
+	  This mark tells the testing machinery that all subsequent examples
+	  should be treated as random (ignoring their output).  They are still
+	  executed, so if a they raise an error, it will be detected as such,
+	  but their output is completely ignored.
+
+	  >>> 1+3
+	  junk goes here...
+
+	  >>> 1+3
+	  klasdfj;
+
+	In [8]: print 'hello'
+	world  # random
+
+	In [9]: iprand()
+	Out[9]: 'iprand'
+	"""
+	return 'iprand'
+
+
+When writing docstrings, you can use the ``@skip_doctest`` decorator to
+indicate that a docstring should *not* be treated as a doctest at all.  The
+difference betwee ``# all-random`` and ``@skip_doctest`` is that the former
+executes the example but ignores output, while the latter doesn't execute any
+code.  ``@skip_doctest`` should be used for docstrings whose examples are
+purely informational.
+
+If a given docstring fails under certain conditions but otherwise is a good
+doctest, you can use code like the following, that relies on the 'null'
+decorator to leave the docstring intact where it works as a test::
+
+  # The docstring for full_path doctests differently on win32 (different path
+  # separator) so just skip the doctest there, and use a null decorator
+  # elsewhere:
+  
+  doctest_deco = dec.skip_doctest if sys.platform == 'win32' else dec.null_deco
+
+  @doctest_deco
+  def full_path(startPath,files):
+      """Make full paths for all the listed files, based on startPath..."""
+
+      # function body follows...
+
+With our nose plugin that understands IPython syntax, an extremely effective
+way to write tests is to simply copy and paste an interactive session into a
+docstring.  You can writing this type of test, where your docstring is meant
+*only* as a test, by prefixing the function name with ``doctest_`` and leaving
+its body *absolutely empty* other than the docstring.  In
+:mod:`IPython.core.tests.test_magic` you can find several examples of this, but
+for completeness sake, your code should look like this (a simple case)::
+
+    def doctest_time():
+	"""
+	In [10]: %time None
+	CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+	Wall time: 0.00 s
+	"""
+	
+This function is only analyzed for its docstring but it is not considered a
+separate test, which is why its body should be empty.
+
+
+Parametric tests done right
+---------------------------
+
+If you need to run multiple tests inside the same standalone function or method
+of a :class:`unittest.TestCase` subclass, IPython provides the ``parametric``
+decorator for this purpose.  This is superior to how test generators work in
+nose, because IPython's keeps intact your stack, which makes debugging vastly
+easier.  For example, these are some parametric tests both in class form and as
+a standalone function (choose in each situation the style that best fits the
+problem at hand, since both work)::
+
+  from IPython.testing import decorators as dec
+
+  def is_smaller(i,j):
+      assert i<j,"%s !< %s" % (i,j)
+
+  class Tester(ParametricTestCase):
+
+      def test_parametric(self):
+	  yield is_smaller(3, 4)
+	  x, y = 1, 2
+	  yield is_smaller(x, y)
+
+  @dec.parametric
+  def test_par_standalone():
+      yield is_smaller(3, 4)
+      x, y = 1, 2
+      yield is_smaller(x, y)
+      
+
+Writing tests for Twisted-using code
+------------------------------------
+      
+Tests of Twisted [Twisted]_ using code should be written by subclassing the
+``TestCase`` class that comes with ``twisted.trial.unittest``. Furthermore, all
+:class:`Deferred` instances that are created in the test must be properly
+chained and the final one *must* be the return value of the test method.
+
+.. note::
+  
+  The best place to see how to use the testing tools, are the tests for these
+  tools themselves, which live in :mod:`IPython.testing.tests`.
+  
+    
+Design requirements
+===================
+
+This section is a set of notes on the key points of the IPython testing needs,
+that were used when writing the system and should be kept for reference as it
+eveolves.
+
+Testing IPython in full requires modifications to the default behavior of nose
+and doctest, because the IPython prompt is not recognized to determine Python
+input, and because IPython admits user input that is not valid Python (things
+like ``%magics`` and ``!system commands``.
+
+We basically need to be able to test the following types of code:
+
+1. Pure Python files containing normal tests.  These are not a problem, since
+   Nose will pick them up as long as they conform to the (flexible) conventions
+   used by nose to recognize tests.
+
+2. Python files containing doctests.  Here, we have two possibilities:
+   - The prompts are the usual ``>>>`` and the input is pure Python.
+   - The prompts are of the form ``In [1]:`` and the input can contain extended
+     IPython expressions.
+
+   In the first case, Nose will recognize the doctests as long as it is called
+   with the ``--with-doctest`` flag.  But the second case will likely require
+   modifications or the writing of a new doctest plugin for Nose that is
+   IPython-aware.
+
+3. ReStructuredText files that contain code blocks.  For this type of file, we
+   have three distinct possibilities for the code blocks:
+   - They use ``>>>`` prompts.
+   - They use ``In [1]:`` prompts.
+   - They are standalone blocks of pure Python code without any prompts.
+
+   The first two cases are similar to the situation #2 above, except that in
+   this case the doctests must be extracted from input code blocks using
+   docutils instead of from the Python docstrings.
+
+   In the third case, we must have a convention for distinguishing code blocks
+   that are meant for execution from others that may be snippets of shell code
+   or other examples not meant to be run.  One possibility is to assume that
+   all indented code blocks are meant for execution, but to have a special
+   docutils directive for input that should not be executed.
+
+   For those code blocks that we will execute, the convention used will simply
+   be that they get called and are considered successful if they run to
+   completion without raising errors.  This is similar to what Nose does for
+   standalone test functions, and by putting asserts or other forms of
+   exception-raising statements it becomes possible to have literate examples
+   that double as lightweight tests.
+
+4. Extension modules with doctests in function and method docstrings.
+   Currently Nose simply can't find these docstrings correctly, because the
+   underlying doctest DocTestFinder object fails there.  Similarly to #2 above,
+   the docstrings could have either pure python or IPython prompts.
 
-.. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/
+Of these, only 3-c (reST with standalone code blocks) is not implemented at
+this point.

docs/source/interactive/reference.txt

@@ -31,7 +31,7 b' your ipythonrc configuration file for details on those. This file'
 typically installed in the $HOME/.ipython directory. For Windows users,
 $HOME resolves to C:\\Documents and Settings\\YourUserName in most
 instances. In the rest of this text, we will refer to this directory as
-IPYTHONDIR.
+IPYTHON_DIR.
 
 
 
@@ -150,9 +150,9 b' All options with a [no] prepended can be specified in negated form'
 	something like Emacs).
 
     -ipythondir <name> 
-	name of your IPython configuration directory IPYTHONDIR. This
+	name of your IPython configuration directory IPYTHON_DIR. This
 	can also be specified through the environment variable
-	IPYTHONDIR.
+	IPYTHON_DIR.
 
     -log, l 
 	generate a log file of all input. The file is named
@@ -211,10 +211,10 b' All options with a [no] prepended can be specified in negated form'
 
 	assume that your config file is ipythonrc-<name> or
 	ipy_profile_<name>.py (looks in current dir first, then in
-	IPYTHONDIR).  This is a quick way to keep and load multiple
+	IPYTHON_DIR).  This is a quick way to keep and load multiple
 	config files for different tasks, especially if you use the
 	include option of config files. You can keep a basic
-	IPYTHONDIR/ipythonrc file and then have other 'profiles' which
+	IPYTHON_DIR/ipythonrc file and then have other 'profiles' which
 	include this one and load extra things for particular
 	tasks. For example:
 
@@ -252,7 +252,7 b' All options with a [no] prepended can be specified in negated form'
     -rcfile <name>   
 	name of your IPython resource configuration file. Normally
 	IPython loads ipythonrc (from current directory) or
-	IPYTHONDIR/ipythonrc.
+	IPYTHON_DIR/ipythonrc.
 
 	If the loading of your config file fails, IPython starts with
 	a bare bones configuration (no modules loaded at all).
@@ -299,7 +299,7 b' All options with a [no] prepended can be specified in negated form'
 	0'. Simply removes all input/output separators.
 
     -upgrade 
-	allows you to upgrade your IPYTHONDIR configuration when you
+	allows you to upgrade your IPYTHON_DIR configuration when you
 	install a new version of IPython. Since new versions may
 	include new command line options or example files, this copies
 	updated ipythonrc-type files. However, it backs up (with a
@@ -542,7 +542,7 b' Persistent command history across sessions'
 
 IPython will save your input history when it leaves and reload it next
 time you restart it. By default, the history file is named
-$IPYTHONDIR/history, but if you've loaded a named profile,
+$IPYTHON_DIR/history, but if you've loaded a named profile,
 '-PROFILE_NAME' is appended to the name. This allows you to keep
 separate histories related to various tasks: commands related to
 numerical work will not be clobbered by a system shell history, for
@@ -636,7 +636,7 b' follows:'
 %logstart [log_name [log_mode]]
 
 If no name is given, it defaults to a file named 'log' in your
-IPYTHONDIR directory, in 'rotate' mode (see below).
+IPYTHON_DIR directory, in 'rotate' mode (see below).
 
 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
 history up to that point and then continues logging.

docs/source/parallel/index.txt

@@ -13,5 +13,7 b' Using IPython for parallel computing'
    parallel_task.txt
    parallel_mpi.txt
    parallel_security.txt
+   parallel_winhpc.txt
+   parallel_demos.txt
 
 

setup.py

@@ -19,6 +19,7 b' requires utilities which are not available under Windows."""'
 
 # Stdlib imports
 import os
+import shutil
 import sys
 
 from glob import glob
@@ -43,6 +44,21 b' from setupbase import ('
 isfile = os.path.isfile
 pjoin = os.path.join
 
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+def cleanup():
+    """Clean up the junk left around by the build process"""
+    if "develop" not in sys.argv:
+        try:
+            shutil.rmtree('ipython.egg-info')
+        except:
+            try:
+                os.unlink('ipython.egg-info')
+            except:
+                pass
+
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
@@ -144,7 +160,6 b" if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):"
             )
         
     [ target_update(*t) for t in to_update ]
-
     
 #---------------------------------------------------------------------------
 # Find all the packages, package data, scripts and data_files
@@ -159,6 +174,14 b' data_files = find_data_files()'
 # Handle dependencies and setuptools specific things
 #---------------------------------------------------------------------------
 
+# For some commands, use setuptools.  Note that we do NOT list install here!
+# If you want a setuptools-enhanced install, just run 'setupegg.py install'
+if len(set(('develop', 'sdist', 'release', 'bdist_egg', 'bdist_rpm',
+           'bdist', 'bdist_dumb', 'bdist_wininst', 'install_egg_info',
+           'build_sphinx', 'egg_info', 'easy_install', 'upload',
+            )).intersection(sys.argv)) > 0:
+    import setuptools
+
 # This dict is used for passing extra arguments that are setuptools 
 # specific to setup
 setuptools_extra_args = {}
@@ -169,9 +192,9 b" if 'setuptools' in sys.modules:"
         'console_scripts': [
             'ipython = IPython.core.ipapp:launch_new_instance',
             'pycolor = IPython.utils.PyColorize:main',
-            'ipcontroller = IPython.kernel.scripts.ipcontroller:main',
-            'ipengine = IPython.kernel.scripts.ipengine:main',
-            'ipcluster = IPython.kernel.scripts.ipcluster:main',
+            'ipcontroller = IPython.kernel.ipcontrollerapp:launch_new_instance',
+            'ipengine = IPython.kernel.ipengineapp:launch_new_instance',
+            'ipcluster = IPython.kernel.ipclusterapp:launch_new_instance',
             'ipythonx = IPython.frontend.wx.ipythonx:main',
             'iptest = IPython.testing.iptest:main',
             'irunner = IPython.lib.irunner:main'
@@ -195,7 +218,6 b' else:'
     # just to make life easy for users.
     check_for_dependencies()
 
-
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
@@ -206,5 +228,7 b" setup_args['scripts'] = scripts"
 setup_args['data_files'] = data_files
 setup_args.update(setuptools_extra_args)
 
+
 if __name__ == '__main__':
     setup(**setup_args)
+    cleanup()

setupbase.py

@@ -119,8 +119,8 b' def find_packages():'
     add_package(packages, 'frontend.wx')
     add_package(packages, 'gui')
     add_package(packages, 'gui.wx')
-    add_package(packages, 'kernel', config=True, tests=True, scripts=True)
-    add_package(packages, 'kernel.core', config=True, tests=True)
+    add_package(packages, 'kernel', config=False, tests=True, scripts=True)
+    add_package(packages, 'kernel.core', config=False, tests=True)
     add_package(packages, 'lib', tests=True)
     add_package(packages, 'quarantine', tests=True)
     add_package(packages, 'scripts')

setupegg.py

@@ -1,12 +1,6 b''
 #!/usr/bin/env python
 """Wrapper to run setup.py using setuptools."""
 
-import sys
-
-# now, import setuptools and call the actual setup
+# Import setuptools and call the actual setup
 import setuptools
 execfile('setup.py')
-
-# clean up the junk left around by setuptools
-if "develop" not in sys.argv:
-    os.system('rm -rf ipython.egg-info build')

tools/build_release

@@ -19,15 +19,15 b" for d in ['build','dist',pjoin('docs','build'),pjoin('docs','dist')]:"
         remove_tree(d)
 
 # Build source and binary distros
-c('./setup.py sdist --formats=gztar')
+c('./setup.py sdist --formats=gztar,zip')
 
 # Build version-specific RPMs, where we must use the --python option to ensure
 # that the resulting RPM is really built with the requested python version (so
 # things go to lib/python2.X/...)
-c("python2.5 ./setup.py bdist_rpm --binary-only --release=py25 "
-  "--python=/usr/bin/python2.5")
-c("python2.6 ./setup.py bdist_rpm --binary-only --release=py26 "
-  "--python=/usr/bin/python2.6")
+#c("python2.5 ./setup.py bdist_rpm --binary-only --release=py25 "
+#  "--python=/usr/bin/python2.5")
+#c("python2.6 ./setup.py bdist_rpm --binary-only --release=py26 "
+#  "--python=/usr/bin/python2.6")
 
 # Build eggs
 c('python2.5 ./setupegg.py bdist_egg')
@@ -36,7 +36,7 b" c('python2.6 ./setupegg.py bdist_egg')"
 # Call the windows build separately, so that the extra Windows scripts don't
 # get pulled into Unix builds (setup.py has code which checks for
 # bdist_wininst)
-c("python setup.py bdist_wininst --install-script=ipython_win_post_install.py")
+c("python ./setup.py bdist_wininst")
 
 # Change name so retarded Vista runs the installer correctly
 c("rename 's/linux-i686/win32-setup/' dist/*.exe")

IPython/config/api.py

@@ -1,102 +0,0 b''
-# encoding: utf-8
-
-"""This is the official entry point to IPython's configuration system.  """
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-import os
-from os.path import join as pjoin
-
-from IPython.utils.genutils import get_home_dir, get_ipython_dir
-from IPython.external.configobj import ConfigObj
-
-
-class ConfigObjManager(object):
-    
-    def __init__(self, configObj, filename):
-        self.current = configObj
-        self.current.indent_type = '    '
-        self.filename = filename
-        # self.write_default_config_file()
-        
-    def get_config_obj(self):
-        return self.current
-    
-    def update_config_obj(self, newConfig):
-        self.current.merge(newConfig)
-        
-    def update_config_obj_from_file(self, filename):
-        newConfig = ConfigObj(filename, file_error=False)
-        self.current.merge(newConfig)
-        
-    def update_config_obj_from_default_file(self, ipythondir=None):
-        fname = self.resolve_file_path(self.filename, ipythondir)
-        self.update_config_obj_from_file(fname)
-
-    def write_config_obj_to_file(self, filename):
-        f = open(filename, 'w')
-        self.current.write(f)
-        f.close()
-
-    def write_default_config_file(self):
-        ipdir = get_ipython_dir()
-        fname = pjoin(ipdir, self.filename)
-        if not os.path.isfile(fname):
-            print "Writing the configuration file to: " + fname
-            self.write_config_obj_to_file(fname)
-    
-    def _import(self, key):
-        package = '.'.join(key.split('.')[0:-1])
-        obj = key.split('.')[-1]
-        execString = 'from %s import %s' % (package, obj)
-        exec execString
-        exec 'temp = %s' % obj 
-        return temp
-
-    def resolve_file_path(self, filename, ipythondir = None):
-        """Resolve filenames into absolute paths.
-
-        This function looks in the following directories in order:
-
-        1.  In the current working directory or by absolute path with ~ expanded
-        2.  In ipythondir if that is set
-        3.  In the IPYTHONDIR environment variable if it exists
-        4.  In the ~/.ipython directory
-
-        Note: The IPYTHONDIR is also used by the trunk version of IPython so
-               changing it will also affect it was well.
-        """
-
-        # In cwd or by absolute path with ~ expanded
-        trythis = os.path.expanduser(filename)
-        if os.path.isfile(trythis):
-            return trythis
-
-        # In ipythondir if it is set
-        if ipythondir is not None:
-            trythis = pjoin(ipythondir, filename)
-            if os.path.isfile(trythis):
-                return trythis        
-
-        trythis = pjoin(get_ipython_dir(), filename)
-        if os.path.isfile(trythis):
-            return trythis
-
-        return None
-
-
-    
-    
-
-    

IPython/core/oldusersetup.py

@@ -1,219 +0,0 b''
-# -*- coding: utf-8 -*-
-"""
-Main IPython Component
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2001 Janko Hauser <[e-mail]>
-#  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
-#  Copyright (C) 2008-2009  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import glob
-import os
-import shutil
-import sys
-
-from IPython.utils.genutils import *
-
-def user_setup(ipythondir,rc_suffix,mode='install',interactive=True):
-    """Install or upgrade the user configuration directory.
-
-    Can be called when running for the first time or to upgrade the user's
-    .ipython/ directory.
-
-    Parameters
-    ----------
-      ipythondir : path
-        The directory to be used for installation/upgrade.  In 'install' mode,
-        if this path already exists, the function exits immediately.
-
-      rc_suffix : str
-        Extension for the config files.  On *nix platforms it is typically the
-        empty string, while Windows normally uses '.ini'.
-
-      mode : str, optional
-        Valid modes are 'install' and 'upgrade'.
-
-      interactive : bool, optional
-        If False, do not wait for user input on any errors.  Normally after
-        printing its status information, this function waits for the user to
-        hit Return before proceeding.  This is because the default use case is
-        when first installing the IPython configuration, so we want the user to
-        acknowledge the initial message, which contains some useful
-        information.
-        """
-
-    # For automatic use, deactivate all i/o
-    if interactive:
-        def wait():
-            try:
-                raw_input("Please press <RETURN> to start IPython.")
-            except EOFError:
-                print >> Term.cout
-            print '*'*70
-
-        def printf(s):
-            print s
-    else:
-        wait = lambda : None
-        printf = lambda s : None
-
-    # Install mode should be re-entrant: if the install dir already exists,
-    # bail out cleanly.
-    # XXX.  This is too hasty to return.  We need to check to make sure that
-    # all the expected config files and directories are actually there. We
-    # currently have a failure mode if someone deletes a needed config file
-    # but still has the ipythondir.
-    if mode == 'install' and os.path.isdir(ipythondir):
-        return
-
-    cwd = os.getcwd()  # remember where we started
-    glb = glob.glob
-
-    printf('*'*70)
-    if mode == 'install':
-        printf(
-"""Welcome to IPython. I will try to create a personal configuration directory
-where you can customize many aspects of IPython's functionality in:\n""")
-    else:
-        printf('I am going to upgrade your configuration in:')
-
-    printf(ipythondir)
-
-    rcdirend = os.path.join('IPython','config','userconfig')
-    cfg = lambda d: os.path.join(d,rcdirend)
-    try:
-        rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
-        printf("Initializing from configuration: %s" % rcdir)
-    except IndexError:
-        warning = """
-Installation error. IPython's directory was not found.
-
-Check the following:
-
-The ipython/IPython directory should be in a directory belonging to your
-PYTHONPATH environment variable (that is, it should be in a directory
-belonging to sys.path). You can copy it explicitly there or just link to it.
-
-IPython will create a minimal default configuration for you.
-
-"""
-        warn(warning)
-        wait()
-
-        if sys.platform =='win32':
-            inif = 'ipythonrc.ini'
-        else:
-            inif = 'ipythonrc'
-        minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults',
-                         inif : '# intentionally left blank' }
-        os.makedirs(ipythondir, mode = 0777)
-        for f, cont in minimal_setup.items():
-            # In 2.5, this can be more cleanly done using 'with'
-            fobj = file(ipythondir + '/' + f,'w')
-            fobj.write(cont)
-            fobj.close()
-
-        return
-
-    if mode == 'install':
-        try:
-            shutil.copytree(rcdir,ipythondir)
-            os.chdir(ipythondir)
-            rc_files = glb("ipythonrc*")
-            for rc_file in rc_files:
-                os.rename(rc_file,rc_file+rc_suffix)
-        except:
-            warning = """
-
-There was a problem with the installation:
-%s
-Try to correct it or contact the developers if you think it's a bug.
-IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
-            warn(warning)
-            wait()
-            return
-
-    elif mode == 'upgrade':
-        try:
-            os.chdir(ipythondir)
-        except:
-            printf("""
-Can not upgrade: changing to directory %s failed. Details:
-%s
-""" % (ipythondir,sys.exc_info()[1]) )
-            wait()
-            return
-        else:
-            sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
-            for new_full_path in sources:
-                new_filename = os.path.basename(new_full_path)
-                if new_filename.startswith('ipythonrc'):
-                    new_filename = new_filename + rc_suffix
-                # The config directory should only contain files, skip any
-                # directories which may be there (like CVS)
-                if os.path.isdir(new_full_path):
-                    continue
-                if os.path.exists(new_filename):
-                    old_file = new_filename+'.old'
-                    if os.path.exists(old_file):
-                        os.remove(old_file)
-                    os.rename(new_filename,old_file)
-                shutil.copy(new_full_path,new_filename)
-    else:
-        raise ValueError('unrecognized mode for install: %r' % mode)
-
-    # Fix line-endings to those native to each platform in the config
-    # directory.
-    try:
-        os.chdir(ipythondir)
-    except:
-        printf("""
-Problem: changing to directory %s failed.
-Details:
-%s
-
-Some configuration files may have incorrect line endings.  This should not
-cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1]) )
-        wait()
-    else:
-        for fname in glb('ipythonrc*'):
-            try:
-                native_line_ends(fname,backup=0)
-            except IOError:
-                pass
-
-    if mode == 'install':
-        printf("""
-Successful installation!
-
-Please read the sections 'Initial Configuration' and 'Quick Tips' in the
-IPython manual (there are both HTML and PDF versions supplied with the
-distribution) to make sure that your system environment is properly configured
-to take advantage of IPython's features.
-
-Important note: the configuration system has changed! The old system is
-still in place, but its setting may be partly overridden by the settings in 
-"~/.ipython/ipy_user_conf.py" config file. Please take a look at the file 
-if some of the new settings bother you. 
-
-""")
-    else:
-        printf("""
-Successful upgrade!
-
-All files in your directory:
-%(ipythondir)s
-which would have been overwritten by the upgrade were backed up with a .old
-extension.  If you had made particular customizations in those files you may
-want to merge them back into the new files.""" % locals() )
-    wait()
-    os.chdir(cwd)
  No newline at end of file

IPython/core/tests/obj_del.py

@@ -1,35 +0,0 b''
-"""Test code for https://bugs.launchpad.net/ipython/+bug/239054
-
-WARNING: this script exits IPython!  It MUST be run in a subprocess.
-
-When you run the following script from CPython it prints:
-__init__ is here
-__del__ is here
-
-and creates the __del__.txt file
-
-When you run it from IPython it prints:
-__init__ is here
-
-When you exit() or Exit from IPython neothing is printed and no file is created
-(the file thing is to make sure __del__ is really never called and not that
-just the output is eaten).
-
-Note that if you call %reset in IPython then everything is Ok.
-
-IPython should do the equivalent of %reset and release all the references it
-holds before exit. This behavior is important when working with binding objects
-that rely on __del__. If the current behavior has some use case then I suggest
-to add a configuration option to IPython to control it.
-"""
-import sys
-
-class A(object):
-    def __del__(self):
-        print 'obj_del.py: object A deleted'
-
-a = A()
-
-# Now, we force an exit, the caller will check that the del printout was given
-_ip = get_ipython()
-_ip.ask_exit()

IPython/kernel/config/__init__.py

@@ -1,126 +0,0 b''
-# encoding: utf-8
-
-"""Default kernel configuration."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-import os, sys
-from os.path import join as pjoin
-
-from IPython.external.configobj import ConfigObj
-from IPython.config.api import ConfigObjManager
-from IPython.utils.genutils import get_ipython_dir, get_security_dir
-
-default_kernel_config = ConfigObj()
-
-# This will raise OSError if ipythondir doesn't exist.
-security_dir = get_security_dir()
-
-#-------------------------------------------------------------------------------
-# Engine Configuration
-#-------------------------------------------------------------------------------
-
-engine_config = dict(
-    logfile = '',    # Empty means log to stdout
-    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
-)
-
-#-------------------------------------------------------------------------------
-# MPI Configuration
-#-------------------------------------------------------------------------------
-
-mpi_config = dict(
-    mpi4py = """from mpi4py import MPI as mpi
-mpi.size = mpi.COMM_WORLD.Get_size()
-mpi.rank = mpi.COMM_WORLD.Get_rank()
-""",
-    pytrilinos = """from PyTrilinos import Epetra
-class SimpleStruct:
-    pass
-mpi = SimpleStruct()
-mpi.rank = 0
-mpi.size = 0
-""",
-    default = ''
-)
-
-#-------------------------------------------------------------------------------
-# Controller Configuration
-#-------------------------------------------------------------------------------
-
-controller_config = dict(
-
-    logfile = '',    # Empty means log to stdout
-    import_statement = '',
-    reuse_furls = False, # If False, old furl files are deleted
-
-    engine_tub = dict(
-        ip = '',         # Empty string means all interfaces
-        port = 0,        # 0 means pick a port for me
-        location = '',    # Empty string means try to set automatically
-        secure = True,
-        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
-    ),
-    engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
-    
-    controller_interfaces = dict(
-        # multiengine = dict(
-        #     controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
-        #     fc_interface = 'IPython.kernel.multienginefc.IFCMultiEngine',
-        #     furl_file = 'ipcontroller-mec.furl'
-        # ),
-        task = dict(
-            controller_interface = 'IPython.kernel.task.ITaskController',
-            fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
-        ),
-        multiengine = dict(
-            controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
-            fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
-        )
-    ),
-
-    client_tub = dict(
-        ip = '',         # Empty string means all interfaces
-        port = 0,        # 0 means pick a port for me
-        location = '',    # Empty string means try to set automatically
-        secure = True,
-        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
-    )
-)
-
-#-------------------------------------------------------------------------------
-# Client Configuration
-#-------------------------------------------------------------------------------
-
-client_config = dict(
-    client_interfaces = dict(
-        task = dict(
-            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
-        ),
-        multiengine = dict(
-            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
-        )
-    )
-)
-
-default_kernel_config['engine'] = engine_config
-default_kernel_config['mpi'] = mpi_config
-default_kernel_config['controller'] = controller_config
-default_kernel_config['client'] = client_config
-
-
-config_manager = ConfigObjManager(default_kernel_config, 'IPython.kernel.ini')
  No newline at end of file

IPython/kernel/core/config/__init__.py

@@ -1,25 +0,0 b''
-# encoding: utf-8
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-from IPython.external.configobj import ConfigObj
-from IPython.config.api import ConfigObjManager
-
-default_core_config = ConfigObj()
-default_core_config['shell'] = dict(
-    shell_class = 'IPython.kernel.core.interpreter.Interpreter',
-    import_statement = ''
-)
-
-config_manager = ConfigObjManager(default_core_config, 'IPython.kernel.core.ini')
  No newline at end of file

IPython/kernel/magic.py

@@ -1,171 +0,0 b''
-# encoding: utf-8
-
-"""Magic command interface for interactive parallel work."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-import new
-
-from IPython.core.iplib import InteractiveShell
-from IPython.core.shell import MTInteractiveShell
-
-from twisted.internet.defer import Deferred
-
-
-#-------------------------------------------------------------------------------
-# Definitions of magic functions for use with IPython
-#-------------------------------------------------------------------------------
-
-NO_ACTIVE_CONTROLLER = """
-Error:  No Controller is activated
-Use activate() on a RemoteController object to activate it for magics.
-"""
-
-def magic_result(self,parameter_s=''):
-    """Print the result of command i on all engines of the active controller.
-    
-    To activate a controller in IPython, first create it and then call
-    the activate() method.
-    
-    Then you can do the following:
-    
-    >>> result                                # Print the latest result
-    Printing result... 
-    [127.0.0.1:0] In [1]: b = 10
-    [127.0.0.1:1] In [1]: b = 10
-    
-    >>> result 0                              # Print result 0
-    In [14]: result 0
-    Printing result... 
-    [127.0.0.1:0] In [0]: a = 5
-    [127.0.0.1:1] In [0]: a = 5
-    """
-    try:
-        activeController = __IPYTHON__.activeController
-    except AttributeError:
-        print NO_ACTIVE_CONTROLLER
-    else:
-        try:
-            index = int(parameter_s)
-        except:
-            index = None
-        result = activeController.get_result(index)
-        return result
-
-def magic_px(self,parameter_s=''):
-    """Executes the given python command on the active IPython Controller.
-    
-    To activate a Controller in IPython, first create it and then call
-    the activate() method.
-    
-    Then you can do the following:
-     
-    >>> %px a = 5       # Runs a = 5 on all nodes
-    """
-
-    try:
-        activeController = __IPYTHON__.activeController
-    except AttributeError:
-        print NO_ACTIVE_CONTROLLER
-    else:
-        print "Parallel execution on engines: %s" % activeController.targets
-        result = activeController.execute(parameter_s)
-        return result
-
-def pxrunsource(self, source, filename="<input>", symbol="single"):
-
-    try:
-        code = self.compile(source, filename, symbol)
-    except (OverflowError, SyntaxError, ValueError):
-        # Case 1
-        self.showsyntaxerror(filename)
-        return None
-
-    if code is None:
-        # Case 2
-        return True
-
-    # Case 3
-    # Because autopx is enabled, we now call executeAll or disable autopx if
-    # %autopx or autopx has been called
-    if 'get_ipython().magic("%autopx' in source or 'get_ipython().magic("autopx' in source:
-        _disable_autopx(self)
-        return False
-    else:
-        try:
-            result = self.activeController.execute(source)
-        except:
-            self.showtraceback()
-        else:
-            print result.__repr__()
-        return False
-        
-def magic_autopx(self, parameter_s=''):
-    """Toggles auto parallel mode for the active IPython Controller.
-    
-    To activate a Controller in IPython, first create it and then call
-    the activate() method.
-    
-    Then you can do the following:
-     
-    >>> %autopx                    # Now all commands are executed in parallel
-    Auto Parallel Enabled
-    Type %autopx to disable
-    ...
-    >>> %autopx                    # Now all commands are locally executed
-    Auto Parallel Disabled
-    """
-  
-    if hasattr(self, 'autopx'):
-        if self.autopx == True:
-            _disable_autopx(self)
-        else:
-            _enable_autopx(self)
-    else:
-        _enable_autopx(self)
-
-def _enable_autopx(self):
-    """Enable %autopx mode by saving the original runsource and installing 
-    pxrunsource.
-    """
-    try:
-        activeController = __IPYTHON__.activeController
-    except AttributeError:
-        print "No active RemoteController found, use RemoteController.activate()."
-    else:
-        self._original_runsource = self.runsource
-        self.runsource = new.instancemethod(pxrunsource, self, self.__class__)
-        self.autopx = True
-        print "Auto Parallel Enabled\nType %autopx to disable"
-
-def _disable_autopx(self):
-    """Disable %autopx by restoring the original runsource."""
-    if hasattr(self, 'autopx'):
-        if self.autopx == True:
-            self.runsource = self._original_runsource
-            self.autopx = False
-            print "Auto Parallel Disabled"
-            
-# Add the new magic function to the class dict:
-
-InteractiveShell.magic_result = magic_result
-InteractiveShell.magic_px = magic_px
-InteractiveShell.magic_autopx = magic_autopx
-
-# And remove the global name to keep global namespace clean.  Don't worry, the
-# copy bound to IPython stays, we're just removing the global name.
-del magic_result
-del magic_px
-del magic_autopx
-

IPython/kernel/scripts/ipcluster.py

@@ -1,813 +0,0 b''
- #!/usr/bin/env python
-# encoding: utf-8
-
-"""Start an IPython cluster = (controller + engines)."""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import os
-import re
-import sys
-import signal
-import tempfile
-pjoin = os.path.join
-
-from twisted.internet import reactor, defer
-from twisted.internet.protocol import ProcessProtocol
-from twisted.internet.error import ProcessDone, ProcessTerminated
-from twisted.internet.utils import getProcessOutput
-from twisted.python import failure, log
-
-from IPython.external import argparse
-from IPython.external import Itpl
-from IPython.utils.genutils import (
-    get_ipython_dir, 
-    get_log_dir, 
-    get_security_dir, 
-    num_cpus
-)
-from IPython.kernel.fcutil import have_crypto
-
-# Create various ipython directories if they don't exist.
-# This must be done before IPython.kernel.config is imported.
-from IPython.core.oldusersetup import user_setup
-if os.name == 'posix':
-    rc_suffix = ''
-else:
-    rc_suffix = '.ini'
-user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
-get_log_dir()
-get_security_dir()
-
-from IPython.kernel.config import config_manager as kernel_config_manager
-from IPython.kernel.error import SecurityError, FileTimeoutError
-from IPython.kernel.fcutil import have_crypto
-from IPython.kernel.twistedutil import gatherBoth, wait_for_file
-from IPython.kernel.util import printer
-
-#-----------------------------------------------------------------------------
-# General process handling code
-#-----------------------------------------------------------------------------
-
-
-class ProcessStateError(Exception):
-    pass
-
-class UnknownStatus(Exception):
-    pass
-
-class LauncherProcessProtocol(ProcessProtocol):
-    """
-    A ProcessProtocol to go with the ProcessLauncher.
-    """
-    def __init__(self, process_launcher):
-        self.process_launcher = process_launcher
-    
-    def connectionMade(self):
-        self.process_launcher.fire_start_deferred(self.transport.pid)
-    
-    def processEnded(self, status):
-        value = status.value
-        if isinstance(value, ProcessDone):
-            self.process_launcher.fire_stop_deferred(0)
-        elif isinstance(value, ProcessTerminated):
-            self.process_launcher.fire_stop_deferred(
-                {'exit_code':value.exitCode,
-                 'signal':value.signal,
-                 'status':value.status
-                }
-            )
-        else:
-            raise UnknownStatus("unknown exit status, this is probably a bug in Twisted")
-    
-    def outReceived(self, data):
-        log.msg(data)
-    
-    def errReceived(self, data):
-        log.err(data)
-
-class ProcessLauncher(object):
-    """
-    Start and stop an external process in an asynchronous manner.
-    
-    Currently this uses deferreds to notify other parties of process state
-    changes.  This is an awkward design and should be moved to using
-    a formal NotificationCenter.
-    """
-    def __init__(self, cmd_and_args):
-        self.cmd = cmd_and_args[0]
-        self.args = cmd_and_args
-        self._reset()
-    
-    def _reset(self):
-        self.process_protocol = None
-        self.pid = None
-        self.start_deferred = None
-        self.stop_deferreds = []
-        self.state = 'before' # before, running, or after
-    
-    @property
-    def running(self):
-        if self.state == 'running':
-            return True
-        else:
-            return False
-    
-    def fire_start_deferred(self, pid):
-        self.pid = pid
-        self.state = 'running'
-        log.msg('Process %r has started with pid=%i' % (self.args, pid))
-        self.start_deferred.callback(pid)
-    
-    def start(self):
-        if self.state == 'before':
-            self.process_protocol = LauncherProcessProtocol(self)
-            self.start_deferred = defer.Deferred()
-            self.process_transport = reactor.spawnProcess(
-                self.process_protocol,
-                self.cmd,
-                self.args,
-                env=os.environ
-            )
-            return self.start_deferred
-        else:
-            s = 'the process has already been started and has state: %r' % \
-                self.state
-            return defer.fail(ProcessStateError(s))
-    
-    def get_stop_deferred(self):
-        if self.state == 'running' or self.state == 'before':
-            d = defer.Deferred()
-            self.stop_deferreds.append(d)
-            return d
-        else:
-            s = 'this process is already complete'
-            return defer.fail(ProcessStateError(s))
-    
-    def fire_stop_deferred(self, exit_code):
-        log.msg('Process %r has stopped with %r' % (self.args, exit_code))
-        self.state = 'after'
-        for d in self.stop_deferreds:
-            d.callback(exit_code)
-    
-    def signal(self, sig):
-        """
-        Send a signal to the process.
-        
-        The argument sig can be ('KILL','INT', etc.) or any signal number.
-        """
-        if self.state == 'running':
-            self.process_transport.signalProcess(sig)
-    
-    # def __del__(self):
-    #     self.signal('KILL')
-    
-    def interrupt_then_kill(self, delay=1.0):
-        self.signal('INT')
-        reactor.callLater(delay, self.signal, 'KILL')
-
-
-#-----------------------------------------------------------------------------
-# Code for launching controller and engines
-#-----------------------------------------------------------------------------
-
-
-class ControllerLauncher(ProcessLauncher):
-    
-    def __init__(self, extra_args=None):
-        if sys.platform == 'win32':
-            # This logic is needed because the ipcontroller script doesn't
-            # always get installed in the same way or in the same location.
-            from IPython.kernel.scripts import ipcontroller
-            script_location = ipcontroller.__file__.replace('.pyc', '.py')
-            # The -u option here turns on unbuffered output, which is required
-            # on Win32 to prevent wierd conflict and problems with Twisted.
-            # Also, use sys.executable to make sure we are picking up the 
-            # right python exe.
-            args = [sys.executable, '-u', script_location]
-        else:
-            args = ['ipcontroller']
-        self.extra_args = extra_args
-        if extra_args is not None:
-            args.extend(extra_args)
-        
-        ProcessLauncher.__init__(self, args)
-
-
-class EngineLauncher(ProcessLauncher):
-    
-    def __init__(self, extra_args=None):
-        if sys.platform == 'win32':
-            # This logic is needed because the ipcontroller script doesn't
-            # always get installed in the same way or in the same location.
-            from IPython.kernel.scripts import ipengine
-            script_location = ipengine.__file__.replace('.pyc', '.py')
-            # The -u option here turns on unbuffered output, which is required
-            # on Win32 to prevent wierd conflict and problems with Twisted.
-            # Also, use sys.executable to make sure we are picking up the 
-            # right python exe.
-            args = [sys.executable, '-u', script_location]
-        else:
-            args = ['ipengine']
-        self.extra_args = extra_args
-        if extra_args is not None:
-            args.extend(extra_args)
-        
-        ProcessLauncher.__init__(self, args)
-
-
-class LocalEngineSet(object):
-    
-    def __init__(self, extra_args=None):
-        self.extra_args = extra_args
-        self.launchers = []
-    
-    def start(self, n):
-        dlist = []
-        for i in range(n):
-            el = EngineLauncher(extra_args=self.extra_args)
-            d = el.start()
-            self.launchers.append(el)
-            dlist.append(d)
-        dfinal = gatherBoth(dlist, consumeErrors=True)
-        dfinal.addCallback(self._handle_start)
-        return dfinal
-    
-    def _handle_start(self, r):
-        log.msg('Engines started with pids: %r' % r)
-        return r
-    
-    def _handle_stop(self, r):
-        log.msg('Engines received signal: %r' % r)
-        return r
-    
-    def signal(self, sig):
-        dlist = []
-        for el in self.launchers:
-            d = el.get_stop_deferred()
-            dlist.append(d)
-            el.signal(sig)
-        dfinal = gatherBoth(dlist, consumeErrors=True)
-        dfinal.addCallback(self._handle_stop)
-        return dfinal
-    
-    def interrupt_then_kill(self, delay=1.0):
-        dlist = []
-        for el in self.launchers:
-            d = el.get_stop_deferred()
-            dlist.append(d)
-            el.interrupt_then_kill(delay)
-        dfinal = gatherBoth(dlist, consumeErrors=True)
-        dfinal.addCallback(self._handle_stop)
-        return dfinal
-
-
-class BatchEngineSet(object):
-    
-    # Subclasses must fill these in.  See PBSEngineSet
-    submit_command = ''
-    delete_command = ''
-    job_id_regexp = ''
-    
-    def __init__(self, template_file, **kwargs):
-        self.template_file = template_file
-        self.context = {}
-        self.context.update(kwargs)
-        self.batch_file = self.template_file+'-run'
-    
-    def parse_job_id(self, output):
-        m = re.match(self.job_id_regexp, output)
-        if m is not None:
-            job_id = m.group()
-        else:
-            raise Exception("job id couldn't be determined: %s" % output)
-        self.job_id = job_id
-        log.msg('Job started with job id: %r' % job_id)
-        return job_id
-    
-    def write_batch_script(self, n):
-        self.context['n'] = n
-        template = open(self.template_file, 'r').read()
-        log.msg('Using template for batch script: %s' % self.template_file)
-        script_as_string = Itpl.itplns(template, self.context)
-        log.msg('Writing instantiated batch script: %s' % self.batch_file)
-        f = open(self.batch_file,'w')
-        f.write(script_as_string)
-        f.close()
-    
-    def handle_error(self, f):
-        f.printTraceback()
-        f.raiseException()
-    
-    def start(self, n):
-        self.write_batch_script(n)
-        d = getProcessOutput(self.submit_command,
-            [self.batch_file],env=os.environ)
-        d.addCallback(self.parse_job_id)
-        d.addErrback(self.handle_error)
-        return d
-    
-    def kill(self):
-        d = getProcessOutput(self.delete_command,
-            [self.job_id],env=os.environ)
-        return d
-
-class PBSEngineSet(BatchEngineSet):
-    
-    submit_command = 'qsub'
-    delete_command = 'qdel'
-    job_id_regexp = '\d+'
-    
-    def __init__(self, template_file, **kwargs):
-        BatchEngineSet.__init__(self, template_file, **kwargs)
-
-
-sshx_template="""#!/bin/sh
-"$@" &> /dev/null &
-echo $!
-"""
-
-engine_killer_template="""#!/bin/sh
-ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM
-"""
-
-class SSHEngineSet(object):
-    sshx_template=sshx_template
-    engine_killer_template=engine_killer_template
-    
-    def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"):
-        """Start a controller on localhost and engines using ssh.
-        
-        The engine_hosts argument is a dict with hostnames as keys and
-        the number of engine (int) as values.  sshx is the name of a local
-        file that will be used to run remote commands.  This file is used
-        to setup the environment properly.
-        """
-        
-        self.temp_dir = tempfile.gettempdir()
-        if sshx is not None:
-            self.sshx = sshx
-        else:
-            # Write the sshx.sh file locally from our template.
-            self.sshx = os.path.join(
-                self.temp_dir,
-                '%s-main-sshx.sh' % os.environ['USER']
-            )
-            f = open(self.sshx, 'w')
-            f.writelines(self.sshx_template)
-            f.close()
-        self.engine_command = ipengine
-        self.engine_hosts = engine_hosts
-        # Write the engine killer script file locally from our template.
-        self.engine_killer = os.path.join(
-            self.temp_dir,  
-            '%s-local-engine_killer.sh' % os.environ['USER']
-        )
-        f = open(self.engine_killer, 'w')
-        f.writelines(self.engine_killer_template)
-        f.close()
-    
-    def start(self, send_furl=False):
-        dlist = []
-        for host in self.engine_hosts.keys():
-            count = self.engine_hosts[host]
-            d = self._start(host, count, send_furl)
-            dlist.append(d)
-        return gatherBoth(dlist, consumeErrors=True)
-    
-    def _start(self, hostname, count=1, send_furl=False):
-        if send_furl:
-            d = self._scp_furl(hostname)
-        else:
-            d = defer.succeed(None)
-        d.addCallback(lambda r: self._scp_sshx(hostname))
-        d.addCallback(lambda r: self._ssh_engine(hostname, count))
-        return d
-        
-    def _scp_furl(self, hostname):
-        scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname)
-        cmd_list = scp_cmd.split()
-        cmd_list[1] = os.path.expanduser(cmd_list[1])
-        log.msg('Copying furl file: %s' % scp_cmd)
-        d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) 
-        return d
-    
-    def _scp_sshx(self, hostname):
-        scp_cmd = "scp %s %s:%s/%s-sshx.sh" % (
-            self.sshx, hostname, 
-            self.temp_dir, os.environ['USER']
-        )
-        print
-        log.msg("Copying sshx: %s" % scp_cmd)
-        sshx_scp = scp_cmd.split()
-        d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ)
-        return d
-    
-    def _ssh_engine(self, hostname, count):
-        exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % (
-            hostname, self.temp_dir, 
-            os.environ['USER'], self.engine_command
-        )
-        cmds = exec_engine.split()
-        dlist = []
-        log.msg("about to start engines...")
-        for i in range(count):
-            log.msg('Starting engines: %s' % exec_engine)
-            d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
-            dlist.append(d)
-        return gatherBoth(dlist, consumeErrors=True)
-    
-    def kill(self):
-        dlist = []
-        for host in self.engine_hosts.keys():
-            d = self._killall(host)
-            dlist.append(d)
-        return gatherBoth(dlist, consumeErrors=True)
-    
-    def _killall(self, hostname):
-        d = self._scp_engine_killer(hostname)
-        d.addCallback(lambda r: self._ssh_kill(hostname))
-        # d.addErrback(self._exec_err)
-        return d
-
-    def _scp_engine_killer(self, hostname):
-        scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % (
-            self.engine_killer, 
-            hostname, 
-            self.temp_dir, 
-            os.environ['USER']
-        )
-        cmds = scp_cmd.split()
-        log.msg('Copying engine_killer: %s' % scp_cmd)
-        d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
-        return d
-    
-    def _ssh_kill(self, hostname):
-        kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % (
-            hostname,
-            self.temp_dir, 
-            os.environ['USER']
-        )
-        log.msg('Killing engine: %s' % kill_cmd)
-        kill_cmd = kill_cmd.split()
-        d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ)
-        return d
-
-    def _exec_err(self, r):
-        log.msg(r)
-
-#-----------------------------------------------------------------------------
-# Main functions for the different types of clusters
-#-----------------------------------------------------------------------------
-
-# TODO:
-# The logic in these codes should be moved into classes like LocalCluster
-# MpirunCluster, PBSCluster, etc.  This would remove alot of the duplications.
-# The main functions should then just parse the command line arguments, create
-# the appropriate class and call a 'start' method.
-
-
-def check_security(args, cont_args):
-    """Check to see if we should run with SSL support."""
-    if (not args.x or not args.y) and not have_crypto:
-        log.err("""
-OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
-Try running ipcluster with the -xy flags:  ipcluster local -xy -n 4""")
-        reactor.stop()
-        return False
-    if args.x:
-        cont_args.append('-x')
-    if args.y:
-        cont_args.append('-y')
-    return True
-
-
-def check_reuse(args, cont_args):
-    """Check to see if we should try to resuse FURL files."""
-    if args.r:
-        cont_args.append('-r')
-        if args.client_port == 0 or args.engine_port == 0:
-            log.err("""
-To reuse FURL files, you must also set the client and engine ports using
-the --client-port and --engine-port options.""")
-            reactor.stop()
-            return False
-        cont_args.append('--client-port=%i' % args.client_port)
-        cont_args.append('--engine-port=%i' % args.engine_port)
-    return True
-
-
-def _err_and_stop(f):
-    """Errback to log a failure and halt the reactor on a fatal error."""
-    log.err(f)
-    reactor.stop()
-
-
-def _delay_start(cont_pid, start_engines, furl_file, reuse):
-    """Wait for controller to create FURL files and the start the engines."""
-    if not reuse:
-        if os.path.isfile(furl_file):
-            os.unlink(furl_file)
-    log.msg('Waiting for controller to finish starting...')
-    d = wait_for_file(furl_file, delay=0.2, max_tries=50)
-    d.addCallback(lambda _: log.msg('Controller started'))
-    d.addCallback(lambda _: start_engines(cont_pid))
-    return d
-
-
-def main_local(args):
-    cont_args = []
-    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
-    
-    # Check security settings before proceeding
-    if not check_security(args, cont_args):
-        return
-    
-    # See if we are reusing FURL files
-    if not check_reuse(args, cont_args):
-        return
-    
-    cl = ControllerLauncher(extra_args=cont_args)
-    dstart = cl.start()
-    def start_engines(cont_pid):
-        engine_args = []
-        engine_args.append('--logfile=%s' % \
-            pjoin(args.logdir,'ipengine%s-' % cont_pid))
-        eset = LocalEngineSet(extra_args=engine_args)
-        def shutdown(signum, frame):
-            log.msg('Stopping local cluster')
-            # We are still playing with the times here, but these seem
-            # to be reliable in allowing everything to exit cleanly.
-            eset.interrupt_then_kill(0.5)
-            cl.interrupt_then_kill(0.5)
-            reactor.callLater(1.0, reactor.stop)
-        signal.signal(signal.SIGINT,shutdown)
-        d = eset.start(args.n)
-        return d
-    config = kernel_config_manager.get_config_obj()
-    furl_file = config['controller']['engine_furl_file']
-    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
-    dstart.addErrback(_err_and_stop)
-
-
-def main_mpi(args):
-    cont_args = []
-    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
-    
-    # Check security settings before proceeding
-    if not check_security(args, cont_args):
-        return
-    
-    # See if we are reusing FURL files
-    if not check_reuse(args, cont_args):
-        return
-    
-    cl = ControllerLauncher(extra_args=cont_args)
-    dstart = cl.start()
-    def start_engines(cont_pid):
-        raw_args = [args.cmd]
-        raw_args.extend(['-n',str(args.n)])
-        raw_args.append('ipengine')
-        raw_args.append('-l')
-        raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid))
-        if args.mpi:
-            raw_args.append('--mpi=%s' % args.mpi)
-        eset = ProcessLauncher(raw_args)
-        def shutdown(signum, frame):
-            log.msg('Stopping local cluster')
-            # We are still playing with the times here, but these seem
-            # to be reliable in allowing everything to exit cleanly.
-            eset.interrupt_then_kill(1.0)
-            cl.interrupt_then_kill(1.0)
-            reactor.callLater(2.0, reactor.stop)
-        signal.signal(signal.SIGINT,shutdown)
-        d = eset.start()
-        return d
-    config = kernel_config_manager.get_config_obj()
-    furl_file = config['controller']['engine_furl_file']
-    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
-    dstart.addErrback(_err_and_stop)
-
-
-def main_pbs(args):
-    cont_args = []
-    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
-    
-    # Check security settings before proceeding
-    if not check_security(args, cont_args):
-        return
-    
-    # See if we are reusing FURL files
-    if not check_reuse(args, cont_args):
-        return
-    
-    cl = ControllerLauncher(extra_args=cont_args)
-    dstart = cl.start()
-    def start_engines(r):
-        pbs_set =  PBSEngineSet(args.pbsscript)
-        def shutdown(signum, frame):
-            log.msg('Stopping pbs cluster')
-            d = pbs_set.kill()
-            d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
-            d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
-        signal.signal(signal.SIGINT,shutdown)
-        d = pbs_set.start(args.n)
-        return d
-    config = kernel_config_manager.get_config_obj()
-    furl_file = config['controller']['engine_furl_file']
-    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
-    dstart.addErrback(_err_and_stop)
-
-
-def main_ssh(args):
-    """Start a controller on localhost and engines using ssh.
-    
-    Your clusterfile should look like::
-    
-        send_furl = False # True, if you want 
-        engines = {
-            'engine_host1' : engine_count, 
-            'engine_host2' : engine_count2
-        } 
-    """
-    clusterfile = {}
-    execfile(args.clusterfile, clusterfile)
-    if not clusterfile.has_key('send_furl'):
-        clusterfile['send_furl'] = False
-        
-    cont_args = []
-    cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
-    
-    # Check security settings before proceeding
-    if not check_security(args, cont_args):
-        return
-    
-    # See if we are reusing FURL files
-    if not check_reuse(args, cont_args):
-        return
-    
-    cl = ControllerLauncher(extra_args=cont_args)
-    dstart = cl.start()
-    def start_engines(cont_pid):
-        ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx)
-        def shutdown(signum, frame):
-            d = ssh_set.kill()
-            cl.interrupt_then_kill(1.0)
-            reactor.callLater(2.0, reactor.stop)
-        signal.signal(signal.SIGINT,shutdown)
-        d = ssh_set.start(clusterfile['send_furl'])
-        return d
-    config = kernel_config_manager.get_config_obj()
-    furl_file = config['controller']['engine_furl_file']
-    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
-    dstart.addErrback(_err_and_stop)
-
-
-def get_args():
-    base_parser = argparse.ArgumentParser(add_help=False)
-    base_parser.add_argument(
-        '-r',
-        action='store_true',
-        dest='r',
-        help='try to reuse FURL files.  Use with --client-port and --engine-port'
-    )
-    base_parser.add_argument(
-        '--client-port',
-        type=int,
-        dest='client_port',
-        help='the port the controller will listen on for client connections',
-        default=0
-    )
-    base_parser.add_argument(
-        '--engine-port',
-        type=int,
-        dest='engine_port',
-        help='the port the controller will listen on for engine connections',
-        default=0
-    )
-    base_parser.add_argument(
-        '-x',
-        action='store_true',
-        dest='x',
-        help='turn off client security'
-    )
-    base_parser.add_argument(
-        '-y',
-        action='store_true',
-        dest='y',
-        help='turn off engine security'
-    )
-    base_parser.add_argument(
-        "--logdir", 
-        type=str,
-        dest="logdir",
-        help="directory to put log files (default=$IPYTHONDIR/log)",
-        default=pjoin(get_ipython_dir(),'log')
-    )
-    base_parser.add_argument(
-        "-n",
-        "--num", 
-        type=int,
-        dest="n",
-        default=2,
-        help="the number of engines to start"
-    )
-    
-    parser = argparse.ArgumentParser(
-        description='IPython cluster startup.  This starts a controller and\
-        engines using various approaches.  Use the IPYTHONDIR environment\
-        variable to change your IPython directory from the default of\
-        .ipython or _ipython.  The log and security subdirectories of your\
-        IPython directory will be used by this script for log files and\
-        security files.'
-    )
-    subparsers = parser.add_subparsers(
-        help='available cluster types.  For help, do "ipcluster TYPE --help"')
-    
-    parser_local = subparsers.add_parser(
-        'local',
-        help='run a local cluster',
-        parents=[base_parser]
-    )
-    parser_local.set_defaults(func=main_local)
-    
-    parser_mpirun = subparsers.add_parser(
-        'mpirun',
-        help='run a cluster using mpirun (mpiexec also works)',
-        parents=[base_parser]
-    )
-    parser_mpirun.add_argument(
-        "--mpi",
-        type=str,
-        dest="mpi", # Don't put a default here to allow no MPI support
-        help="how to call MPI_Init (default=mpi4py)"
-    )
-    parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun')
-
-    parser_mpiexec = subparsers.add_parser(
-        'mpiexec',
-        help='run a cluster using mpiexec (mpirun also works)',
-        parents=[base_parser]
-    )
-    parser_mpiexec.add_argument(
-        "--mpi",
-        type=str,
-        dest="mpi", # Don't put a default here to allow no MPI support
-        help="how to call MPI_Init (default=mpi4py)"
-    )
-    parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec')
-    
-    parser_pbs = subparsers.add_parser(
-        'pbs', 
-        help='run a pbs cluster',
-        parents=[base_parser]
-    )
-    parser_pbs.add_argument(
-        '--pbs-script',
-        type=str, 
-        dest='pbsscript',
-        help='PBS script template',
-        default='pbs.template'
-    )
-    parser_pbs.set_defaults(func=main_pbs)
-    
-    parser_ssh = subparsers.add_parser(
-        'ssh',
-        help='run a cluster using ssh, should have ssh-keys setup',
-        parents=[base_parser]
-    )
-    parser_ssh.add_argument(
-        '--clusterfile', 
-        type=str,
-        dest='clusterfile',
-        help='python file describing the cluster',
-        default='clusterfile.py',
-    )
-    parser_ssh.add_argument(
-        '--sshx', 
-        type=str,
-        dest='sshx',
-        help='sshx launcher helper'
-    )
-    parser_ssh.set_defaults(func=main_ssh)
-    
-    args = parser.parse_args()
-    return args
-
-def main():
-    args = get_args()
-    reactor.callWhenRunning(args.func, args)
-    log.startLogging(sys.stdout)
-    reactor.run()
-
-if __name__ == '__main__':
-    main()

IPython/kernel/scripts/ipcontroller.py

@@ -1,416 +0,0 b''
-#!/usr/bin/env python
-# encoding: utf-8
-
-"""The IPython controller."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-# Python looks for an empty string at the beginning of sys.path to enable
-# importing from the cwd.
-import sys
-sys.path.insert(0, '')
-
-from optparse import OptionParser
-import os
-import time
-import tempfile
-
-from twisted.application import internet, service
-from twisted.internet import reactor, error, defer
-from twisted.python import log
-
-from IPython.kernel.fcutil import Tub, UnauthenticatedTub, have_crypto
-
-# from IPython.utils import growl
-# growl.start("IPython1 Controller")
-
-from IPython.kernel.error import SecurityError
-from IPython.kernel import controllerservice
-from IPython.kernel.fcutil import check_furl_file_security
-
-# Create various ipython directories if they don't exist.
-# This must be done before IPython.kernel.config is imported.
-from IPython.core.oldusersetup import user_setup
-from IPython.utils.genutils import get_ipython_dir, get_log_dir, get_security_dir
-if os.name == 'posix':
-    rc_suffix = ''
-else:
-    rc_suffix = '.ini'
-user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
-get_log_dir()
-get_security_dir()
-
-from IPython.kernel.config import config_manager as kernel_config_manager
-from IPython.utils.importstring import import_item
-
-
-#-------------------------------------------------------------------------------
-# Code
-#-------------------------------------------------------------------------------
-
-def get_temp_furlfile(filename):
-    return tempfile.mktemp(dir=os.path.dirname(filename),
-                           prefix=os.path.basename(filename))
-
-def make_tub(ip, port, secure, cert_file):
-    """
-    Create a listening tub given an ip, port, and cert_file location.
-    
-    :Parameters:
-        ip : str
-            The ip address that the tub should listen on.  Empty means all
-        port : int
-            The port that the tub should listen on.  A value of 0 means
-            pick a random port
-        secure: boolean
-            Will the connection be secure (in the foolscap sense)
-        cert_file:
-            A filename of a file to be used for theSSL certificate
-    """
-    if secure:
-        if have_crypto:
-            tub = Tub(certFile=cert_file)
-        else:
-            raise SecurityError("""
-OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
-Try running without security using 'ipcontroller -xy'.
-""")
-    else:
-        tub = UnauthenticatedTub()
-    
-    # Set the strport based on the ip and port and start listening
-    if ip == '':
-        strport = "tcp:%i" % port
-    else:
-        strport = "tcp:%i:interface=%s" % (port, ip)
-    listener = tub.listenOn(strport)
-    
-    return tub, listener
-
-def make_client_service(controller_service, config):
-    """
-    Create a service that will listen for clients.
-    
-    This service is simply a `foolscap.Tub` instance that has a set of Referenceables
-    registered with it.
-    """
-
-    # Now create the foolscap tub
-    ip = config['controller']['client_tub']['ip']
-    port = config['controller']['client_tub'].as_int('port')
-    location = config['controller']['client_tub']['location']
-    secure = config['controller']['client_tub']['secure']
-    cert_file = config['controller']['client_tub']['cert_file']
-    client_tub, client_listener = make_tub(ip, port, secure, cert_file)
-    
-    # Set the location in the trivial case of localhost
-    if ip == 'localhost' or ip == '127.0.0.1':
-        location = "127.0.0.1"
-    
-    if not secure:
-        log.msg("WARNING: you are running the controller with no client security")
-    
-    def set_location_and_register():
-        """Set the location for the tub and return a deferred."""
-
-        def register(empty, ref, furl_file):
-            # We create and then move to make sure that when the file
-            # appears to other processes, the buffer has the flushed
-            # and the file has been closed
-            temp_furl_file = get_temp_furlfile(furl_file)
-            client_tub.registerReference(ref, furlFile=temp_furl_file)
-            os.rename(temp_furl_file, furl_file)
-        
-        if location == '':
-            d = client_tub.setLocationAutomatically()
-        else:
-            d = defer.maybeDeferred(client_tub.setLocation, "%s:%i" % (location, client_listener.getPortnum()))
-        
-        for ciname, ci in config['controller']['controller_interfaces'].iteritems():
-            log.msg("Adapting Controller to interface: %s" % ciname)
-            furl_file = ci['furl_file']
-            log.msg("Saving furl for interface [%s] to file: %s" % (ciname, furl_file))
-            check_furl_file_security(furl_file, secure)
-            adapted_controller = import_item(ci['controller_interface'])(controller_service)
-            d.addCallback(register, import_item(ci['fc_interface'])(adapted_controller), 
-                furl_file=ci['furl_file'])
-    
-    reactor.callWhenRunning(set_location_and_register)
-    return client_tub
-
-
-def make_engine_service(controller_service, config):
-    """
-    Create a service that will listen for engines.
-    
-    This service is simply a `foolscap.Tub` instance that has a set of Referenceables
-    registered with it.
-    """
-
-    # Now create the foolscap tub
-    ip = config['controller']['engine_tub']['ip']
-    port = config['controller']['engine_tub'].as_int('port')
-    location = config['controller']['engine_tub']['location']
-    secure = config['controller']['engine_tub']['secure']
-    cert_file = config['controller']['engine_tub']['cert_file']
-    engine_tub, engine_listener = make_tub(ip, port, secure, cert_file)
-    
-    # Set the location in the trivial case of localhost
-    if ip == 'localhost' or ip == '127.0.0.1':
-        location = "127.0.0.1"
-    
-    if not secure:
-        log.msg("WARNING: you are running the controller with no engine security")
-    
-    def set_location_and_register():
-        """Set the location for the tub and return a deferred."""
-
-        def register(empty, ref, furl_file):
-            # We create and then move to make sure that when the file
-            # appears to other processes, the buffer has the flushed
-            # and the file has been closed
-            temp_furl_file = get_temp_furlfile(furl_file)
-            engine_tub.registerReference(ref, furlFile=temp_furl_file)
-            os.rename(temp_furl_file, furl_file)
-        
-        if location == '':
-            d = engine_tub.setLocationAutomatically()
-        else:
-            d = defer.maybeDeferred(engine_tub.setLocation, "%s:%i" % (location, engine_listener.getPortnum()))
-    
-        furl_file = config['controller']['engine_furl_file']
-        engine_fc_interface = import_item(config['controller']['engine_fc_interface'])
-        log.msg("Saving furl for the engine to file: %s" % furl_file)
-        check_furl_file_security(furl_file, secure)
-        fc_controller = engine_fc_interface(controller_service)
-        d.addCallback(register, fc_controller, furl_file=furl_file)
-    
-    reactor.callWhenRunning(set_location_and_register)
-    return engine_tub
-    
-def start_controller():
-    """
-    Start the controller by creating the service hierarchy and starting the reactor.
-    
-    This method does the following:
-    
-        * It starts the controller logging
-        * In execute an import statement for the controller
-        * It creates 2 `foolscap.Tub` instances for the client and the engines
-          and registers `foolscap.Referenceables` with the tubs to expose the
-          controller to engines and clients.
-    """
-    config = kernel_config_manager.get_config_obj()
-    
-    # Start logging
-    logfile = config['controller']['logfile']
-    if logfile:
-        logfile = logfile + str(os.getpid()) + '.log'
-        try:
-            openLogFile = open(logfile, 'w')
-        except:
-            openLogFile = sys.stdout
-    else:
-        openLogFile = sys.stdout
-    log.startLogging(openLogFile)
-    
-    # Execute any user defined import statements
-    cis = config['controller']['import_statement']
-    if cis:
-        try:
-            exec cis in globals(), locals()
-        except:
-            log.msg("Error running import_statement: %s" % cis)
-    
-    # Delete old furl files unless the reuse_furls is set
-    reuse = config['controller']['reuse_furls']
-    if not reuse:
-        paths = (config['controller']['engine_furl_file'],
-            config['controller']['controller_interfaces']['task']['furl_file'],
-            config['controller']['controller_interfaces']['multiengine']['furl_file']
-        )
-        for p in paths:
-            if os.path.isfile(p):
-                os.remove(p)
-    
-    # Create the service hierarchy
-    main_service = service.MultiService()
-    # The controller service
-    controller_service = controllerservice.ControllerService()
-    controller_service.setServiceParent(main_service)
-    # The client tub and all its refereceables
-    client_service = make_client_service(controller_service, config)
-    client_service.setServiceParent(main_service)
-    # The engine tub
-    engine_service = make_engine_service(controller_service, config)
-    engine_service.setServiceParent(main_service)
-    # Start the controller service and set things running
-    main_service.startService()
-    reactor.run()
-
-def init_config():
-    """
-    Initialize the configuration using default and command line options.
-    """
-    
-    parser = OptionParser("""ipcontroller [options]
-
-Start an IPython controller.
-
-Use the IPYTHONDIR environment variable to change your IPython directory 
-from the default of .ipython or _ipython.  The log and security 
-subdirectories of your IPython directory will be used by this script 
-for log files and security files.""")
-    
-    # Client related options
-    parser.add_option(
-        "--client-ip", 
-        type="string",
-        dest="client_ip",
-        help="the IP address or hostname the controller will listen on for client connections"
-    )
-    parser.add_option(
-        "--client-port", 
-        type="int", 
-        dest="client_port",
-        help="the port the controller will listen on for client connections"
-    )    
-    parser.add_option(
-        '--client-location',
-        type="string",
-        dest="client_location",
-        help="hostname or ip for clients to connect to"
-    )
-    parser.add_option(
-        "-x",
-        action="store_false",
-        dest="client_secure",
-        help="turn off all client security"
-    )
-    parser.add_option(
-        '--client-cert-file',
-        type="string",
-        dest="client_cert_file",
-        help="file to store the client SSL certificate"
-    )
-    parser.add_option(
-        '--task-furl-file',
-        type="string",
-        dest="task_furl_file",
-        help="file to store the FURL for task clients to connect with"
-    )
-    parser.add_option(
-        '--multiengine-furl-file',
-        type="string",
-        dest="multiengine_furl_file",
-        help="file to store the FURL for multiengine clients to connect with"
-    )
-    # Engine related options
-    parser.add_option(
-        "--engine-ip", 
-        type="string",
-        dest="engine_ip",
-        help="the IP address or hostname the controller will listen on for engine connections"
-    )
-    parser.add_option(
-        "--engine-port", 
-        type="int", 
-        dest="engine_port",
-        help="the port the controller will listen on for engine connections"
-    )    
-    parser.add_option(
-        '--engine-location',
-        type="string",
-        dest="engine_location",
-        help="hostname or ip for engines to connect to"
-    )
-    parser.add_option(
-        "-y",
-        action="store_false",
-        dest="engine_secure",
-        help="turn off all engine security"
-    )
-    parser.add_option(
-        '--engine-cert-file',
-        type="string",
-        dest="engine_cert_file",
-        help="file to store the engine SSL certificate"
-    )
-    parser.add_option(
-        '--engine-furl-file',
-        type="string",
-        dest="engine_furl_file",
-        help="file to store the FURL for engines to connect with"
-    )
-    parser.add_option(
-        "-l", "--logfile",
-        type="string",
-        dest="logfile",
-        help="log file name (default is stdout)"
-    )
-    parser.add_option(
-        "-r",
-        action="store_true",
-        dest="reuse_furls",
-        help="try to reuse all furl files"
-    )
-    
-    (options, args) = parser.parse_args()
-    
-    config = kernel_config_manager.get_config_obj()
-    
-    # Update with command line options
-    if options.client_ip is not None:
-        config['controller']['client_tub']['ip'] = options.client_ip
-    if options.client_port is not None:
-        config['controller']['client_tub']['port'] = options.client_port
-    if options.client_location is not None:
-        config['controller']['client_tub']['location'] = options.client_location
-    if options.client_secure is not None:
-        config['controller']['client_tub']['secure'] = options.client_secure
-    if options.client_cert_file is not None:
-        config['controller']['client_tub']['cert_file'] = options.client_cert_file
-    if options.task_furl_file is not None:
-        config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
-    if options.multiengine_furl_file is not None:
-        config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
-    if options.engine_ip is not None:
-        config['controller']['engine_tub']['ip'] = options.engine_ip
-    if options.engine_port is not None:
-        config['controller']['engine_tub']['port'] = options.engine_port
-    if options.engine_location is not None:
-        config['controller']['engine_tub']['location'] = options.engine_location
-    if options.engine_secure is not None:
-        config['controller']['engine_tub']['secure'] = options.engine_secure
-    if options.engine_cert_file is not None:
-        config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
-    if options.engine_furl_file is not None:
-        config['controller']['engine_furl_file'] = options.engine_furl_file
-    if options.reuse_furls is not None:
-            config['controller']['reuse_furls'] = options.reuse_furls
-
-    if options.logfile is not None:
-        config['controller']['logfile'] = options.logfile
-    
-    kernel_config_manager.update_config_obj(config)
-
-def main():
-    """
-    After creating the configuration information, start the controller.
-    """
-    init_config()
-    start_controller()
-    
-if __name__ == "__main__":
-    main()

IPython/kernel/scripts/ipengine.py

@@ -1,193 +0,0 b''
-#!/usr/bin/env python
-# encoding: utf-8
-
-"""Start the IPython Engine."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
-
-# Python looks for an empty string at the beginning of sys.path to enable
-# importing from the cwd.
-import sys
-sys.path.insert(0, '')
-
-from optparse import OptionParser
-import os
-
-from twisted.application import service
-from twisted.internet import reactor
-from twisted.python import log
-
-from IPython.kernel.fcutil import Tub, UnauthenticatedTub
-
-from IPython.kernel.core.config import config_manager as core_config_manager
-from IPython.utils.importstring import import_item
-from IPython.kernel.engineservice import EngineService
-
-# Create various ipython directories if they don't exist.
-# This must be done before IPython.kernel.config is imported.
-from IPython.core.oldusersetup import user_setup
-from IPython.utils.genutils import get_ipython_dir, get_log_dir, get_security_dir
-if os.name == 'posix':
-    rc_suffix = ''
-else:
-    rc_suffix = '.ini'
-user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
-get_log_dir()
-get_security_dir()
-
-from IPython.kernel.config import config_manager as kernel_config_manager
-from IPython.kernel.engineconnector import EngineConnector
-
-
-#-------------------------------------------------------------------------------
-# Code
-#-------------------------------------------------------------------------------
-
-def start_engine():
-    """
-    Start the engine, by creating it and starting the Twisted reactor.
-    
-    This method does:
-    
-        * If it exists, runs the `mpi_import_statement` to call `MPI_Init`
-        * Starts the engine logging
-        * Creates an IPython shell and wraps it in an `EngineService`
-        * Creates a `foolscap.Tub` to use in connecting to a controller.
-        * Uses the tub and the `EngineService` along with a Foolscap URL
-          (or FURL) to connect to the controller and register the engine
-          with the controller
-    """
-    kernel_config = kernel_config_manager.get_config_obj()
-    core_config = core_config_manager.get_config_obj()
-    
-
-    # Execute the mpi import statement that needs to call MPI_Init
-    global mpi
-    mpikey = kernel_config['mpi']['default']
-    mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
-    if mpi_import_statement is not None:
-        try:
-            exec mpi_import_statement in globals()
-        except:
-            mpi = None
-    else:
-        mpi = None
-    
-    # Start logging
-    logfile = kernel_config['engine']['logfile']
-    if logfile:
-        logfile = logfile + str(os.getpid()) + '.log'
-        try:
-            openLogFile = open(logfile, 'w')
-        except:
-            openLogFile = sys.stdout
-    else:
-        openLogFile = sys.stdout
-    log.startLogging(openLogFile)
-    
-    # Create the underlying shell class and EngineService
-    shell_class = import_item(core_config['shell']['shell_class'])
-    engine_service = EngineService(shell_class, mpi=mpi)
-    shell_import_statement = core_config['shell']['import_statement']
-    if shell_import_statement:
-        try:
-            engine_service.execute(shell_import_statement)
-        except:
-            log.msg("Error running import_statement: %s" % shell_import_statement)
-    
-    # Create the service hierarchy
-    main_service = service.MultiService()
-    engine_service.setServiceParent(main_service)
-    tub_service = Tub()
-    tub_service.setServiceParent(main_service)
-    # This needs to be called before the connection is initiated
-    main_service.startService()
-    
-    # This initiates the connection to the controller and calls
-    # register_engine to tell the controller we are ready to do work
-    engine_connector = EngineConnector(tub_service)
-    furl_file = kernel_config['engine']['furl_file']
-    log.msg("Using furl file: %s" % furl_file)
-    
-    def call_connect(engine_service, furl_file):
-        d = engine_connector.connect_to_controller(engine_service, furl_file)
-        def handle_error(f):
-            # If this print statement is replaced by a log.err(f) I get
-            # an unhandled error, which makes no sense.  I shouldn't have
-            # to use a print statement here.  My only thought is that
-            # at the beginning of the process the logging is still starting up
-            print "error connecting to controller:", f.getErrorMessage()
-            reactor.callLater(0.1, reactor.stop)
-        d.addErrback(handle_error)
-    
-    reactor.callWhenRunning(call_connect, engine_service, furl_file)
-    reactor.run()
-
-
-def init_config():
-    """
-    Initialize the configuration using default and command line options.
-    """
-    
-    parser = OptionParser("""ipengine [options]
-
-Start an IPython engine.
-
-Use the IPYTHONDIR environment variable to change your IPython directory 
-from the default of .ipython or _ipython.  The log and security 
-subdirectories of your IPython directory will be used by this script 
-for log files and security files.""")
-    
-    parser.add_option(
-        "--furl-file",
-        type="string", 
-        dest="furl_file",
-        help="The filename containing the FURL of the controller"
-    )
-    parser.add_option(
-        "--mpi",
-        type="string",
-        dest="mpi",
-        help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
-    )
-    parser.add_option(
-        "-l",
-        "--logfile",
-        type="string",
-        dest="logfile",
-        help="log file name (default is stdout)"
-    )
-    
-    (options, args) = parser.parse_args()
-    
-    kernel_config = kernel_config_manager.get_config_obj()
-    # Now override with command line options
-    if options.furl_file is not None:
-        kernel_config['engine']['furl_file'] = options.furl_file
-    if options.logfile is not None:
-        kernel_config['engine']['logfile'] = options.logfile
-    if options.mpi is not None:
-        kernel_config['mpi']['default'] = options.mpi
-
-
-def main():
-    """
-    After creating the configuration information, start the engine.
-    """
-    init_config()
-    start_engine()
-
-
-if __name__ == "__main__":
-    main()

IPython/quarantine/InterpreterPasteInput.py

@@ -1,124 +0,0 b''
-# -*- coding: utf-8 -*-
-"""Modified input prompt for entering text with >>> or ... at the start.
-
-We define a special input line filter to allow typing lines which begin with
-'>>> ' or '... '. These two strings, if present at the start of the input
-line, are stripped. This allows for direct pasting of code from examples such
-as those available in the standard Python tutorial.
-
-Normally pasting such code is one chunk is impossible because of the
-extraneous >>> and ..., requiring one to do a line by line paste with careful
-removal of those characters. This module allows pasting that kind of
-multi-line examples in one pass.
-
-Here is an 'screenshot' of a section of the tutorial pasted into IPython with
-this feature enabled:
-
-In [1]: >>> def fib2(n): # return Fibonacci series up to n
-   ...: ...     '''Return a list containing the Fibonacci series up to n.'''
-   ...: ...     result = []
-   ...: ...     a, b = 0, 1
-   ...: ...     while b < n:
-   ...: ...         result.append(b)    # see below
-   ...: ...         a, b = b, a+b
-   ...: ...     return result
-   ...:
-
-In [2]: fib2(10)
-Out[2]: [1, 1, 2, 3, 5, 8]
-
-The >>> and ... are stripped from the input so that the python interpreter
-only sees the real part of the code.
-
-All other input is processed normally.
-
-Notes
-=====
-
-* You can even paste code that has extra initial spaces, such as is common in
-doctests:
-
-In [3]:     >>> a = ['Mary', 'had', 'a', 'little', 'lamb']
-
-In [4]:     >>> for i in range(len(a)):
-   ...:         ...     print i, a[i]
-   ...:     ...
-0 Mary
-1 had
-2 a
-3 little
-4 lamb
-
-
-Authors
--------
-- Fernando Perez <[e-mail]>
-"""
-
-#*****************************************************************************
-#       Copyright (C) 2008-2009 The IPython Development Team
-#       Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#*****************************************************************************
-
-# This file is an example of how to modify IPython's line-processing behavior
-# without touching the internal code. We'll define an alternate pre-processing
-# stage which allows a special form of input (which is invalid Python syntax)
-# for certain quantities, rewrites a line of proper Python in those cases, and
-# then passes it off to IPython's normal processor for further work.
-
-# With this kind of customization, IPython can be adapted for many
-# special-purpose scenarios providing alternate input syntaxes.
-
-# This file can be imported like a regular module.
-
-# IPython has a prefilter() function that analyzes each input line. We redefine
-# it here to first pre-process certain forms of input
-
-# The prototype of any alternate prefilter must be like this one (the name
-# doesn't matter):
-# - line is a string containing the user input line.
-# - continuation is a parameter which tells us if we are processing a first
-# line of user input or the second or higher of a multi-line statement.
-
-import re
-
-from IPython.core.iplib import InteractiveShell
-
-PROMPT_RE = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
-
-def prefilter_paste(self,line,continuation):
-    """Alternate prefilter for input of pasted code from an interpreter.
-    """
-    if not line:
-        return ''
-    m = PROMPT_RE.match(line)
-    if m:
-        # In the end, always call the default IPython _prefilter() function.
-        # Note that self must be passed explicitly, b/c we're calling the
-        # unbound class method (since this method will overwrite the instance
-        # prefilter())
-        return self._prefilter(line[len(m.group(0)):],continuation)
-    elif line.strip() == '...':
-        return self._prefilter('',continuation)
-    elif line.isspace():
-        # This allows us to recognize multiple input prompts separated by blank
-        # lines and pasted in a single chunk, very common when pasting doctests
-        # or long tutorial passages.
-        return ''
-    else:
-        return self._prefilter(line,continuation)
-
-def activate_prefilter():
-    """Rebind the input-pasting filter to be the new IPython prefilter"""
-    InteractiveShell.prefilter = prefilter_paste
-
-def deactivate_prefilter():
-    """Reset the filter."""
-    InteractiveShell.prefilter = InteractiveShell._prefilter
-
-# Just a heads up at the console
-activate_prefilter()
-print '*** Pasting of code with ">>>" or "..." has been enabled.'

IPython/testing/decorators_numpy.py

@@ -1,90 +0,0 b''
-"""Decorators for labeling test objects
-
-Decorators that merely return a modified version of the original
-function object are straightforward.  Decorators that return a new
-function object need to use
-nose.tools.make_decorator(original_function)(decorator) in returning
-the decorator, in order to preserve metadata such as function name,
-setup and teardown functions and so on - see nose.tools for more
-information.
-
-"""
-
-def slow(t):
-    """Labels a test as 'slow'.
-
-    The exact definition of a slow test is obviously both subjective and
-    hardware-dependent, but in general any individual test that requires more
-    than a second or two should be labeled as slow (the whole suite consits of
-    thousands of tests, so even a second is significant)."""
-
-    t.slow = True
-    return t
-
-def setastest(tf=True):
-    ''' Signals to nose that this function is or is not a test
-
-    Parameters
-    ----------
-    tf : bool
-        If True specifies this is a test, not a test otherwise
-
-    This decorator cannot use the nose namespace, because it can be
-    called from a non-test module. See also istest and nottest in
-    nose.tools
-
-    '''
-    def set_test(t):
-        t.__test__ = tf
-        return t
-    return set_test
-
-def skipif(skip_condition=True, msg=None):
-    ''' Make function raise SkipTest exception if skip_condition is true
-
-    Parameters
-    ----------
-    skip_condition : bool or callable. 
-        Flag to determine whether to skip test.  If the condition is a 
-        callable, it is used at runtime to dynamically make the decision.  This 
-        is useful for tests that may require costly imports, to delay the cost 
-        until the test suite is actually executed.        
-    msg : string
-        Message to give on raising a SkipTest exception
-
-   Returns
-   -------
-   decorator : function
-       Decorator, which, when applied to a function, causes SkipTest
-       to be raised when the skip_condition was True, and the function
-       to be called normally otherwise.
-
-    Notes
-    -----
-    You will see from the code that we had to further decorate the
-    decorator with the nose.tools.make_decorator function in order to
-    transmit function name, and various other metadata.
-    '''
-    if msg is None:
-        msg = 'Test skipped due to test condition'
-    def skip_decorator(f):
-        # Local import to avoid a hard nose dependency and only incur the 
-        # import time overhead at actual test-time. 
-        import nose
-        def skipper(*args, **kwargs):
-            if skip_condition:
-                raise nose.SkipTest, msg
-            else:
-                return f(*args, **kwargs)
-        return nose.tools.make_decorator(f)(skipper)
-    return skip_decorator
-
-def skipknownfailure(f):
-    ''' Decorator to raise SkipTest for test known to fail
-    '''
-    # Local import to avoid a hard nose dependency and only incur the 
-    # import time overhead at actual test-time. 
-    import nose
-    def skipper(*args, **kwargs):
-        raise nose.SkipTest, 'This test is known to fail'
-    return nose.tools.make_decorator(f)(skipper)

IPython/testing/decorators_trial.py

@@ -1,132 +0,0 b''
-# encoding: utf-8
-"""
-Testing related decorators for use with twisted.trial.
-
-The decorators in this files are designed to follow the same API as those
-in the decorators module (in this same directory).  But they can be used
-with twisted.trial
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import os
-import sys
-
-from IPython.testing.decorators import make_label_dec
-
-#-----------------------------------------------------------------------------
-# Testing decorators
-#-----------------------------------------------------------------------------
-
-
-def skipif(skip_condition, msg=None):
-    """Create a decorator that marks a test function for skipping.
-
-    The is a decorator factory that returns a decorator that will 
-    conditionally skip a test based on the value of skip_condition.  The
-    skip_condition argument can either be a boolean or a callable that returns
-    a boolean.
-
-    Parameters
-    ----------
-    skip_condition : boolean or callable
-        If this evaluates to True, the test is skipped.
-    msg : str
-        The message to print if the test is skipped.
-
-    Returns
-    -------
-    decorator : function
-        The decorator function that can be applied to the test function.
-    """
-
-    def skip_decorator(f):
-
-        # Allow for both boolean or callable skip conditions.
-        if callable(skip_condition):
-            skip_val = lambda : skip_condition()
-        else:
-            skip_val = lambda : skip_condition
-
-        if msg is None:
-            out = 'Test skipped due to test condition.'
-        else: 
-            out = msg
-        final_msg = "Skipping test: %s. %s" % (f.__name__,out)
-
-        if skip_val():
-            f.skip = final_msg
-
-        return f
-    return skip_decorator
-
-
-def skip(msg=None):
-    """Create a decorator that marks a test function for skipping.
-
-    This is a decorator factory that returns a decorator that will cause
-    tests to be skipped.
-
-    Parameters
-    ----------
-    msg : str
-        Optional message to be added.
-
-    Returns
-    -------
-    decorator : function
-        Decorator, which, when applied to a function, sets the skip
-        attribute of the function causing `twisted.trial` to skip it.
-    """
-
-    return skipif(True,msg)
-
-
-def numpy_not_available():
-    """Can numpy be imported?  Returns true if numpy does NOT import.
-
-    This is used to make a decorator to skip tests that require numpy to be
-    available, but delay the 'import numpy' to test execution time.
-    """
-    try:
-        import numpy
-        np_not_avail = False
-    except ImportError:
-        np_not_avail = True
-
-    return np_not_avail
-
-#-----------------------------------------------------------------------------
-# Decorators for public use
-#-----------------------------------------------------------------------------
-
-# Decorators to skip certain tests on specific platforms.
-skip_win32 = skipif(sys.platform == 'win32',
-                    "This test does not run under Windows")
-skip_linux = skipif(sys.platform == 'linux2',
-                    "This test does not run under Linux")
-skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
-
-# Decorators to skip tests if not on specific platforms.
-skip_if_not_win32 = skipif(sys.platform != 'win32',
-                           "This test only runs under Windows")
-skip_if_not_linux = skipif(sys.platform != 'linux2',
-                           "This test only runs under Linux")
-skip_if_not_osx = skipif(sys.platform != 'darwin',
-                         "This test only runs under OSX")
-
-# Other skip decorators
-skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
-
-skipknownfailure = skip('This test is known to fail')
-
-