merged from ~fdo.perez/ipython/trunk-dev

IPython/config/default/ipcluster_config.py

@@ -0,0 +1,184 b''
+import os
+
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Select which launchers to use
+#-----------------------------------------------------------------------------
+
+# This allows you to control what method is used to start the controller
+# and engines.  The following methods are currently supported:
+# - Start as a regular process on localhost.
+# - Start using mpiexec.
+# - Start using the Windows HPC Server 2008 scheduler
+# - Start using PBS
+# - Start using SSH (currently broken)
+
+
+# The selected launchers can be configured below.
+
+# Options are:
+# - LocalControllerLauncher
+# - MPIExecControllerLauncher
+# - PBSControllerLauncher
+# - WindowsHPCControllerLauncher
+# c.Global.controller_launcher = 'IPython.kernel.launcher.LocalControllerLauncher'
+
+# Options are:
+# - LocalEngineSetLauncher
+# - MPIExecEngineSetLauncher
+# - PBSEngineSetLauncher
+# - WindowsHPCEngineSetLauncher
+# c.Global.engine_launcher = 'IPython.kernel.launcher.LocalEngineSetLauncher'
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# The default number of engines that will be started. This is overridden by
+# the -n command line option: "ipcluster start -n 4"
+# c.Global.n = 2
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+
+#-----------------------------------------------------------------------------
+# Local process launchers
+#-----------------------------------------------------------------------------
+
+# The command line arguments to call the controller with.
+# c.LocalControllerLauncher.controller_args = \
+#    ['--log-to-file','--log-level', '40']
+
+# The working directory for the controller
+# c.LocalEngineSetLauncher.work_dir = u''
+
+# Command line argument passed to the engines.
+# c.LocalEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
+
+#-----------------------------------------------------------------------------
+# MPIExec launchers
+#-----------------------------------------------------------------------------
+
+# The mpiexec/mpirun command to use in started the controller.
+# c.MPIExecControllerLauncher.mpi_cmd = ['mpiexec']
+
+# Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecControllerLauncher.mpi_args = []
+
+# The command line argument to call the controller with.
+# c.MPIExecControllerLauncher.controller_args = \
+#     ['--log-to-file','--log-level', '40']
+
+
+# The mpiexec/mpirun command to use in started the controller.
+# c.MPIExecEngineSetLauncher.mpi_cmd = ['mpiexec']
+
+# Additional arguments to pass to the actual mpiexec command.
+# c.MPIExecEngineSetLauncher.mpi_args = []
+
+# Command line argument passed to the engines.
+# c.MPIExecEngineSetLauncher.engine_args = ['--log-to-file','--log-level', '40']
+
+# The default number of engines to start if not given elsewhere.
+# c.MPIExecEngineSetLauncher.n = 1
+
+#-----------------------------------------------------------------------------
+# SSH launchers
+#-----------------------------------------------------------------------------
+
+# Todo
+
+
+#-----------------------------------------------------------------------------
+# Unix batch (PBS) schedulers launchers
+#-----------------------------------------------------------------------------
+
+# The command line program to use to submit a PBS job.
+# c.PBSControllerLauncher.submit_command = 'qsub'
+
+# The command line program to use to delete a PBS job.
+# c.PBSControllerLauncher.delete_command = 'qdel'
+
+# A regular expression that takes the output of qsub and find the job id.
+# c.PBSControllerLauncher.job_id_regexp = r'\d+'
+
+# The batch submission script used to start the controller. This is where
+# environment variables would be setup, etc. This string is interpolated using
+# the Itpl module in IPython.external. Basically, you can use ${n} for the
+# number of engine and ${cluster_dir} for the cluster_dir.
+# c.PBSControllerLauncher.batch_template = """"""
+
+# The name of the instantiated batch script that will actually be used to
+# submit the job. This will be written to the cluster directory.
+# c.PBSControllerLauncher.batch_file_name = u'pbs_batch_script_controller'
+
+
+# The command line program to use to submit a PBS job.
+# c.PBSEngineSetLauncher.submit_command = 'qsub'
+
+# The command line program to use to delete a PBS job.
+# c.PBSEngineSetLauncher.delete_command = 'qdel'
+
+# A regular expression that takes the output of qsub and find the job id.
+# c.PBSEngineSetLauncher.job_id_regexp = r'\d+'
+
+# The batch submission script used to start the engines. This is where
+# environment variables would be setup, etc. This string is interpolated using
+# the Itpl module in IPython.external. Basically, you can use ${n} for the
+# number of engine and ${cluster_dir} for the cluster_dir.
+# c.PBSEngineSetLauncher.batch_template = """"""
+
+# The name of the instantiated batch script that will actually be used to
+# submit the job. This will be written to the cluster directory.
+# c.PBSEngineSetLauncher.batch_file_name = u'pbs_batch_script_engines'
+
+#-----------------------------------------------------------------------------
+# Windows HPC Server 2008 launcher configuration
+#-----------------------------------------------------------------------------
+
+# c.IPControllerJob.job_name = 'IPController'
+# c.IPControllerJob.is_exclusive = False
+# c.IPControllerJob.username = r'USERDOMAIN\USERNAME'
+# c.IPControllerJob.priority = 'Highest'
+# c.IPControllerJob.requested_nodes = ''
+# c.IPControllerJob.project = 'MyProject'
+
+# c.IPControllerTask.task_name = 'IPController'
+# c.IPControllerTask.controller_cmd = [u'ipcontroller.exe']
+# c.IPControllerTask.controller_args = ['--log-to-file', '--log-level', '40']
+# c.IPControllerTask.environment_variables = {}
+
+# c.WindowsHPCControllerLauncher.scheduler = 'HEADNODE'
+# c.WindowsHPCControllerLauncher.job_file_name = u'ipcontroller_job.xml'
+
+
+# c.IPEngineSetJob.job_name = 'IPEngineSet'
+# c.IPEngineSetJob.is_exclusive = False
+# c.IPEngineSetJob.username = r'USERDOMAIN\USERNAME'
+# c.IPEngineSetJob.priority = 'Highest'
+# c.IPEngineSetJob.requested_nodes = ''
+# c.IPEngineSetJob.project = 'MyProject'
+
+# c.IPEngineTask.task_name = 'IPEngine'
+# c.IPEngineTask.engine_cmd = [u'ipengine.exe']
+# c.IPEngineTask.engine_args = ['--log-to-file', '--log-level', '40']
+# c.IPEngineTask.environment_variables = {}
+
+# c.WindowsHPCEngineSetLauncher.scheduler = 'HEADNODE'
+# c.WindowsHPCEngineSetLauncher.job_file_name = u'ipengineset_job.xml'
+
+
+
+
+
+
+

IPython/config/default/ipcontroller_config.py

@@ -0,0 +1,136 b''
+from IPython.config.loader import Config
+
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# Basic Global config attributes
+
+# Start up messages are logged to stdout using the logging module.
+# These all happen before the twisted reactor is started and are
+# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
+# and smaller is more verbose.
+# c.Global.log_level = 20
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# A list of Python statements that will be run before starting the 
+# controller. This is provided because occasionally certain things need to 
+# be imported in the controller for pickling to work.
+# c.Global.import_statements = ['import math']
+
+# Reuse the controller's FURL files. If False, FURL files are regenerated
+# each time the controller is run. If True, they will be reused, *but*, you
+# also must set the network ports by hand. If set, this will override the
+# values set for the client and engine connections below.
+# c.Global.reuse_furls = True
+
+# Enable SSL encryption on all connections to the controller. If set, this
+# will override the values set for the client and engine connections below.
+# c.Global.secure = True
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+#-----------------------------------------------------------------------------
+# Configure the client services
+#-----------------------------------------------------------------------------
+
+# Basic client service config attributes
+
+# The network interface the controller will listen on for client connections.
+# This should be an IP address or hostname of the controller's host. The empty
+# string means listen on all interfaces.
+# c.FCClientServiceFactory.ip = ''
+
+# The TCP/IP port the controller will listen on for client connections. If 0
+# a random port will be used. If the controller's host has a firewall running
+# it must allow incoming traffic on this port.
+# c.FCClientServiceFactory.port = 0
+
+# The client learns how to connect to the controller by looking at the
+# location field embedded in the FURL. If this field is empty, all network
+# interfaces that the controller is listening on will be listed. To have the
+# client connect on a particular interface, list it here.
+# c.FCClientServiceFactory.location = ''
+
+# Use SSL encryption for the client connection.
+# c.FCClientServiceFactory.secure =  True
+
+# Reuse the client FURL each time the controller is started. If set, you must
+# also pick a specific network port above (FCClientServiceFactory.port).
+# c.FCClientServiceFactory.reuse_furls = False
+
+#-----------------------------------------------------------------------------
+# Configure the engine services
+#-----------------------------------------------------------------------------
+
+# Basic config attributes for the engine services.
+
+# The network interface the controller will listen on for engine connections.
+# This should be an IP address or hostname of the controller's host. The empty
+# string means listen on all interfaces.
+# c.FCEngineServiceFactory.ip = ''
+
+# The TCP/IP port the controller will listen on for engine connections. If 0
+# a random port will be used. If the controller's host has a firewall running
+# it must allow incoming traffic on this port.
+# c.FCEngineServiceFactory.port = 0
+
+# The engine learns how to connect to the controller by looking at the
+# location field embedded in the FURL. If this field is empty, all network
+# interfaces that the controller is listening on will be listed. To have the
+# client connect on a particular interface, list it here.
+# c.FCEngineServiceFactory.location = ''
+
+# Use SSL encryption for the engine connection.
+# c.FCEngineServiceFactory.secure = True
+
+# Reuse the client FURL each time the controller is started. If set, you must
+# also pick a specific network port above (FCClientServiceFactory.port).
+# c.FCEngineServiceFactory.reuse_furls = False
+
+#-----------------------------------------------------------------------------
+# Developer level configuration attributes
+#-----------------------------------------------------------------------------
+
+# You shouldn't have to modify anything in this section. These attributes
+# are more for developers who want to change the behavior of the controller
+# at a fundamental level.
+
+# c.FCClientServiceFactory.cert_file = u'ipcontroller-client.pem'
+
+# default_client_interfaces = Config()
+# default_client_interfaces.Task.interface_chain = [
+#     'IPython.kernel.task.ITaskController',
+#     'IPython.kernel.taskfc.IFCTaskController'
+# ]
+# 
+# default_client_interfaces.Task.furl_file = u'ipcontroller-tc.furl'
+# 
+# default_client_interfaces.MultiEngine.interface_chain = [
+#     'IPython.kernel.multiengine.IMultiEngine',
+#     'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+# ]
+# 
+# default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_client_interfaces
+
+# c.FCEngineServiceFactory.cert_file = u'ipcontroller-engine.pem'
+
+# default_engine_interfaces = Config()
+# default_engine_interfaces.Default.interface_chain = [
+#     'IPython.kernel.enginefc.IFCControllerBase'
+# ]
+# 
+# default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
+# 
+# c.FCEngineServiceFactory.interfaces = default_engine_interfaces

IPython/config/default/ipengine_config.py

@@ -0,0 +1,90 b''
+c = get_config()
+
+#-----------------------------------------------------------------------------
+# Global configuration
+#-----------------------------------------------------------------------------
+
+# Start up messages are logged to stdout using the logging module.
+# These all happen before the twisted reactor is started and are
+# useful for debugging purposes. Can be (10=DEBUG,20=INFO,30=WARN,40=CRITICAL) 
+# and smaller is more verbose.
+# c.Global.log_level = 20
+
+# Log to a file in cluster_dir/log, otherwise just log to sys.stdout.
+# c.Global.log_to_file = False
+
+# Remove old logs from cluster_dir/log before starting.
+# c.Global.clean_logs = True
+
+# A list of strings that will be executed in the users namespace on the engine
+# before it connects to the controller.
+# c.Global.exec_lines = ['import numpy']
+
+# The engine will try to connect to the controller multiple times, to allow
+# the controller time to startup and write its FURL file. These parameters 
+# control the number of retries (connect_max_tries) and the initial delay
+# (connect_delay) between attemps. The actual delay between attempts gets
+# longer each time by a factor of 1.5 (delay[i] = 1.5*delay[i-1])
+# those attemps.
+# c.Global.connect_delay = 0.1
+# c.Global.connect_max_tries = 15
+
+# By default, the engine will look for the controller's FURL file in its own
+# cluster directory. Sometimes, the FURL file will be elsewhere and this 
+# attribute can be set to the full path of the FURL file.
+# c.Global.furl_file = u''
+
+# The working directory for the process. The application will use os.chdir
+# to change to this directory before starting.
+# c.Global.work_dir = os.getcwd()
+
+#-----------------------------------------------------------------------------
+# MPI configuration
+#-----------------------------------------------------------------------------
+
+# Upon starting the engine can be configured to call MPI_Init. This section
+# configures that.
+
+# Select which MPI section to execute to setup MPI. The value of this 
+# attribute must match the name of another attribute in the MPI config 
+# section (mpi4py, pytrilinos, etc.). This can also be set by the --mpi
+# command line option.
+# c.MPI.use = ''
+
+# Initialize MPI using mpi4py. To use this, set c.MPI.use = 'mpi4py' to use
+# --mpi=mpi4py at the command line.
+# c.MPI.mpi4py = """from mpi4py import MPI as mpi
+# mpi.size = mpi.COMM_WORLD.Get_size()
+# mpi.rank = mpi.COMM_WORLD.Get_rank()
+# """
+
+# Initialize MPI using pytrilinos. To use this, set c.MPI.use = 'pytrilinos' 
+# to use --mpi=pytrilinos at the command line.
+# c.MPI.pytrilinos = """from PyTrilinos import Epetra
+# class SimpleStruct:
+# pass
+# mpi = SimpleStruct()
+# mpi.rank = 0
+# mpi.size = 0
+# """
+
+#-----------------------------------------------------------------------------
+# Developer level configuration attributes
+#-----------------------------------------------------------------------------
+
+# You shouldn't have to modify anything in this section. These attributes
+# are more for developers who want to change the behavior of the controller
+# at a fundamental level.
+
+# You should not have to change these attributes.
+
+# c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+# c.Global.furl_file_name = u'ipcontroller-engine.furl'
+
+
+
+
+
+
+

IPython/config/profile/ipython_config_cluster.py

@@ -0,0 +1,24 b''
+c = get_config()
+
+# This can be used at any point in a config file to load a sub config
+# and merge it into the current one.
+load_subconfig('ipython_config.py')
+
+lines = """
+from IPython.kernel.client import *
+"""
+
+# You have to make sure that attributes that are containers already
+# exist before using them.  Simple assigning a new list will override
+# all previous values.
+if hasattr(c.Global, 'exec_lines'):
+    c.Global.exec_lines.append(lines)
+else:
+    c.Global.exec_lines = [lines]
+
+# Load the parallelmagic extension to enable %result, %px, %autopx magics.
+if hasattr(c.Global, 'extensions'):
+    c.Global.extensions.append('parallelmagic')
+else:
+    c.Global.extensions = ['parallelmagic']
+

IPython/core/pylabtools.py

@@ -0,0 +1,145 b''
+# -*- coding: utf-8 -*-
+"""Pylab (matplotlib) support utilities.
+
+Authors
+-------
+Fernando Perez.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from IPython.utils.genutils import flag_calls
+
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
+
+def pylab_activate(user_ns, gui=None, import_all=True):
+    """Activate pylab mode in the user's namespace.
+
+    Loads and initializes numpy, matplotlib and friends for interactive use.
+
+    Parameters
+    ----------
+    user_ns : dict
+      Namespace where the imports will occur.
+
+    gui : optional, string
+      A valid gui name following the conventions of the %gui magic.
+
+    import_all : optional, boolean
+      If true, an 'import *' is done from numpy and pylab.
+
+    Returns
+    -------
+    The actual gui used (if not given as input, it was obtained from matplotlib
+    itself, and will be needed next to configure IPython's gui integration.
+    """
+
+    # Initialize matplotlib to interactive mode always
+    import matplotlib
+
+    # If user specifies a GUI, that dictates the backend, otherwise we read the
+    # user's mpl default from the mpl rc structure
+    g2b = {'tk': 'TkAgg',
+           'gtk': 'GTKAgg',
+           'wx': 'WXAgg',
+           'qt': 'Qt4Agg', # qt3 not supported
+           'qt4': 'Qt4Agg' }
+
+    if gui:
+        # select backend based on requested gui
+        backend = g2b[gui]
+    else:
+        backend = matplotlib.rcParams['backend']
+        # In this case, we need to find what the appropriate gui selection call
+        # should be for IPython, so we can activate inputhook accordingly
+        b2g = dict(zip(g2b.values(),g2b.keys()))
+        gui = b2g[backend]
+
+    # We must set the desired backend before importing pylab
+    matplotlib.use(backend)
+    
+    # This must be imported last in the matplotlib series, after
+    # backend/interactivity choices have been made
+    import matplotlib.pylab as pylab
+
+    # XXX For now leave this commented out, but depending on discussions with
+    # mpl-dev, we may be able to allow interactive switching...
+    #import matplotlib.pyplot
+    #matplotlib.pyplot.switch_backend(backend)
+
+    pylab.show._needmain = False
+    # We need to detect at runtime whether show() is called by the user.
+    # For this, we wrap it into a decorator which adds a 'called' flag.
+    pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)
+
+    # Import numpy as np/pyplot as plt are conventions we're trying to
+    # somewhat standardize on.  Making them available to users by default
+    # will greatly help this. 
+    exec ("import numpy\n"
+          "import matplotlib\n"
+          "from matplotlib import pylab, mlab, pyplot\n"
+          "np = numpy\n"
+          "plt = pyplot\n"
+          ) in user_ns
+
+    if import_all:
+        exec("from matplotlib.pylab import *\n"
+             "from numpy import *\n") in user_ns
+
+    matplotlib.interactive(True)
+
+    print """
+Welcome to pylab, a matplotlib-based Python environment [backend: %s].
+For more information, type 'help(pylab)'.""" % backend
+    
+    return gui
+
+# We need a little factory function here to create the closure where
+# safe_execfile can live.
+def mpl_runner(safe_execfile):
+    """Factory to return a matplotlib-enabled runner for %run.
+
+    Parameters
+    ----------
+    safe_execfile : function
+      This must be a function with the same interface as the
+      :meth:`safe_execfile` method of IPython.
+
+    Returns
+    -------
+    A function suitable for use as the ``runner`` argument of the %run magic
+    function.
+    """
+    
+    def mpl_execfile(fname,*where,**kw):
+        """matplotlib-aware wrapper around safe_execfile.
+
+        Its interface is identical to that of the :func:`execfile` builtin.
+
+        This is ultimately a call to execfile(), but wrapped in safeties to
+        properly handle interactive rendering."""
+
+        import matplotlib
+        import matplotlib.pylab as pylab
+
+        #print '*** Matplotlib runner ***' # dbg
+        # turn off rendering until end of script
+        is_interactive = matplotlib.rcParams['interactive']
+        matplotlib.interactive(False)
+        safe_execfile(fname,*where,**kw)
+        matplotlib.interactive(is_interactive)
+        # make rendering call now, if the user tried to do it
+        if pylab.draw_if_interactive.called:
+            pylab.draw()
+            pylab.draw_if_interactive.called = False
+
+    return mpl_execfile

IPython/core/tests/simpleerr.py

@@ -0,0 +1,32 b''
+"""Error script. DO NOT EDIT FURTHER! It will break exception doctests!!!"""
+import sys
+
+def div0():
+    "foo"
+    x = 1
+    y = 0
+    x/y
+
+def sysexit(stat, mode):
+    raise SystemExit(stat, 'Mode = %s' % mode)
+
+def bar(mode):
+    "bar"
+    if mode=='div':
+        div0()
+    elif mode=='exit':
+        try:
+            stat = int(sys.argv[2])
+        except:
+            stat = 1
+        sysexit(stat, mode)
+    else:
+        raise ValueError('Unknown mode')
+
+if __name__ == '__main__':
+    try:
+        mode = sys.argv[1]
+    except IndexError:
+        mode = 'div'
+
+    bar(mode)

IPython/core/tests/test_completer.py

@@ -0,0 +1,35 b''
+"""Tests for the IPython tab-completion machinery.
+"""
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import sys
+
+# third party
+import nose.tools as nt
+
+# our own packages
+from IPython.core import completer
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+def test_protect_filename():
+    pairs = [ ('abc','abc'),
+              (' abc',r'\ abc'),
+              ('a bc',r'a\ bc'),
+              ('a  bc',r'a\ \ bc'),
+              ('  bc',r'\ \ bc'),
+              ]
+    # On posix, we also protect parens
+    if sys.platform != 'win32':
+        pairs.extend( [('a(bc',r'a\(bc'),
+                       ('a)bc',r'a\)bc'),
+                       ('a( )bc',r'a\(\ \)bc'),
+                       ] )
+    # run the actual tests
+    for s1, s2 in pairs:
+        s1p = completer.protect_filename(s1)
+        nt.assert_equals(s1p, s2)

IPython/core/tests/test_prefilter.py

@@ -0,0 +1,34 b''
+"""Tests for input manipulation machinery."""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import nose.tools as nt
+
+from IPython.testing import tools as tt, decorators as dec
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
[e-mail]
+def test_prefilter():
+    """Test user input conversions"""
+
+    # pairs of (raw, expected correct) input
+    pairs = [ ('2+2','2+2'),
+              ('>>> 2+2','2+2'),
+              ('>>> # This is a comment\n'
+               '... 2+2',
+               '# This is a comment\n'
+               '2+2'),
+              # Some IPython input
+              ('In [1]: 1', '1'),
+              ('In [2]: for i in range(5):\n'
+               '   ...:     print i,',
+               'for i in range(5):\n'
+               '    print i,'),
+             ]
+
+    ip = get_ipython()
+    for raw, correct in pairs:
+        yield nt.assert_equals(ip.prefilter(raw), correct)

IPython/core/tests/test_run.py

@@ -0,0 +1,174 b''
+"""Tests for code execution (%run and related), which is particularly tricky.
+
+Because of how %run manages namespaces, and the fact that we are trying here to
+verify subtle object deletion and reference counting issues, the %run tests
+will be kept in this separate file.  This makes it easier to aggregate in one
+place the tricks needed to handle it; most other magics are much easier to test
+and we do so in a common test_magic file.
+"""
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import os
+import sys
+import tempfile
+
+# third-party
+import nose.tools as nt
+
+# our own
+from IPython.utils.platutils import find_cmd
+from IPython.utils import genutils
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
+def doctest_refbug():
+    """Very nasty problem with references held by multiple runs of a script.
+    See: https://bugs.launchpad.net/ipython/+bug/269966
+
+    In [1]: _ip.clear_main_mod_cache()
+    # random
+    
+    In [2]: %run refbug
+
+    In [3]: call_f()
+    lowercased: hello
+
+    In [4]: %run refbug
+
+    In [5]: call_f()
+    lowercased: hello
+    lowercased: hello
+    """
+
+
+def doctest_run_builtins():
+    r"""Check that %run doesn't damage __builtins__.
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: fname = tempfile.mkstemp('.py')[1]
+
+    In [3]: f = open(fname,'w')
+
+    In [4]: f.write('pass\n')
+
+    In [5]: f.flush()
+
+    In [6]: t1 = type(__builtins__)
+
+    In [7]: %run "$fname"
+
+    In [7]: f.close()
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: t2 = type(__builtins__)
+
+    In [10]: t1 == t2
+    Out[10]: True
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+
+    In [12]: try:
+       ....:     os.unlink(fname)
+       ....: except:
+       ....:     pass
+       ....: 
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TestMagicRunPass(tt.TempFileMixin):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        self.mktmp('pass\n')
+
+    def run_tmpfile(self):
+        _ip = get_ipython()
+        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
+        # See below and ticket https://bugs.launchpad.net/bugs/366353
+        _ip.magic('run "%s"' % self.fname)
+
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+        _ip = get_ipython()
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        _ip = get_ipython()
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        _ip = get_ipython()
+        p2 = str(_ip.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+
+class TestMagicRunSimple(tt.TempFileMixin):
+
+    def test_simpledef(self):
+        """Test that simple class definitions work."""
+        src = ("class foo: pass\n"
+               "def f(): return foo()")
+        self.mktmp(src)
+        _ip.magic('run %s' % self.fname)
+        _ip.runlines('t = isinstance(f(), foo)')
+        nt.assert_true(_ip.user_ns['t'])
+
+    # We have to skip these in win32 because genutils.getoutputerr() crashes,
+    # due to the fact that subprocess does not support close_fds when
+    # redirecting stdout/err.  So unless someone who knows more tells us how to
+    # implement genutils.getoutputerr() in win32, we're stuck avoiding these.
+    @dec.skip_win32
+    def test_obj_del(self):
+        """Test that object's __del__ methods are called on exit."""
+        
+        # This test is known to fail on win32.
+        # See ticket https://bugs.launchpad.net/bugs/366334
+        src = ("class A(object):\n"
+               "    def __del__(self):\n"
+               "        print 'object A deleted'\n"
+               "a = A()\n")
+        self.mktmp(src)
+        tt.ipexec_validate(self.fname, 'object A deleted')
+
+    @dec.skip_win32
+    def test_tclass(self):
+        mydir = os.path.dirname(__file__)
+        tc = os.path.join(mydir, 'tclass')
+        src = ("%%run '%s' C-first\n"
+               "%%run '%s' C-second\n") % (tc, tc)
+        self.mktmp(src, '.ipy')
+        out = """\
+ARGV 1-: ['C-first']
+ARGV 1-: ['C-second']
+tclass.py: deleting object: C-first
+"""
+        tt.ipexec_validate(self.fname, out)

IPython/extensions/parallelmagic.py

@@ -0,0 +1,209 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""Magic command interface for interactive parallel work."""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import new
+
+from IPython.core.component import Component
+from IPython.utils.traitlets import Bool, Any
+from IPython.utils.autoattr import auto_attr
+from IPython.testing import decorators as testdec
+
+#-----------------------------------------------------------------------------
+# Definitions of magic functions for use with IPython
+#-----------------------------------------------------------------------------
+
+
+NO_ACTIVE_MULTIENGINE_CLIENT = """
+Use activate() on a MultiEngineClient object to activate it for magics.
+"""
+
+
+class ParalleMagicComponent(Component):
+    """A component to manage the %result, %px and %autopx magics."""
+
+    active_multiengine_client = Any()
+    verbose = Bool(False, config=True)
+
+    def __init__(self, parent, name=None, config=None):
+        super(ParalleMagicComponent, self).__init__(parent, name=name, config=config)
+        self._define_magics()
+        # A flag showing if autopx is activated or not
+        self.autopx = False
+
+    # Access other components like this rather than by a regular attribute.
+    # This won't lookup the InteractiveShell object until it is used and
+    # then it is cached.  This is both efficient and couples this class 
+    # more loosely to InteractiveShell.
+    @auto_attr
+    def shell(self):
+        return Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell')[0]
+
+    def _define_magics(self):
+        """Define the magic functions."""
+        self.shell.define_magic('result', self.magic_result)
+        self.shell.define_magic('px', self.magic_px)
+        self.shell.define_magic('autopx', self.magic_autopx)
+
+    @testdec.skip_doctest
+    def magic_result(self, ipself, parameter_s=''):
+        """Print the result of command i on all engines..
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method.
+
+        Then you can do the following::
+
+            In [23]: %result
+            Out[23]: 
+            <Results List>
+            [0] In [6]: a = 10
+            [1] In [6]: a = 10
+    
+            In [22]: %result 6
+            Out[22]: 
+            <Results List>
+            [0] In [6]: a = 10
+            [1] In [6]: a = 10
+        """
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+
+        try:
+            index = int(parameter_s)
+        except:
+            index = None
+        result = self.active_multiengine_client.get_result(index)
+        return result
+
+    @testdec.skip_doctest
+    def magic_px(self, ipself, parameter_s=''):
+        """Executes the given python command in parallel.
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method.
+        
+        Then you can do the following::
+
+            In [24]: %px a = 5
+            Parallel execution on engines: all
+            Out[24]: 
+            <Results List>
+            [0] In [7]: a = 5
+            [1] In [7]: a = 5
+        """
+
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+        print "Parallel execution on engines: %s" % self.active_multiengine_client.targets
+        result = self.active_multiengine_client.execute(parameter_s)
+        return result
+
+    @testdec.skip_doctest
+    def magic_autopx(self, ipself, parameter_s=''):
+        """Toggles auto parallel mode.
+
+        To use this a :class:`MultiEngineClient` instance must be created 
+        and then activated by calling its :meth:`activate` method. Once this
+        is called, all commands typed at the command line are send to
+        the engines to be executed in parallel. To control which engine
+        are used, set the ``targets`` attributed of the multiengine client
+        before entering ``%autopx`` mode.
+
+        Then you can do the following::
+
+            In [25]: %autopx
+            %autopx to enabled
+
+            In [26]: a = 10
+            <Results List>
+            [0] In [8]: a = 10
+            [1] In [8]: a = 10
+
+
+            In [27]: %autopx
+            %autopx disabled
+        """
+        if self.autopx:
+            self._disable_autopx()
+        else:
+            self._enable_autopx()
+
+    def _enable_autopx(self):
+        """Enable %autopx mode by saving the original runsource and installing 
+        pxrunsource.
+        """
+        if self.active_multiengine_client is None:
+            print NO_ACTIVE_MULTIENGINE_CLIENT
+            return
+
+        self._original_runsource = self.shell.runsource
+        self.shell.runsource = new.instancemethod(
+            self.pxrunsource, self.shell, self.shell.__class__
+        )
+        self.autopx = True
+        print "%autopx enabled"
+    
+    def _disable_autopx(self):
+        """Disable %autopx by restoring the original InteractiveShell.runsource."""
+        if self.autopx:
+            self.shell.runsource = self._original_runsource
+            self.autopx = False
+            print "%autopx disabled"
+
+    def pxrunsource(self, ipself, source, filename="<input>", symbol="single"):
+        """A parallel replacement for InteractiveShell.runsource."""
+
+        try:
+            code = ipself.compile(source, filename, symbol)
+        except (OverflowError, SyntaxError, ValueError):
+            # Case 1
+            ipself.showsyntaxerror(filename)
+            return None
+
+        if code is None:
+            # Case 2
+            return True
+
+        # Case 3
+        # Because autopx is enabled, we now call executeAll or disable autopx if
+        # %autopx or autopx has been called
+        if 'get_ipython().magic("%autopx' in source or 'get_ipython().magic("autopx' in source:
+            self._disable_autopx()
+            return False
+        else:
+            try:
+                result = self.active_multiengine_client.execute(source)
+            except:
+                ipself.showtraceback()
+            else:
+                print result.__repr__()
+            return False
+
+
+_loaded = False
+
+
+def load_ipython_extension(ip):
+    """Load the extension in IPython."""
+    global _loaded
+    if not _loaded:
+        prd = ParalleMagicComponent(ip, name='parallel_magic')
+        _loaded = True
+

IPython/external/_numpy_testing_utils.py

@@ -0,0 +1,120 b''
+# IPython: modified copy of numpy.testing.utils, so numpy.testing.decorators
+# works without numpy being installed.
+"""
+Utility function to facilitate testing.
+"""
+
+import os
+import sys
+import re
+import operator
+import types
+import warnings
+
+# The following two classes are copied from python 2.6 warnings module (context
+# manager)
+class WarningMessage(object):
+
+    """
+    Holds the result of a single showwarning() call.
+
+    Notes
+    -----
+    `WarningMessage` is copied from the Python 2.6 warnings module,
+    so it can be used in NumPy with older Python versions.
+
+    """
+
+    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+                        "line")
+
+    def __init__(self, message, category, filename, lineno, file=None,
+                    line=None):
+        local_values = locals()
+        for attr in self._WARNING_DETAILS:
+            setattr(self, attr, local_values[attr])
+        if category:
+            self._category_name = category.__name__
+        else:
+            self._category_name = None
+
+    def __str__(self):
+        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
+                    "line : %r}" % (self.message, self._category_name,
+                                    self.filename, self.lineno, self.line))
+
+class WarningManager:
+    """
+    A context manager that copies and restores the warnings filter upon
+    exiting the context.
+
+    The 'record' argument specifies whether warnings should be captured by a
+    custom implementation of ``warnings.showwarning()`` and be appended to a
+    list returned by the context manager. Otherwise None is returned by the
+    context manager. The objects appended to the list are arguments whose
+    attributes mirror the arguments to ``showwarning()``.
+
+    The 'module' argument is to specify an alternative module to the module
+    named 'warnings' and imported under that name. This argument is only useful
+    when testing the warnings module itself.
+
+    Notes
+    -----
+    `WarningManager` is a copy of the ``catch_warnings`` context manager
+    from the Python 2.6 warnings module, with slight modifications.
+    It is copied so it can be used in NumPy with older Python versions.
+
+    """
+    def __init__(self, record=False, module=None):
+        self._record = record
+        if module is None:
+            self._module = sys.modules['warnings']
+        else:
+            self._module = module
+        self._entered = False
+
+    def __enter__(self):
+        if self._entered:
+            raise RuntimeError("Cannot enter %r twice" % self)
+        self._entered = True
+        self._filters = self._module.filters
+        self._module.filters = self._filters[:]
+        self._showwarning = self._module.showwarning
+        if self._record:
+            log = []
+            def showwarning(*args, **kwargs):
+                log.append(WarningMessage(*args, **kwargs))
+            self._module.showwarning = showwarning
+            return log
+        else:
+            return None
+
+    def __exit__(self):
+        if not self._entered:
+            raise RuntimeError("Cannot exit %r without entering first" % self)
+        self._module.filters = self._filters
+        self._module.showwarning = self._showwarning
+
+def assert_warns(warning_class, func, *args, **kw):
+    """Fail unless a warning of class warning_class is thrown by callable when
+    invoked with arguments args and keyword arguments kwargs.
+    
+    If a different type of warning is thrown, it will not be caught, and the
+    test case will be deemed to have suffered an error.
+    """
+
+    # XXX: once we may depend on python >= 2.6, this can be replaced by the
+    # warnings module context manager.
+    ctx = WarningManager(record=True)
+    l = ctx.__enter__()
+    warnings.simplefilter('always')
+    try:
+        func(*args, **kw)
+        if not len(l) > 0:
+            raise AssertionError("No warning raised when calling %s"
+                    % func.__name__)
+        if not l[0].category is warning_class:
+            raise AssertionError("First warning for %s is not a " \
+                    "%s( is %s)" % (func.__name__, warning_class, l[0]))
+    finally:
+        ctx.__exit__()

IPython/external/decorators.py

@@ -0,0 +1,284 b''
+"""
+Decorators for labeling and modifying behavior of test objects.
+
+Decorators that merely return a modified version of the original
+function object are straightforward. Decorators that return a new
+function object need to use
+::
+
+  nose.tools.make_decorator(original_function)(decorator)
+
+in returning the decorator, in order to preserve meta-data such as
+function name, setup and teardown functions and so on - see
+``nose.tools`` for more information.
+
+"""
+import warnings
+import sys
+
+# IPython changes: make this work if numpy not available
+# Original code:
+#from numpy.testing.utils import \
+#        WarningManager, WarningMessage
+# Our version:
+try:
+    from numpy.testing.utils import WarningManager, WarningMessage
+except ImportError:
+    from _numpy_testing_utils import WarningManager, WarningMessage
+    
+# End IPython changes
+
+def slow(t):
+    """
+    Label a test as 'slow'.
+
+    The exact definition of a slow test is obviously both subjective and
+    hardware-dependent, but in general any individual test that requires more
+    than a second or two should be labeled as slow (the whole suite consits of
+    thousands of tests, so even a second is significant).
+
+    Parameters
+    ----------
+    t : callable
+        The test to label as slow.
+
+    Returns
+    -------
+    t : callable
+        The decorated test `t`.
+
+    Examples
+    --------
+    The `numpy.testing` module includes ``import decorators as dec``.
+    A test can be decorated as slow like this::
+
+      from numpy.testing import *
+
+      @dec.slow
+      def test_big(self):
+          print 'Big, slow test'
+
+    """
+
+    t.slow = True
+    return t
+
+def setastest(tf=True):
+    """
+    Signals to nose that this function is or is not a test.
+
+    Parameters
+    ----------
+    tf : bool
+        If True, specifies that the decorated callable is a test.
+        If False, specifies that the decorated callable is not a test.
+        Default is True.
+
+    Notes
+    -----
+    This decorator can't use the nose namespace, because it can be
+    called from a non-test module. See also ``istest`` and ``nottest`` in
+    ``nose.tools``.
+
+    Examples
+    --------
+    `setastest` can be used in the following way::
+
+      from numpy.testing.decorators import setastest
+
+      @setastest(False)
+      def func_with_test_in_name(arg1, arg2):
+          pass
+
+    """
+    def set_test(t):
+        t.__test__ = tf
+        return t
+    return set_test
+
+def skipif(skip_condition, msg=None):
+    """
+    Make function raise SkipTest exception if a given condition is true.
+
+    If the condition is a callable, it is used at runtime to dynamically
+    make the decision. This is useful for tests that may require costly
+    imports, to delay the cost until the test suite is actually executed.
+
+    Parameters
+    ----------
+    skip_condition : bool or callable
+        Flag to determine whether to skip the decorated test.
+    msg : str, optional
+        Message to give on raising a SkipTest exception. Default is None.
+
+    Returns
+    -------
+    decorator : function
+        Decorator which, when applied to a function, causes SkipTest
+        to be raised when `skip_condition` is True, and the function
+        to be called normally otherwise.
+
+    Notes
+    -----
+    The decorator itself is decorated with the ``nose.tools.make_decorator``
+    function in order to transmit function name, and various other metadata.
+
+    """
+
+    def skip_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+
+        # Allow for both boolean or callable skip conditions.
+        if callable(skip_condition):
+            skip_val = lambda : skip_condition()
+        else:
+            skip_val = lambda : skip_condition
+
+        def get_msg(func,msg=None):
+            """Skip message with information about function being skipped."""
+            if msg is None: 
+                out = 'Test skipped due to test condition'
+            else: 
+                out = '\n'+msg
+
+            return "Skipping test: %s%s" % (func.__name__,out)
+
+        # We need to define *two* skippers because Python doesn't allow both
+        # return with value and yield inside the same function.
+        def skipper_func(*args, **kwargs):
+            """Skipper for normal test functions."""
+            if skip_val():
+                raise nose.SkipTest(get_msg(f,msg))
+            else:
+                return f(*args, **kwargs)
+
+        def skipper_gen(*args, **kwargs):
+            """Skipper for test generators."""
+            if skip_val():
+                raise nose.SkipTest(get_msg(f,msg))
+            else:
+                for x in f(*args, **kwargs):
+                    yield x
+
+        # Choose the right skipper to use when building the actual decorator.
+        if nose.util.isgenerator(f):
+            skipper = skipper_gen
+        else:
+            skipper = skipper_func
+            
+        return nose.tools.make_decorator(f)(skipper)
+
+    return skip_decorator
+
+
+def knownfailureif(fail_condition, msg=None):
+    """
+    Make function raise KnownFailureTest exception if given condition is true.
+
+    If the condition is a callable, it is used at runtime to dynamically
+    make the decision. This is useful for tests that may require costly
+    imports, to delay the cost until the test suite is actually executed.
+
+    Parameters
+    ----------
+    fail_condition : bool or callable
+        Flag to determine whether to mark the decorated test as a known
+        failure (if True) or not (if False).
+    msg : str, optional
+        Message to give on raising a KnownFailureTest exception.
+        Default is None.
+
+    Returns
+    -------
+    decorator : function
+        Decorator, which, when applied to a function, causes SkipTest
+        to be raised when `skip_condition` is True, and the function
+        to be called normally otherwise.
+
+    Notes
+    -----
+    The decorator itself is decorated with the ``nose.tools.make_decorator``
+    function in order to transmit function name, and various other metadata.
+
+    """
+    if msg is None:
+        msg = 'Test skipped due to known failure'
+
+    # Allow for both boolean or callable known failure conditions.
+    if callable(fail_condition):
+        fail_val = lambda : fail_condition()
+    else:
+        fail_val = lambda : fail_condition
+
+    def knownfail_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+        from noseclasses import KnownFailureTest
+        def knownfailer(*args, **kwargs):
+            if fail_val():
+                raise KnownFailureTest, msg
+            else:
+                return f(*args, **kwargs)
+        return nose.tools.make_decorator(f)(knownfailer)
+
+    return knownfail_decorator
+
+def deprecated(conditional=True):
+    """
+    Filter deprecation warnings while running the test suite.
+
+    This decorator can be used to filter DeprecationWarning's, to avoid
+    printing them during the test suite run, while checking that the test
+    actually raises a DeprecationWarning.
+
+    Parameters
+    ----------
+    conditional : bool or callable, optional
+        Flag to determine whether to mark test as deprecated or not. If the
+        condition is a callable, it is used at runtime to dynamically make the
+        decision. Default is True.
+
+    Returns
+    -------
+    decorator : function
+        The `deprecated` decorator itself.
+
+    Notes
+    -----
+    .. versionadded:: 1.4.0
+
+    """
+    def deprecate_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+        from noseclasses import KnownFailureTest
+
+        def _deprecated_imp(*args, **kwargs):
+            # Poor man's replacement for the with statement
+            ctx = WarningManager(record=True)
+            l = ctx.__enter__()
+            warnings.simplefilter('always')
+            try:
+                f(*args, **kwargs)
+                if not len(l) > 0:
+                    raise AssertionError("No warning raised when calling %s"
+                            % f.__name__)
+                if not l[0].category is DeprecationWarning:
+                    raise AssertionError("First warning for %s is not a " \
+                            "DeprecationWarning( is %s)" % (f.__name__, l[0]))
+            finally:
+                ctx.__exit__()
+
+        if callable(conditional):
+            cond = conditional()
+        else:
+            cond = conditional
+        if cond:
+            return nose.tools.make_decorator(f)(_deprecated_imp)
+        else:
+            return f
+    return deprecate_decorator

IPython/kernel/clusterdir.py

@@ -0,0 +1,450 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython cluster directory
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import os
+import shutil
+import sys
+
+from twisted.python import log
+
+from IPython.core import release
+from IPython.config.loader import PyFileConfigLoader
+from IPython.core.application import Application
+from IPython.core.component import Component
+from IPython.utils.traitlets import Unicode, Bool
+from IPython.utils import genutils
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+
+class ClusterDirError(Exception):
+    pass
+
+
+class PIDFileError(Exception):
+    pass
+
+
+class ClusterDir(Component):
+    """An object to manage the cluster directory and its resources.
+
+    The cluster directory is used by :command:`ipcontroller`, 
+    :command:`ipcontroller` and :command:`ipcontroller` to manage the
+    configuration, logging and security of these applications.
+
+    This object knows how to find, create and manage these directories. This
+    should be used by any code that want's to handle cluster directories.
+    """
+
+    security_dir_name = Unicode('security')
+    log_dir_name = Unicode('log')
+    pid_dir_name = Unicode('pid')
+    security_dir = Unicode(u'')
+    log_dir = Unicode(u'')
+    pid_dir = Unicode(u'')
+    location = Unicode(u'')
+
+    def __init__(self, location):
+        super(ClusterDir, self).__init__(None)
+        self.location = location
+
+    def _location_changed(self, name, old, new):
+        if not os.path.isdir(new):
+            os.makedirs(new)
+        self.security_dir = os.path.join(new, self.security_dir_name)
+        self.log_dir = os.path.join(new, self.log_dir_name)
+        self.pid_dir = os.path.join(new, self.pid_dir_name)
+        self.check_dirs()
+
+    def _log_dir_changed(self, name, old, new):
+        self.check_log_dir()
+
+    def check_log_dir(self):
+        if not os.path.isdir(self.log_dir):
+            os.mkdir(self.log_dir)
+
+    def _security_dir_changed(self, name, old, new):
+        self.check_security_dir()
+
+    def check_security_dir(self):
+        if not os.path.isdir(self.security_dir):
+            os.mkdir(self.security_dir, 0700)
+        os.chmod(self.security_dir, 0700)
+
+    def _pid_dir_changed(self, name, old, new):
+        self.check_pid_dir()
+
+    def check_pid_dir(self):
+        if not os.path.isdir(self.pid_dir):
+            os.mkdir(self.pid_dir, 0700)
+        os.chmod(self.pid_dir, 0700)
+
+    def check_dirs(self):
+        self.check_security_dir()
+        self.check_log_dir()
+        self.check_pid_dir()
+
+    def load_config_file(self, filename):
+        """Load a config file from the top level of the cluster dir.
+
+        Parameters
+        ----------
+        filename : unicode or str
+            The filename only of the config file that must be located in
+            the top-level of the cluster directory.
+        """
+        loader = PyFileConfigLoader(filename, self.location)
+        return loader.load_config()
+
+    def copy_config_file(self, config_file, path=None, overwrite=False):
+        """Copy a default config file into the active cluster directory.
+
+        Default configuration files are kept in :mod:`IPython.config.default`.
+        This function moves these from that location to the working cluster
+        directory.
+        """
+        if path is None:
+            import IPython.config.default
+            path = IPython.config.default.__file__.split(os.path.sep)[:-1]
+            path = os.path.sep.join(path)
+        src = os.path.join(path, config_file)
+        dst = os.path.join(self.location, config_file)
+        if not os.path.isfile(dst) or overwrite:
+            shutil.copy(src, dst)
+
+    def copy_all_config_files(self, path=None, overwrite=False):
+        """Copy all config files into the active cluster directory."""
+        for f in [u'ipcontroller_config.py', u'ipengine_config.py',
+                  u'ipcluster_config.py']:
+            self.copy_config_file(f, path=path, overwrite=overwrite)
+
+    @classmethod
+    def create_cluster_dir(csl, cluster_dir):
+        """Create a new cluster directory given a full path.
+
+        Parameters
+        ----------
+        cluster_dir : str
+            The full path to the cluster directory.  If it does exist, it will
+            be used.  If not, it will be created.
+        """
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def create_cluster_dir_by_profile(cls, path, profile=u'default'):
+        """Create a cluster dir by profile name and path.
+
+        Parameters
+        ----------
+        path : str
+            The path (directory) to put the cluster directory in.
+        profile : str
+            The name of the profile.  The name of the cluster directory will
+            be "cluster_<profile>".
+        """
+        if not os.path.isdir(path):
+            raise ClusterDirError('Directory not found: %s' % path)
+        cluster_dir = os.path.join(path, u'cluster_' + profile)
+        return ClusterDir(cluster_dir)
+
+    @classmethod
+    def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
+        """Find an existing cluster dir by profile name, return its ClusterDir.
+
+        This searches through a sequence of paths for a cluster dir.  If it
+        is not found, a :class:`ClusterDirError` exception will be raised.
+
+        The search path algorithm is:
+        1. ``os.getcwd()``
+        2. ``ipython_dir``
+        3. The directories found in the ":" separated 
+           :env:`IPCLUSTER_DIR_PATH` environment variable.
+
+        Parameters
+        ----------
+        ipython_dir : unicode or str
+            The IPython directory to use.
+        profile : unicode or str
+            The name of the profile.  The name of the cluster directory
+            will be "cluster_<profile>".
+        """
+        dirname = u'cluster_' + profile
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
+        for p in paths:
+            cluster_dir = os.path.join(p, dirname)
+            if os.path.isdir(cluster_dir):
+                return ClusterDir(cluster_dir)
+        else:
+            raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
+
+    @classmethod
+    def find_cluster_dir(cls, cluster_dir):
+        """Find/create a cluster dir and return its ClusterDir.
+
+        This will create the cluster directory if it doesn't exist.
+
+        Parameters
+        ----------
+        cluster_dir : unicode or str
+            The path of the cluster directory.  This is expanded using
+            :func:`IPython.utils.genutils.expand_path`.
+        """
+        cluster_dir = genutils.expand_path(cluster_dir)
+        if not os.path.isdir(cluster_dir):
+            raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
+        return ClusterDir(cluster_dir)
+
+
+# Default command line options for IPython cluster applications.
+cl_args = (
+    (('--ipython-dir',), dict(
+        dest='Global.ipython_dir',type=unicode,
+        help='Set to override default location of Global.ipython_dir.',
+        metavar='Global.ipython_dir') ),
+    (('-p', '--profile',), dict(
+        dest='Global.profile',type=unicode,
+        help=
+        """The string name of the profile to be used. This determines the name
+        of the cluster dir as: cluster_<profile>. The default profile is named
+        'default'.  The cluster directory is resolve this way if the
+        --cluster-dir option is not used.""",
+        metavar='Global.profile') ),
+    (('--cluster-dir',), dict(
+        dest='Global.cluster_dir',type=unicode,
+        help="""Set the cluster dir. This overrides the logic used by the
+        --profile option.""",
+        metavar='Global.cluster_dir') ),
+    (('--work-dir',), dict(
+      dest='Global.work_dir',type=unicode,
+      help='Set the working dir for the process.',
+      metavar='Global.work_dir') ),
+    (('--clean-logs',), dict(
+      dest='Global.clean_logs', action='store_true',
+      help='Delete old log flies before starting.') ),
+    (('--no-clean-logs',), dict(
+      dest='Global.clean_logs', action='store_false',
+      help="Don't Delete old log flies before starting.") ),
+    )
+
+
+class ApplicationWithClusterDir(Application):
+    """An application that puts everything into a cluster directory.
+
+    Instead of looking for things in the ipython_dir, this type of application
+    will use its own private directory called the "cluster directory"
+    for things like config files, log files, etc.
+
+    The cluster directory is resolved as follows:
+
+    * If the ``--cluster-dir`` option is given, it is used.
+    * If ``--cluster-dir`` is not given, the application directory is 
+      resolve using the profile name as ``cluster_<profile>``. The search 
+      path for this directory is then i) cwd if it is found there
+      and ii) in ipython_dir otherwise.
+
+    The config file for the application is to be put in the cluster
+    dir and named the value of the ``config_file_name`` class attribute.
+    """
+
+    auto_create_cluster_dir = True
+
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(ApplicationWithClusterDir, self).create_default_config()
+        self.default_config.Global.profile = u'default'
+        self.default_config.Global.cluster_dir = u''
+        self.default_config.Global.work_dir = os.getcwd()
+        self.default_config.Global.log_to_file = False
+        self.default_config.Global.clean_logs = False
+
+    def find_resources(self):
+        """This resolves the cluster directory.
+
+        This tries to find the cluster directory and if successful, it will
+        have done:
+        * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for 
+          the application.
+        * Sets ``self.cluster_dir`` attribute of the application and config
+          objects.
+
+        The algorithm used for this is as follows:
+        1. Try ``Global.cluster_dir``.
+        2. Try using ``Global.profile``.
+        3. If both of these fail and ``self.auto_create_cluster_dir`` is
+           ``True``, then create the new cluster dir in the IPython directory.
+        4. If all fails, then raise :class:`ClusterDirError`.
+        """
+
+        try:
+            cluster_dir = self.command_line_config.Global.cluster_dir
+        except AttributeError:
+            cluster_dir = self.default_config.Global.cluster_dir
+        cluster_dir = genutils.expand_path(cluster_dir)
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        try:
+            self.profile = self.command_line_config.Global.profile
+        except AttributeError:
+            self.profile = self.default_config.Global.profile
+        try:
+            self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                self.ipython_dir, self.profile)
+        except ClusterDirError:
+            pass
+        else:
+            self.log.info('Using existing cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+            return
+
+        if self.auto_create_cluster_dir:
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                self.ipython_dir, self.profile
+            )
+            self.log.info('Creating new cluster dir: %s' % \
+                self.cluster_dir_obj.location
+            )
+            self.finish_cluster_dir()
+        else:
+            raise ClusterDirError('Could not find a valid cluster directory.')
+
+    def finish_cluster_dir(self):
+        # Set the cluster directory
+        self.cluster_dir = self.cluster_dir_obj.location
+        
+        # These have to be set because they could be different from the one
+        # that we just computed.  Because command line has the highest
+        # priority, this will always end up in the master_config.
+        self.default_config.Global.cluster_dir = self.cluster_dir
+        self.command_line_config.Global.cluster_dir = self.cluster_dir
+
+        # Set the search path to the cluster directory
+        self.config_file_paths = (self.cluster_dir,)
+
+    def find_config_file_name(self):
+        """Find the config file name for this application."""
+        # For this type of Application it should be set as a class attribute.
+        if not hasattr(self, 'config_file_name'):
+            self.log.critical("No config filename found")
+
+    def find_config_file_paths(self):
+        # Set the search path to the cluster directory
+        self.config_file_paths = (self.cluster_dir,)
+
+    def pre_construct(self):
+        # The log and security dirs were set earlier, but here we put them
+        # into the config and log them.
+        config = self.master_config
+        sdir = self.cluster_dir_obj.security_dir
+        self.security_dir = config.Global.security_dir = sdir
+        ldir = self.cluster_dir_obj.log_dir
+        self.log_dir = config.Global.log_dir = ldir
+        pdir = self.cluster_dir_obj.pid_dir
+        self.pid_dir = config.Global.pid_dir = pdir
+        self.log.info("Cluster directory set to: %s" % self.cluster_dir)
+        config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
+        # Change to the working directory. We do this just before construct
+        # is called so all the components there have the right working dir.
+        self.to_work_dir()
+
+    def to_work_dir(self):
+        wd = self.master_config.Global.work_dir
+        if unicode(wd) != unicode(os.getcwd()):
+            os.chdir(wd)
+            self.log.info("Changing to working dir: %s" % wd)
+
+    def start_logging(self):
+        # Remove old log files
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if f.startswith(self.name + u'-') and f.endswith('.log'):
+                    os.remove(os.path.join(log_dir, f))
+        # Start logging to the new log file
+        if self.master_config.Global.log_to_file:
+            log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
+            logfile = os.path.join(self.log_dir, log_filename)
+            open_log_file = open(logfile, 'w')
+        else:
+            open_log_file = sys.stdout
+        log.startLogging(open_log_file)
+
+    def write_pid_file(self, overwrite=False):
+        """Create a .pid file in the pid_dir with my pid.
+
+        This must be called after pre_construct, which sets `self.pid_dir`.
+        This raises :exc:`PIDFileError` if the pid file exists already.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            pid = self.get_pid_from_file()
+            if not overwrite:
+                raise PIDFileError(
+                    'The pid file [%s] already exists. \nThis could mean that this '
+                    'server is already running with [pid=%s].' % (pid_file, pid)
+                )
+        with open(pid_file, 'w') as f:
+            self.log.info("Creating pid file: %s" % pid_file)
+            f.write(repr(os.getpid())+'\n')
+
+    def remove_pid_file(self):
+        """Remove the pid file.
+
+        This should be called at shutdown by registering a callback with
+        :func:`reactor.addSystemEventTrigger`. This needs to return
+        ``None``.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            try:
+                self.log.info("Removing pid file: %s" % pid_file)
+                os.remove(pid_file)
+            except:
+                self.log.warn("Error removing the pid file: %s" % pid_file)
+
+    def get_pid_from_file(self):
+        """Get the pid from the pid file.
+
+        If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
+        """
+        pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
+        if os.path.isfile(pid_file):
+            with open(pid_file, 'r') as f:
+                pid = int(f.read().strip())
+                return pid
+        else:
+            raise PIDFileError('pid file not found: %s' % pid_file)

IPython/kernel/configobjfactory.py

@@ -0,0 +1,79 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+A class for creating a Twisted service that is configured using IPython's
+configuration system.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import zope.interface as zi
+
+from IPython.core.component import Component
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+class IConfiguredObjectFactory(zi.Interface):
+    """I am a component that creates a configured object.
+
+    This class is useful if you want to configure a class that is not a 
+    subclass of :class:`IPython.core.component.Component`.
+    """
+
+    def __init__(config):
+        """Get ready to configure the object using config."""
+
+    def create():
+        """Return an instance of the configured object."""
+
+
+class ConfiguredObjectFactory(Component):
+
+    zi.implements(IConfiguredObjectFactory)
+
+    def __init__(self, config):
+        super(ConfiguredObjectFactory, self).__init__(None, config=config)
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
+
+
+class IAdaptedConfiguredObjectFactory(zi.Interface):
+    """I am a component that adapts and configures an object.
+
+    This class is useful if you have the adapt an instance and configure it.
+    """
+
+    def __init__(config, adaptee=None):
+        """Get ready to adapt adaptee and then configure it using config."""
+
+    def create():
+        """Return an instance of the adapted and configured object."""
+
+
+class AdaptedConfiguredObjectFactory(Component):
+
+    # zi.implements(IAdaptedConfiguredObjectFactory)
+
+    def __init__(self, config, adaptee):
+        # print
+        # print "config pre:", config
+        super(AdaptedConfiguredObjectFactory, self).__init__(None, config=config)
+        # print
+        # print "config post:", config
+        self.adaptee = adaptee
+
+    def create(self):
+        raise NotImplementedError('create must be implemented in a subclass')
  No newline at end of file

IPython/kernel/ipclusterapp.py

@@ -0,0 +1,460 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The ipcluster application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import logging
+import os
+import signal
+
+if os.name=='posix':
+    from twisted.scripts._twistd_unix import daemonize
+
+from IPython.core import release
+from IPython.external.argparse import ArgumentParser
+from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
+from IPython.utils.importstring import import_item
+
+from IPython.kernel.clusterdir import (
+    ApplicationWithClusterDir, ClusterDirError, PIDFileError
+)
+
+from twisted.internet import reactor, defer
+from twisted.python import log, failure
+
+
+#-----------------------------------------------------------------------------
+# The ipcluster application
+#-----------------------------------------------------------------------------
+
+
+# Exit codes for ipcluster
+
+# This will be the exit code if the ipcluster appears to be running because
+# a .pid file exists
+ALREADY_STARTED = 10
+
+# This will be the exit code if ipcluster stop is run, but there is not .pid
+# file to be found.
+ALREADY_STOPPED = 11
+
+
+class IPClusterCLLoader(ArgParseConfigLoader):
+
+    def _add_other_arguments(self):
+        # This has all the common options that all subcommands use
+        parent_parser1 = ArgumentParser(add_help=False,
+                                        argument_default=NoConfigDefault)
+        parent_parser1.add_argument('--ipython-dir', 
+            dest='Global.ipython_dir',type=unicode,
+            help='Set to override default location of Global.ipython_dir.',
+            metavar='Global.ipython_dir')
+        parent_parser1.add_argument('--log-level',
+            dest="Global.log_level",type=int,
+            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
+            metavar='Global.log_level')
+
+        # This has all the common options that other subcommands use
+        parent_parser2 = ArgumentParser(add_help=False,
+                                        argument_default=NoConfigDefault)
+        parent_parser2.add_argument('-p','--profile',
+            dest='Global.profile',type=unicode,
+            help='The string name of the profile to be used. This determines '
+            'the name of the cluster dir as: cluster_<profile>. The default profile '
+            'is named "default".  The cluster directory is resolve this way '
+            'if the --cluster-dir option is not used.',
+            metavar='Global.profile')
+        parent_parser2.add_argument('--cluster-dir',
+            dest='Global.cluster_dir',type=unicode,
+            help='Set the cluster dir. This overrides the logic used by the '
+            '--profile option.',
+            metavar='Global.cluster_dir'),
+        parent_parser2.add_argument('--work-dir',
+            dest='Global.work_dir',type=unicode,
+            help='Set the working dir for the process.',
+            metavar='Global.work_dir')
+        parent_parser2.add_argument('--log-to-file',
+            action='store_true', dest='Global.log_to_file', 
+            help='Log to a file in the log directory (default is stdout)'
+        )
+
+        subparsers = self.parser.add_subparsers(
+            dest='Global.subcommand',
+            title='ipcluster subcommands',
+            description='ipcluster has a variety of subcommands. '
+            'The general way of running ipcluster is "ipcluster <cmd> '
+            ' [options]""',
+            help='For more help, type "ipcluster <cmd> -h"')
+
+        parser_list = subparsers.add_parser(
+            'list',
+            help='List all clusters in cwd and ipython_dir.',
+            parents=[parent_parser1]
+        )
+
+        parser_create = subparsers.add_parser(
+            'create',
+            help='Create a new cluster directory.',
+            parents=[parent_parser1, parent_parser2] 
+        )
+        parser_create.add_argument(
+            '--reset-config',
+            dest='Global.reset_config', action='store_true',
+            default=NoConfigDefault,
+            help='Recopy the default config files to the cluster directory. '
+            'You will loose any modifications you have made to these files.'
+        )
+
+        parser_start = subparsers.add_parser(
+            'start',
+            help='Start a cluster.',
+            parents=[parent_parser1, parent_parser2]
+        )
+        parser_start.add_argument(
+            '-n', '--number',
+            type=int, dest='Global.n',
+            help='The number of engines to start.',
+            metavar='Global.n'
+        )
+        parser_start.add_argument('--clean-logs',
+            dest='Global.clean_logs', action='store_true',
+            help='Delete old log flies before starting.',
+        )
+        parser_start.add_argument('--no-clean-logs',
+            dest='Global.clean_logs', action='store_false',
+            help="Don't delete old log flies before starting.",
+        )
+        parser_start.add_argument('--daemon',
+            dest='Global.daemonize', action='store_true',
+            help='Daemonize the ipcluster program. This implies --log-to-file',
+        )
+        parser_start.add_argument('--no-daemon',
+            dest='Global.daemonize', action='store_false',
+            help="Dont't daemonize the ipcluster program.",
+        )
+
+        parser_start = subparsers.add_parser(
+            'stop',
+            help='Stop a cluster.',
+            parents=[parent_parser1, parent_parser2]
+        )
+        parser_start.add_argument('--signal',
+            dest='Global.signal', type=int,
+            help="The signal number to use in stopping the cluster (default=2).",
+            metavar="Global.signal",
+        )
+
+
+default_config_file_name = u'ipcluster_config.py'
+
+
+_description = """Start an IPython cluster for parallel computing.\n\n
+
+An IPython cluster consists of 1 controller and 1 or more engines.
+This command automates the startup of these processes using a wide
+range of startup methods (SSH, local processes, PBS, mpiexec,
+Windows HPC Server 2008). To start a cluster with 4 engines on your
+local host simply do "ipcluster start -n 4". For more complex usage 
+you will typically do "ipcluster create -p mycluster", then edit
+configuration files, followed by "ipcluster start -p mycluster -n 4".
+"""
+
+
+class IPClusterApp(ApplicationWithClusterDir):
+
+    name = u'ipcluster'
+    description = _description
+    config_file_name = default_config_file_name
+    default_log_level = logging.INFO
+    auto_create_cluster_dir = False
+
+    def create_default_config(self):
+        super(IPClusterApp, self).create_default_config()
+        self.default_config.Global.controller_launcher = \
+            'IPython.kernel.launcher.LocalControllerLauncher'
+        self.default_config.Global.engine_launcher = \
+            'IPython.kernel.launcher.LocalEngineSetLauncher'
+        self.default_config.Global.n = 2
+        self.default_config.Global.reset_config = False
+        self.default_config.Global.clean_logs = True
+        self.default_config.Global.signal = 2
+        self.default_config.Global.daemonize = False
+
+    def create_command_line_config(self):
+        """Create and return a command line config loader."""
+        return IPClusterCLLoader(
+            description=self.description, 
+            version=release.version
+        )
+
+    def find_resources(self):
+        subcommand = self.command_line_config.Global.subcommand
+        if subcommand=='list':
+            self.list_cluster_dirs()
+            # Exit immediately because there is nothing left to do.
+            self.exit()
+        elif subcommand=='create':
+            self.auto_create_cluster_dir = True
+            super(IPClusterApp, self).find_resources()
+        elif subcommand=='start' or subcommand=='stop':
+            self.auto_create_cluster_dir = True
+            try:
+                super(IPClusterApp, self).find_resources()
+            except ClusterDirError:
+                raise ClusterDirError(
+                    "Could not find a cluster directory. A cluster dir must "
+                    "be created before running 'ipcluster start'.  Do "
+                    "'ipcluster create -h' or 'ipcluster list -h' for more "
+                    "information about creating and listing cluster dirs."
+                )
+
+    def list_cluster_dirs(self):
+        # Find the search paths
+        cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
+        if cluster_dir_paths:
+            cluster_dir_paths = cluster_dir_paths.split(':')
+        else:
+            cluster_dir_paths = []
+        try:
+            ipython_dir = self.command_line_config.Global.ipython_dir
+        except AttributeError:
+            ipython_dir = self.default_config.Global.ipython_dir
+        paths = [os.getcwd(), ipython_dir] + \
+            cluster_dir_paths
+        paths = list(set(paths))
+
+        self.log.info('Searching for cluster dirs in paths: %r' % paths)
+        for path in paths:
+            files = os.listdir(path)
+            for f in files:
+                full_path = os.path.join(path, f)
+                if os.path.isdir(full_path) and f.startswith('cluster_'):
+                    profile = full_path.split('_')[-1]
+                    start_cmd = 'ipcluster start -p %s -n 4' % profile
+                    print start_cmd + " ==> " + full_path
+
+    def pre_construct(self):
+        # IPClusterApp.pre_construct() is where we cd to the working directory.
+        super(IPClusterApp, self).pre_construct()
+        config = self.master_config
+        try:
+            daemon = config.Global.daemonize
+            if daemon:
+                config.Global.log_to_file = True
+        except AttributeError:
+            pass
+
+    def construct(self):
+        config = self.master_config
+        subcmd = config.Global.subcommand
+        reset = config.Global.reset_config
+        if subcmd == 'list':
+            return
+        if subcmd == 'create':
+            self.log.info('Copying default config files to cluster directory '
+            '[overwrite=%r]' % (reset,))
+            self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
+        if subcmd =='start':
+            self.cluster_dir_obj.copy_all_config_files(overwrite=False)
+            self.start_logging()
+            reactor.callWhenRunning(self.start_launchers)
+
+    def start_launchers(self):
+        config = self.master_config
+
+        # Create the launchers. In both bases, we set the work_dir of
+        # the launcher to the cluster_dir. This is where the launcher's
+        # subprocesses will be launched. It is not where the controller
+        # and engine will be launched.
+        el_class = import_item(config.Global.engine_launcher)
+        self.engine_launcher = el_class(
+            work_dir=self.cluster_dir, config=config
+        )
+        cl_class = import_item(config.Global.controller_launcher)
+        self.controller_launcher = cl_class(
+            work_dir=self.cluster_dir, config=config
+        )
+
+        # Setup signals
+        signal.signal(signal.SIGINT, self.sigint_handler)
+
+        # Setup the observing of stopping. If the controller dies, shut
+        # everything down as that will be completely fatal for the engines.
+        d1 = self.controller_launcher.observe_stop()
+        d1.addCallback(self.stop_launchers)
+        # But, we don't monitor the stopping of engines. An engine dying
+        # is just fine and in principle a user could start a new engine.
+        # Also, if we did monitor engine stopping, it is difficult to
+        # know what to do when only some engines die. Currently, the
+        # observing of engine stopping is inconsistent. Some launchers
+        # might trigger on a single engine stopping, other wait until
+        # all stop.  TODO: think more about how to handle this.
+        
+        # Start the controller and engines
+        self._stopping = False  # Make sure stop_launchers is not called 2x.
+        d = self.start_controller()
+        d.addCallback(self.start_engines)
+        d.addCallback(self.startup_message)
+        # If the controller or engines fail to start, stop everything
+        d.addErrback(self.stop_launchers)
+        return d
+
+    def startup_message(self, r=None):
+        log.msg("IPython cluster: started")
+        return r
+        
+    def start_controller(self, r=None):
+        # log.msg("In start_controller")
+        config = self.master_config
+        d = self.controller_launcher.start(
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+            
+    def start_engines(self, r=None):
+        # log.msg("In start_engines")
+        config = self.master_config
+        d = self.engine_launcher.start(
+            config.Global.n,
+            cluster_dir=config.Global.cluster_dir
+        )
+        return d
+
+    def stop_controller(self, r=None):
+        # log.msg("In stop_controller")
+        if self.controller_launcher.running:
+            d = self.controller_launcher.stop()
+            d.addErrback(self.log_err)
+            return d
+        else:
+            return defer.succeed(None)
+
+    def stop_engines(self, r=None):
+        # log.msg("In stop_engines")
+        if self.engine_launcher.running:
+            d = self.engine_launcher.stop()
+            d.addErrback(self.log_err)
+            return d
+        else:
+            return defer.succeed(None)
+
+    def log_err(self, f):
+        log.msg(f.getTraceback())
+        return None
+        
+    def stop_launchers(self, r=None):
+        if not self._stopping:
+            self._stopping = True
+            if isinstance(r, failure.Failure):
+                log.msg('Unexpected error in ipcluster:')
+                log.msg(r.getTraceback())
+            log.msg("IPython cluster: stopping")
+            self.stop_engines()
+            self.stop_controller()
+            # Wait a few seconds to let things shut down.
+            reactor.callLater(4.0, reactor.stop)
+
+    def sigint_handler(self, signum, frame):
+        self.stop_launchers()
+        
+    def start_logging(self):
+        # Remove old log files of the controller and engine
+        if self.master_config.Global.clean_logs:
+            log_dir = self.master_config.Global.log_dir
+            for f in os.listdir(log_dir):
+                if f.startswith('ipengine' + '-'):
+                    if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
+                        os.remove(os.path.join(log_dir, f))
+                if f.startswith('ipcontroller' + '-'):
+                    if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
+                        os.remove(os.path.join(log_dir, f))
+        # This will remote old log files for ipcluster itself
+        super(IPClusterApp, self).start_logging()
+
+    def start_app(self):
+        """Start the application, depending on what subcommand is used."""
+        subcmd = self.master_config.Global.subcommand
+        if subcmd=='create' or subcmd=='list':
+            return
+        elif subcmd=='start':
+            self.start_app_start()
+        elif subcmd=='stop':
+            self.start_app_stop()
+
+    def start_app_start(self):
+        """Start the app for the start subcommand."""
+        config = self.master_config
+        # First see if the cluster is already running
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            pass
+        else:
+            self.log.critical(
+                'Cluster is already running with [pid=%s]. '
+                'use "ipcluster stop" to stop the cluster.' % pid
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STARTED)
+
+        # Now log and daemonize
+        self.log.info(
+            'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
+        )
+        # TODO: Get daemonize working on Windows or as a Windows Server.
+        if config.Global.daemonize:
+            if os.name=='posix':
+                daemonize()
+
+        # Now write the new pid file AFTER our new forked pid is active.
+        self.write_pid_file()
+        reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
+        reactor.run()
+
+    def start_app_stop(self):
+        """Start the app for the stop subcommand."""
+        config = self.master_config
+        try:
+            pid = self.get_pid_from_file()
+        except PIDFileError:
+            self.log.critical(
+                'Problem reading pid file, cluster is probably not running.'
+            )
+            # Here I exit with a unusual exit status that other processes
+            # can watch for to learn how I existed.
+            self.exit(ALREADY_STOPPED)
+        else:
+            if os.name=='posix':
+                sig = config.Global.signal
+                self.log.info(
+                    "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
+                )
+                os.kill(pid, sig)
+            elif os.name=='nt':
+                # As of right now, we don't support daemonize on Windows, so
+                # stop will not do anything. Minimally, it should clean up the
+                # old .pid files.
+                self.remove_pid_file()
+
+def launch_new_instance():
+    """Create and run the IPython cluster."""
+    app = IPClusterApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/kernel/ipcontrollerapp.py

@@ -0,0 +1,255 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
+
+import copy
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.config.loader import Config, NoConfigDefault
+from IPython.core import release
+from IPython.core.application import Application
+from IPython.kernel import controllerservice
+from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.fcutil import FCServiceFactory
+from IPython.utils.traitlets import Str, Instance, Unicode
+
+#-----------------------------------------------------------------------------
+# Default interfaces
+#-----------------------------------------------------------------------------
+
+# The default client interfaces for FCClientServiceFactory.interfaces
+default_client_interfaces = Config()
+default_client_interfaces.Task.interface_chain = [
+    'IPython.kernel.task.ITaskController',
+    'IPython.kernel.taskfc.IFCTaskController'
+]
+
+default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
+
+default_client_interfaces.MultiEngine.interface_chain = [
+    'IPython.kernel.multiengine.IMultiEngine',
+    'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
+]
+
+default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
+
+
+
+# The default engine interfaces for FCEngineServiceFactory.interfaces
+default_engine_interfaces = Config()
+default_engine_interfaces.Default.interface_chain = [
+    'IPython.kernel.enginefc.IFCControllerBase'
+]
+
+default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
+
+# Make this a dict we can pass to Config.__init__ for the default
+default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
+
+
+#-----------------------------------------------------------------------------
+# Service factories
+#-----------------------------------------------------------------------------
+
+
+class FCClientServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the client services."""
+
+    cert_file = Unicode(u'ipcontroller-client.pem', config=True)
+    interfaces = Instance(klass=Config, kw=default_client_interfaces,
+                          allow_none=False, config=True)
+
+
+class FCEngineServiceFactory(FCServiceFactory):
+    """A Foolscap implementation of the engine services."""
+
+    cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
+    interfaces = Instance(klass=dict, kw=default_engine_interfaces,
+                          allow_none=False, config=True)
+
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+
+cl_args = (
+    # Client config
+    (('--client-ip',), dict(
+        type=str, dest='FCClientServiceFactory.ip', 
+        help='The IP address or hostname the controller will listen on for '
+        'client connections.',
+        metavar='FCClientServiceFactory.ip')
+    ),
+    (('--client-port',), dict(
+        type=int, dest='FCClientServiceFactory.port', 
+        help='The port the controller will listen on for client connections. '
+        'The default is to use 0, which will autoselect an open port.',
+        metavar='FCClientServiceFactory.port')
+    ),
+    (('--client-location',), dict(
+        type=str, dest='FCClientServiceFactory.location',
+        help='The hostname or IP that clients should connect to. This does '
+        'not control which interface the controller listens on. Instead, this '
+        'determines the hostname/IP that is listed in the FURL, which is how '
+        'clients know where to connect. Useful if the controller is listening '
+        'on multiple interfaces.',
+        metavar='FCClientServiceFactory.location')
+    ),
+    # Engine config
+    (('--engine-ip',), dict(
+        type=str, dest='FCEngineServiceFactory.ip', 
+        help='The IP address or hostname the controller will listen on for '
+        'engine connections.',
+        metavar='FCEngineServiceFactory.ip')
+    ),
+    (('--engine-port',), dict(
+        type=int, dest='FCEngineServiceFactory.port',
+        help='The port the controller will listen on for engine connections. '
+        'The default is to use 0, which will autoselect an open port.',
+        metavar='FCEngineServiceFactory.port')
+    ),
+    (('--engine-location',), dict(
+        type=str, dest='FCEngineServiceFactory.location',
+        help='The hostname or IP that engines should connect to. This does '
+        'not control which interface the controller listens on. Instead, this '
+        'determines the hostname/IP that is listed in the FURL, which is how '
+        'engines know where to connect. Useful if the controller is listening '
+        'on multiple interfaces.',
+        metavar='FCEngineServiceFactory.location')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file',
+        help='Log to a file in the log directory (default is stdout)')
+    ),
+    (('-r','--reuse-furls'), dict(
+        action='store_true', dest='Global.reuse_furls',
+        help='Try to reuse all FURL files. If this is not set all FURL files '
+        'are deleted before the controller starts. This must be set if '
+        'specific ports are specified by --engine-port or --client-port.')
+    ),
+    (('--no-secure',), dict(
+        action='store_false', dest='Global.secure',
+        help='Turn off SSL encryption for all connections.')
+    ),
+    (('--secure',), dict(
+        action='store_true', dest='Global.secure',
+        help='Turn off SSL encryption for all connections.')
+    )
+)
+
+
+_description = """Start the IPython controller for parallel computing.
+
+The IPython controller provides a gateway between the IPython engines and
+clients. The controller needs to be started before the engines and can be
+configured using command line options or using a cluster directory. Cluster
+directories contain config, log and security files and are usually located in
+your .ipython directory and named as "cluster_<profile>". See the --profile
+and --cluster-dir options for details.
+"""
+
+default_config_file_name = u'ipcontroller_config.py'
+
+
+class IPControllerApp(ApplicationWithClusterDir):
+
+    name = u'ipcontroller'
+    description = _description
+    config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(IPControllerApp, self).create_default_config()
+        self.default_config.Global.reuse_furls = False
+        self.default_config.Global.secure = True
+        self.default_config.Global.import_statements = []
+        self.default_config.Global.clean_logs = True
+
+    def post_load_command_line_config(self):
+        # Now setup reuse_furls
+        c = self.command_line_config
+        if hasattr(c.Global, 'reuse_furls'):
+            c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
+            c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
+            del c.Global.reuse_furls
+        if hasattr(c.Global, 'secure'):
+            c.FCClientServiceFactory.secure = c.Global.secure
+            c.FCEngineServiceFactory.secure = c.Global.secure
+            del c.Global.secure
+
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+        self.start_logging()
+        self.import_statements()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        # The controller service
+        controller_service = controllerservice.ControllerService()
+        controller_service.setServiceParent(self.main_service)
+        # The client tub and all its refereceables
+        csfactory = FCClientServiceFactory(self.master_config, controller_service)
+        client_service = csfactory.create()
+        client_service.setServiceParent(self.main_service)
+        # The engine tub
+        esfactory = FCEngineServiceFactory(self.master_config, controller_service)
+        engine_service = esfactory.create()
+        engine_service.setServiceParent(self.main_service)
+
+    def import_statements(self):
+        statements = self.master_config.Global.import_statements
+        for s in statements:
+            try:
+                log.msg("Executing statement: '%s'" % s)
+                exec s in globals(), locals()
+            except:
+                log.msg("Error running statement: %s" % s)
+
+    def start_app(self):
+        # Start the controller service.
+        self.main_service.startService()
+        # Write the .pid file overwriting old ones. This allow multiple
+        # controllers to clober each other. But Windows is not cleaning
+        # these up properly.
+        self.write_pid_file(overwrite=True)
+        # Add a trigger to delete the .pid file upon shutting down.
+        reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPControllerApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()

IPython/kernel/ipengineapp.py

@@ -0,0 +1,229 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+The IPython controller application
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from twisted.application import service
+from twisted.internet import reactor
+from twisted.python import log
+
+from IPython.core.application import Application
+from IPython.kernel.clusterdir import ApplicationWithClusterDir
+from IPython.kernel.engineconnector import EngineConnector
+from IPython.kernel.engineservice import EngineService
+from IPython.kernel.fcutil import Tub
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# The main application
+#-----------------------------------------------------------------------------
+
+cl_args = (
+    # Controller config
+    (('--furl-file',), dict(
+        type=unicode, dest='Global.furl_file',
+        help='The full location of the file containing the FURL of the '
+        'controller. If this is not given, the FURL file must be in the '
+        'security directory of the cluster directory.  This location is '
+        'resolved using the --profile and --app-dir options.',
+        metavar='Global.furl_file')
+    ),
+    # MPI
+    (('--mpi',), dict(
+        type=str, dest='MPI.use',
+        help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
+        metavar='MPI.use')
+    ),
+    # Global config
+    (('--log-to-file',), dict(
+        action='store_true', dest='Global.log_to_file',
+        help='Log to a file in the log directory (default is stdout)')
+    )
+)
+
+
+mpi4py_init = """from mpi4py import MPI as mpi
+mpi.size = mpi.COMM_WORLD.Get_size()
+mpi.rank = mpi.COMM_WORLD.Get_rank()
+"""
+
+pytrilinos_init = """from PyTrilinos import Epetra
+class SimpleStruct:
+pass
+mpi = SimpleStruct()
+mpi.rank = 0
+mpi.size = 0
+"""
+
+
+default_config_file_name = u'ipengine_config.py'
+
+
+_description = """Start an IPython engine for parallel computing.\n\n
+
+IPython engines run in parallel and perform computations on behalf of a client
+and controller. A controller needs to be started before the engines. The
+engine can be configured using command line options or using a cluster
+directory. Cluster directories contain config, log and security files and are
+usually located in your .ipython directory and named as "cluster_<profile>".
+See the --profile and --cluster-dir options for details.
+"""
+
+
+class IPEngineApp(ApplicationWithClusterDir):
+
+    name = u'ipengine'
+    description = _description
+    config_file_name = default_config_file_name
+    auto_create_cluster_dir = True
+    cl_arguments = Application.cl_arguments + cl_args
+
+    def create_default_config(self):
+        super(IPEngineApp, self).create_default_config()
+
+        # The engine should not clean logs as we don't want to remove the
+        # active log files of other running engines.
+        self.default_config.Global.clean_logs = False
+
+        # Global config attributes
+        self.default_config.Global.exec_lines = []
+        self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
+
+        # Configuration related to the controller
+        # This must match the filename (path not included) that the controller
+        # used for the FURL file.
+        self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
+        # If given, this is the actual location of the controller's FURL file.
+        # If not, this is computed using the profile, app_dir and furl_file_name
+        self.default_config.Global.furl_file = u''
+
+        # The max number of connection attemps and the initial delay between
+        # those attemps.
+        self.default_config.Global.connect_delay = 0.1
+        self.default_config.Global.connect_max_tries = 15
+
+        # MPI related config attributes
+        self.default_config.MPI.use = ''
+        self.default_config.MPI.mpi4py = mpi4py_init
+        self.default_config.MPI.pytrilinos = pytrilinos_init
+
+    def post_load_command_line_config(self):
+        pass
+
+    def pre_construct(self):
+        super(IPEngineApp, self).pre_construct()
+        self.find_cont_furl_file()
+
+    def find_cont_furl_file(self):
+        """Set the furl file.
+
+        Here we don't try to actually see if it exists for is valid as that
+        is hadled by the connection logic.
+        """
+        config = self.master_config
+        # Find the actual controller FURL file
+        if not config.Global.furl_file:
+            try_this = os.path.join(
+                config.Global.cluster_dir, 
+                config.Global.security_dir,
+                config.Global.furl_file_name
+            )
+            config.Global.furl_file = try_this
+
+    def construct(self):
+        # This is the working dir by now.
+        sys.path.insert(0, '')
+
+        self.start_mpi()
+        self.start_logging()
+
+        # Create the underlying shell class and EngineService
+        shell_class = import_item(self.master_config.Global.shell_class)
+        self.engine_service = EngineService(shell_class, mpi=mpi)
+
+        self.exec_lines()
+
+        # Create the service hierarchy
+        self.main_service = service.MultiService()
+        self.engine_service.setServiceParent(self.main_service)
+        self.tub_service = Tub()
+        self.tub_service.setServiceParent(self.main_service)
+        # This needs to be called before the connection is initiated
+        self.main_service.startService()
+
+        # This initiates the connection to the controller and calls
+        # register_engine to tell the controller we are ready to do work
+        self.engine_connector = EngineConnector(self.tub_service)
+
+        log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
+
+        reactor.callWhenRunning(self.call_connect)
+
+    def call_connect(self):
+        d = self.engine_connector.connect_to_controller(
+            self.engine_service, 
+            self.master_config.Global.furl_file,
+            self.master_config.Global.connect_delay,
+            self.master_config.Global.connect_max_tries
+        )
+
+        def handle_error(f):
+            log.msg('Error connecting to controller. This usually means that '
+            'i) the controller was not started, ii) a firewall was blocking '
+            'the engine from connecting to the controller or iii) the engine '
+            ' was not pointed at the right FURL file:')
+            log.msg(f.getErrorMessage())
+            reactor.callLater(0.1, reactor.stop)
+
+        d.addErrback(handle_error)
+
+    def start_mpi(self):
+        global mpi
+        mpikey = self.master_config.MPI.use
+        mpi_import_statement = self.master_config.MPI.get(mpikey, None)
+        if mpi_import_statement is not None:
+            try:
+                self.log.info("Initializing MPI:")
+                self.log.info(mpi_import_statement)
+                exec mpi_import_statement in globals()
+            except:
+                mpi = None
+        else:
+            mpi = None
+
+    def exec_lines(self):
+        for line in self.master_config.Global.exec_lines:
+            try:
+                log.msg("Executing statement: '%s'" % line)
+                self.engine_service.execute(line)
+            except:
+                log.msg("Error executing statement: %s" % line)
+
+    def start_app(self):
+        reactor.run()
+
+
+def launch_new_instance():
+    """Create and run the IPython controller"""
+    app = IPEngineApp()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/kernel/launcher.py

@@ -0,0 +1,869 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Facilities for launching IPython processes asynchronously.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import re
+import sys
+
+from IPython.core.component import Component
+from IPython.external import Itpl
+from IPython.utils.traitlets import Str, Int, List, Unicode, Enum
+from IPython.utils.platutils import find_cmd
+from IPython.kernel.twistedutil import gatherBoth, make_deferred, sleep_deferred
+from IPython.kernel.winhpcjob import (
+    WinHPCJob, WinHPCTask,
+    IPControllerTask, IPEngineTask,
+    IPControllerJob, IPEngineSetJob
+)
+
+from twisted.internet import reactor, defer
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.protocol import ProcessProtocol
+from twisted.internet.utils import getProcessOutput
+from twisted.internet.error import ProcessDone, ProcessTerminated
+from twisted.python import log
+from twisted.python.failure import Failure
+
+#-----------------------------------------------------------------------------
+# Utilities
+#-----------------------------------------------------------------------------
+
+
+def find_controller_cmd():
+    """Find the command line ipcontroller program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipcontroller script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipcontrollerapp
+        script_location = ipcontrollerapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipcontroller']
+    return cmd
+
+
+def find_engine_cmd():
+    """Find the command line ipengine program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipengine script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipengineapp
+        script_location = ipengineapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipengine']
+    return cmd
+
+
+#-----------------------------------------------------------------------------
+# Base launchers and errors
+#-----------------------------------------------------------------------------
+
+
+class LauncherError(Exception):
+    pass
+
+
+class ProcessStateError(LauncherError):
+    pass
+
+
+class UnknownStatus(LauncherError):
+    pass
+
+
+class BaseLauncher(Component):
+    """An asbtraction for starting, stopping and signaling a process."""
+
+    # In all of the launchers, the work_dir is where child processes will be
+    # run. This will usually be the cluster_dir, but may not be. any work_dir
+    # passed into the __init__ method will override the config value.
+    # This should not be used to set the work_dir for the actual engine
+    # and controller. Instead, use their own config files or the
+    # controller_args, engine_args attributes of the launchers to add
+    # the --work-dir option.
+    work_dir = Unicode(u'')
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(BaseLauncher, self).__init__(parent, name, config)
+        self.work_dir = work_dir
+        self.state = 'before' # can be before, running, after
+        self.stop_deferreds = []
+        self.start_data = None
+        self.stop_data = None
+
+    @property
+    def args(self):
+        """A list of cmd and args that will be used to start the process.
+
+        This is what is passed to :func:`spawnProcess` and the first element
+        will be the process name.
+        """
+        return self.find_args()
+
+    def find_args(self):
+        """The ``.args`` property calls this to find the args list.
+
+        Subcommand should implement this to construct the cmd and args.
+        """
+        raise NotImplementedError('find_args must be implemented in a subclass')
+
+    @property
+    def arg_str(self):
+        """The string form of the program arguments."""
+        return ' '.join(self.args)
+
+    @property
+    def running(self):
+        """Am I running."""
+        if self.state == 'running':
+            return True
+        else:
+            return False
+
+    def start(self):
+        """Start the process.
+
+        This must return a deferred that fires with information about the
+        process starting (like a pid, job id, etc.).
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'start must be implemented in a subclass')
+            )
+        )
+
+    def stop(self):
+        """Stop the process and notify observers of stopping.
+
+        This must return a deferred that fires with information about the
+        processing stopping, like errors that occur while the process is
+        attempting to be shut down. This deferred won't fire when the process
+        actually stops. To observe the actual process stopping, see
+        :func:`observe_stop`.
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'stop must be implemented in a subclass')
+            )
+        )
+
+    def observe_stop(self):
+        """Get a deferred that will fire when the process stops.
+
+        The deferred will fire with data that contains information about
+        the exit status of the process.
+        """
+        if self.state=='after':
+            return defer.succeed(self.stop_data)
+        else:
+            d = defer.Deferred()
+            self.stop_deferreds.append(d)
+            return d
+
+    def notify_start(self, data):
+        """Call this to trigger startup actions.
+
+        This logs the process startup and sets the state to 'running'.  It is
+        a pass-through so it can be used as a callback.
+        """
+
+        log.msg('Process %r started: %r' % (self.args[0], data))
+        self.start_data = data
+        self.state = 'running'
+        return data
+
+    def notify_stop(self, data):
+        """Call this to trigger process stop actions.
+
+        This logs the process stopping and sets the state to 'after'. Call
+        this to trigger all the deferreds from :func:`observe_stop`."""
+
+        log.msg('Process %r stopped: %r' % (self.args[0], data))
+        self.stop_data = data
+        self.state = 'after'
+        for i in range(len(self.stop_deferreds)):
+            d = self.stop_deferreds.pop()
+            d.callback(data)
+        return data
+
+    def signal(self, sig):
+        """Signal the process.
+
+        Return a semi-meaningless deferred after signaling the process.
+
+        Parameters
+        ----------
+        sig : str or int
+            'KILL', 'INT', etc., or any signal number
+        """
+        return defer.fail(
+            Failure(NotImplementedError(
+                'signal must be implemented in a subclass')
+            )
+        )
+
+
+#-----------------------------------------------------------------------------
+# Local process launchers
+#-----------------------------------------------------------------------------
+
+
+class LocalProcessLauncherProtocol(ProcessProtocol):
+    """A ProcessProtocol to go with the LocalProcessLauncher."""
+
+    def __init__(self, process_launcher):
+        self.process_launcher = process_launcher
+        self.pid = None
+
+    def connectionMade(self):
+        self.pid = self.transport.pid
+        self.process_launcher.notify_start(self.transport.pid)
+
+    def processEnded(self, status):
+        value = status.value
+        if isinstance(value, ProcessDone):
+            self.process_launcher.notify_stop(
+                {'exit_code':0,
+                 'signal':None,
+                 'status':None,
+                 'pid':self.pid
+                }
+            )
+        elif isinstance(value, ProcessTerminated):
+            self.process_launcher.notify_stop(
+                {'exit_code':value.exitCode,
+                 'signal':value.signal,
+                 'status':value.status,
+                 'pid':self.pid
+                }
+            )
+        else:
+            raise UnknownStatus("Unknown exit status, this is probably a "
+                                "bug in Twisted")
+
+    def outReceived(self, data):
+        log.msg(data)
+
+    def errReceived(self, data):
+        log.err(data)
+
+
+class LocalProcessLauncher(BaseLauncher):
+    """Start and stop an external process in an asynchronous manner.
+
+    This will launch the external process with a working directory of
+    ``self.work_dir``.
+    """
+
+    # This is used to to construct self.args, which is passed to 
+    # spawnProcess.
+    cmd_and_args = List([])
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(LocalProcessLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.process_protocol = None
+        self.start_deferred = None
+
+    def find_args(self):
+        return self.cmd_and_args
+
+    def start(self):
+        if self.state == 'before':
+            self.process_protocol = LocalProcessLauncherProtocol(self)
+            self.start_deferred = defer.Deferred()
+            self.process_transport = reactor.spawnProcess(
+                self.process_protocol,
+                str(self.args[0]),  # twisted expects these to be str, not unicode
+                [str(a) for a in self.args],  # str expected, not unicode
+                env=os.environ,
+                path=self.work_dir  # start in the work_dir
+            )
+            return self.start_deferred
+        else:
+            s = 'The process was already started and has state: %r' % self.state
+            return defer.fail(ProcessStateError(s))
+
+    def notify_start(self, data):
+        super(LocalProcessLauncher, self).notify_start(data)
+        self.start_deferred.callback(data)
+
+    def stop(self):
+        return self.interrupt_then_kill()
+
+    @make_deferred
+    def signal(self, sig):
+        if self.state == 'running':
+            self.process_transport.signalProcess(sig)
+
+    @inlineCallbacks
+    def interrupt_then_kill(self, delay=2.0):
+        """Send INT, wait a delay and then send KILL."""
+        yield self.signal('INT')
+        yield sleep_deferred(delay)
+        yield self.signal('KILL')
+
+
+class LocalControllerLauncher(LocalProcessLauncher):
+    """Launch a controller as a regular external process."""
+
+    controller_cmd = List(find_controller_cmd(), config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', '40'], config=True)
+
+    def find_args(self):
+        return self.controller_cmd + self.controller_args
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting LocalControllerLauncher: %r" % self.args)
+        return super(LocalControllerLauncher, self).start()
+
+
+class LocalEngineLauncher(LocalProcessLauncher):
+    """Launch a single engine as a regular externall process."""
+
+    engine_cmd = List(find_engine_cmd(), config=True)
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+
+    def find_args(self):
+        return self.engine_cmd + self.engine_args
+
+    def start(self, cluster_dir):
+        """Start the engine by cluster_dir."""
+        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        return super(LocalEngineLauncher, self).start()
+
+
+class LocalEngineSetLauncher(BaseLauncher):
+    """Launch a set of engines as regular external processes."""
+
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(LocalEngineSetLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.launchers = []
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.cluster_dir = unicode(cluster_dir)
+        dlist = []
+        for i in range(n):
+            el = LocalEngineLauncher(self.work_dir, self)
+            # Copy the engine args over to each engine launcher.
+            import copy
+            el.engine_args = copy.deepcopy(self.engine_args)
+            d = el.start(cluster_dir)
+            if i==0:
+                log.msg("Starting LocalEngineSetLauncher: %r" % el.args)
+            self.launchers.append(el)
+            dlist.append(d)
+        # The consumeErrors here could be dangerous
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        dfinal.addCallback(self.notify_start)
+        return dfinal
+
+    def find_args(self):
+        return ['engine set']
+
+    def signal(self, sig):
+        dlist = []
+        for el in self.launchers:
+            d = el.signal(sig)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dfinal
+
+    def interrupt_then_kill(self, delay=1.0):
+        dlist = []
+        for el in self.launchers:
+            d = el.interrupt_then_kill(delay)
+            dlist.append(d)
+        dfinal = gatherBoth(dlist, consumeErrors=True)
+        return dfinal
+
+    def stop(self):
+        return self.interrupt_then_kill()
+
+    def observe_stop(self):
+        dlist = [el.observe_stop() for el in self.launchers]
+        dfinal = gatherBoth(dlist, consumeErrors=False)
+        dfinal.addCallback(self.notify_stop)
+        return dfinal
+
+
+#-----------------------------------------------------------------------------
+# MPIExec launchers
+#-----------------------------------------------------------------------------
+
+
+class MPIExecLauncher(LocalProcessLauncher):
+    """Launch an external process using mpiexec."""
+
+    # The mpiexec command to use in starting the process.
+    mpi_cmd = List(['mpiexec'], config=True)
+    # The command line arguments to pass to mpiexec.
+    mpi_args = List([], config=True)
+    # The program to start using mpiexec.
+    program = List(['date'], config=True)
+    # The command line argument to the program.
+    program_args = List([], config=True)
+    # The number of instances of the program to start.
+    n = Int(1, config=True)
+
+    def find_args(self):
+        """Build self.args using all the fields."""
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.program + self.program_args
+
+    def start(self, n):
+        """Start n instances of the program using mpiexec."""
+        self.n = n
+        return super(MPIExecLauncher, self).start()
+
+
+class MPIExecControllerLauncher(MPIExecLauncher):
+    """Launch a controller using mpiexec."""
+
+    controller_cmd = List(find_controller_cmd(), config=True)
+    # Command line arguments to ipcontroller.
+    controller_args = List(['--log-to-file','--log-level', '40'], config=True)
+    n = Int(1, config=False)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.controller_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting MPIExecControllerLauncher: %r" % self.args)
+        return super(MPIExecControllerLauncher, self).start(1)
+
+    def find_args(self):
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.controller_cmd + self.controller_args
+
+
+class MPIExecEngineSetLauncher(MPIExecLauncher):
+
+    engine_cmd = List(find_engine_cmd(), config=True)
+    # Command line arguments for ipengine.
+    engine_args = List(
+        ['--log-to-file','--log-level', '40'], config=True
+    )
+    n = Int(1, config=True)
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.engine_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        self.n = n
+        log.msg('Starting MPIExecEngineSetLauncher: %r' % self.args)
+        return super(MPIExecEngineSetLauncher, self).start(n)
+
+    def find_args(self):
+        return self.mpi_cmd + ['-n', self.n] + self.mpi_args + \
+               self.engine_cmd + self.engine_args
+
+
+#-----------------------------------------------------------------------------
+# SSH launchers
+#-----------------------------------------------------------------------------
+
+# TODO: Get SSH Launcher working again.
+
+class SSHLauncher(BaseLauncher):
+    """A minimal launcher for ssh.
+
+    To be useful this will probably have to be extended to use the ``sshx``
+    idea for environment variables.  There could be other things this needs
+    as well.
+    """
+
+    ssh_cmd = List(['ssh'], config=True)
+    ssh_args = List([], config=True)
+    program = List(['date'], config=True)
+    program_args = List([], config=True)
+    hostname = Str('', config=True)
+    user = Str('', config=True)
+    location = Str('')
+
+    def _hostname_changed(self, name, old, new):
+        self.location = '%s@%s' % (self.user, new)
+
+    def _user_changed(self, name, old, new):
+        self.location = '%s@%s' % (new, self.hostname)
+
+    def find_args(self):
+        return self.ssh_cmd + self.ssh_args + [self.location] + \
+               self.program + self.program_args
+
+    def start(self, n, hostname=None, user=None):
+        if hostname is not None:
+            self.hostname = hostname
+        if user is not None:
+            self.user = user
+        return super(SSHLauncher, self).start()
+
+
+class SSHControllerLauncher(SSHLauncher):
+    pass
+
+
+class SSHEngineSetLauncher(BaseLauncher):
+    pass
+
+
+#-----------------------------------------------------------------------------
+# Windows HPC Server 2008 scheduler launchers
+#-----------------------------------------------------------------------------
+
+
+# This is only used on Windows.
+def find_job_cmd():
+    if os.name=='nt':
+        return find_cmd('job')
+    else:
+        return 'job'
+
+
+class WindowsHPCLauncher(BaseLauncher):
+
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = Str(r'\d+', config=True)
+    # The filename of the instantiated job script.
+    job_file_name = Unicode(u'ipython_job.xml', config=True)
+    # The full path to the instantiated job script. This gets made dynamically
+    # by combining the work_dir with the job_file_name.
+    job_file = Unicode(u'')
+    # The hostname of the scheduler to submit the job to
+    scheduler = Str('', config=True)
+    job_cmd = Str(find_job_cmd(), config=True)
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(WindowsHPCLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+
+    @property
+    def job_file(self):
+        return os.path.join(self.work_dir, self.job_file_name)
+
+    def write_job_file(self, n):
+        raise NotImplementedError("Implement write_job_file in a subclass.")
+
+    def find_args(self):
+        return ['job.exe']
+        
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.search(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+
+    @inlineCallbacks
+    def start(self, n):
+        """Start n copies of the process using the Win HPC job scheduler."""
+        self.write_job_file(n)
+        args = [
+            'submit',
+            '/jobfile:%s' % self.job_file,
+            '/scheduler:%s' % self.scheduler
+        ]
+        log.msg("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+        output = yield getProcessOutput(str(self.job_cmd),
+            [str(a) for a in args],
+            env=dict((str(k),str(v)) for k,v in os.environ.items()),
+            path=self.work_dir
+        )
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        defer.returnValue(job_id)
+
+    @inlineCallbacks
+    def stop(self):
+        args = [
+            'cancel',
+            self.job_id,
+            '/scheduler:%s' % self.scheduler
+        ]
+        log.msg("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
+        try:
+            # Twisted will raise DeprecationWarnings if we try to pass unicode to this
+            output = yield getProcessOutput(str(self.job_cmd),
+                [str(a) for a in args],
+                env=dict((str(k),str(v)) for k,v in os.environ.items()),
+                path=self.work_dir
+            )
+        except:
+            output = 'The job already appears to be stoppped: %r' % self.job_id
+        self.notify_stop(output)  # Pass the output of the kill cmd
+        defer.returnValue(output)
+
+
+class WindowsHPCControllerLauncher(WindowsHPCLauncher):
+
+    job_file_name = Unicode(u'ipcontroller_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPControllerJob(self)
+
+        t = IPControllerTask(self)
+        # The tasks work directory is *not* the actual work directory of 
+        # the controller. It is used as the base path for the stdout/stderr
+        # files that the scheduler redirects to.
+        t.work_directory = self.cluster_dir
+        # Add the --cluster-dir and from self.start().
+        t.controller_args.extend(self.extra_args)
+        job.add_task(t)
+
+        log.msg("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCControllerLauncher, self).start(1)
+
+
+class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
+
+    job_file_name = Unicode(u'ipengineset_job.xml', config=True)
+    extra_args = List([], config=False)
+
+    def write_job_file(self, n):
+        job = IPEngineSetJob(self)
+
+        for i in range(n):
+            t = IPEngineTask(self)
+            # The tasks work directory is *not* the actual work directory of 
+            # the engine. It is used as the base path for the stdout/stderr
+            # files that the scheduler redirects to.
+            t.work_directory = self.cluster_dir
+            # Add the --cluster-dir and from self.start().
+            t.engine_args.extend(self.extra_args)
+            job.add_task(t)
+
+        log.msg("Writing job description file: %s" % self.job_file)
+        job.write(self.job_file)
+
+    @property
+    def job_file(self):
+        return os.path.join(self.cluster_dir, self.job_file_name)
+
+    def start(self, n, cluster_dir):
+        """Start the controller by cluster_dir."""
+        self.extra_args = ['--cluster-dir', cluster_dir]
+        self.cluster_dir = unicode(cluster_dir)
+        return super(WindowsHPCEngineSetLauncher, self).start(n)
+
+
+#-----------------------------------------------------------------------------
+# Batch (PBS) system launchers
+#-----------------------------------------------------------------------------
+
+# TODO: Get PBS launcher working again.
+
+class BatchSystemLauncher(BaseLauncher):
+    """Launch an external process using a batch system.
+
+    This class is designed to work with UNIX batch systems like PBS, LSF,
+    GridEngine, etc.  The overall model is that there are different commands
+    like qsub, qdel, etc. that handle the starting and stopping of the process.
+
+    This class also has the notion of a batch script. The ``batch_template``
+    attribute can be set to a string that is a template for the batch script.
+    This template is instantiated using Itpl. Thus the template can use
+    ${n} fot the number of instances. Subclasses can add additional variables
+    to the template dict.
+    """
+
+    # Subclasses must fill these in.  See PBSEngineSet
+    # The name of the command line program used to submit jobs.
+    submit_command = Str('', config=True)
+    # The name of the command line program used to delete jobs.
+    delete_command = Str('', config=True)
+    # A regular expression used to get the job id from the output of the 
+    # submit_command.
+    job_id_regexp = Str('', config=True)
+    # The string that is the batch script template itself.
+    batch_template = Str('', config=True)
+    # The filename of the instantiated batch script.
+    batch_file_name = Unicode(u'batch_script', config=True)
+    # The full path to the instantiated batch script.
+    batch_file = Unicode(u'')
+
+    def __init__(self, work_dir, parent=None, name=None, config=None):
+        super(BatchSystemLauncher, self).__init__(
+            work_dir, parent, name, config
+        )
+        self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
+        self.context = {}
+
+    def parse_job_id(self, output):
+        """Take the output of the submit command and return the job id."""
+        m = re.match(self.job_id_regexp, output)
+        if m is not None:
+            job_id = m.group()
+        else:
+            raise LauncherError("Job id couldn't be determined: %s" % output)
+        self.job_id = job_id
+        log.msg('Job started with job id: %r' % job_id)
+        return job_id
+
+    def write_batch_script(self, n):
+        """Instantiate and write the batch script to the work_dir."""
+        self.context['n'] = n
+        script_as_string = Itpl.itplns(self.batch_template, self.context)
+        log.msg('Writing instantiated batch script: %s' % self.batch_file)
+        f = open(self.batch_file, 'w')
+        f.write(script_as_string)
+        f.close()
+
+    @inlineCallbacks
+    def start(self, n):
+        """Start n copies of the process using a batch system."""
+        self.write_batch_script(n)
+        output = yield getProcessOutput(self.submit_command,
+            [self.batch_file], env=os.environ)
+        job_id = self.parse_job_id(output)
+        self.notify_start(job_id)
+        defer.returnValue(job_id)
+
+    @inlineCallbacks
+    def stop(self):
+        output = yield getProcessOutput(self.delete_command,
+            [self.job_id], env=os.environ
+        )
+        self.notify_stop(output)  # Pass the output of the kill cmd
+        defer.returnValue(output)
+
+
+class PBSLauncher(BatchSystemLauncher):
+    """A BatchSystemLauncher subclass for PBS."""
+
+    submit_command = Str('qsub', config=True)
+    delete_command = Str('qdel', config=True)
+    job_id_regexp = Str(r'\d+', config=True)
+    batch_template = Str('', config=True)
+    batch_file_name = Unicode(u'pbs_batch_script', config=True)
+    batch_file = Unicode(u'')
+
+
+class PBSControllerLauncher(PBSLauncher):
+    """Launch a controller using PBS."""
+
+    batch_file_name = Unicode(u'pbs_batch_script_controller', config=True)
+
+    def start(self, cluster_dir):
+        """Start the controller by profile or cluster_dir."""
+        # Here we save profile and cluster_dir in the context so they
+        # can be used in the batch script template as ${profile} and
+        # ${cluster_dir}
+        self.context['cluster_dir'] = cluster_dir
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg("Starting PBSControllerLauncher: %r" % self.args)
+        return super(PBSControllerLauncher, self).start(1)
+
+
+class PBSEngineSetLauncher(PBSLauncher):
+
+    batch_file_name = Unicode(u'pbs_batch_script_engines', config=True)
+
+    def start(self, n, cluster_dir):
+        """Start n engines by profile or cluster_dir."""
+        self.program_args.extend(['--cluster-dir', cluster_dir])
+        self.cluster_dir = unicode(cluster_dir)
+        log.msg('Starting PBSEngineSetLauncher: %r' % self.args)
+        return super(PBSEngineSetLauncher, self).start(n)
+
+
+#-----------------------------------------------------------------------------
+# A launcher for ipcluster itself!
+#-----------------------------------------------------------------------------
+
+
+def find_ipcluster_cmd():
+    """Find the command line ipcluster program in a cross platform way."""
+    if sys.platform == 'win32':
+        # This logic is needed because the ipcluster script doesn't
+        # always get installed in the same way or in the same location.
+        from IPython.kernel import ipclusterapp
+        script_location = ipclusterapp.__file__.replace('.pyc', '.py')
+        # The -u option here turns on unbuffered output, which is required
+        # on Win32 to prevent wierd conflict and problems with Twisted.
+        # Also, use sys.executable to make sure we are picking up the 
+        # right python exe.
+        cmd = [sys.executable, '-u', script_location]
+    else:
+        # ipcontroller has to be on the PATH in this case.
+        cmd = ['ipcluster']
+    return cmd
+
+
+class IPClusterLauncher(LocalProcessLauncher):
+    """Launch the ipcluster program in an external process."""
+
+    ipcluster_cmd = List(find_ipcluster_cmd(), config=True)
+    # Command line arguments to pass to ipcluster.
+    ipcluster_args = List(
+        ['--clean-logs', '--log-to-file', '--log-level', '40'], config=True)
+    ipcluster_subcommand = Str('start')
+    ipcluster_n = Int(2)
+
+    def find_args(self):
+        return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
+            ['-n', repr(self.ipcluster_n)] + self.ipcluster_args
+
+    def start(self):
+        log.msg("Starting ipcluster: %r" % self.args)
+        return super(IPClusterLauncher, self).start()
+

IPython/__init__.py

@@ -16,18 +16,17 b' IPython is a set of tools for interactive and exploratory computing in Python.'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import absolute_import
 
 import os
 import sys
-from IPython.core import release
 
 #-----------------------------------------------------------------------------
 # Setup everything
 #-----------------------------------------------------------------------------
 
-
-if sys.version[0:3] < '2.4':
-    raise ImportError('Python Version 2.4 or above is required for IPython.')
+if sys.version[0:3] < '2.5':
+    raise ImportError('Python Version 2.5 or above is required for IPython.')
 
 
 # Make it easy to import extensions - they are always directly on pythonpath.
@@ -39,11 +38,16 b' sys.path.append(os.path.join(os.path.dirname(__file__), "extensions"))'
 #-----------------------------------------------------------------------------
 
 # In some cases, these are causing circular imports.
-from IPython.core.iplib import InteractiveShell
-from IPython.core.embed import embed
-from IPython.core.error import TryNext
+from .config.loader import Config
+from .core import release
+from .core.application import Application
+from .core.ipapp import IPythonApp
+from .core.embed import embed
+from .core.error import TryNext
+from .core.iplib import InteractiveShell
+from .testing import test
 
-from IPython.lib import (
+from .lib import (
     enable_wx, disable_wx,
     enable_gtk, disable_gtk,
     enable_qt4, disable_qt4,
@@ -61,4 +65,3 b' for author, email in release.authors.values():'
 __license__  = release.license
 __version__  = release.version
 __revision__ = release.revision
-

IPython/config/default/ipython_config.py

@@ -13,7 +13,7 b' c = get_config()'
 
 # Set this to determine the detail of what is logged at startup.
 # The default is 30 and possible values are 0,10,20,30,40,50.
-c.Global.log_level = 20
+# c.Global.log_level = 20
 
 # This should be a list of importable Python modules that have an
 # load_in_ipython(ip) method.  This method gets called when the extension
@@ -35,7 +35,7 b' c.Global.log_level = 20'
 # These files are run in IPython in the user's namespace.  Files with a .py
 # extension need to be pure Python.  Files with a .ipy extension can have
 # custom IPython syntax (like magics, etc.).  
-# These files need to be in the cwd, the ipythondir or be absolute paths.
+# These files need to be in the cwd, the ipython_dir or be absolute paths.
 # c.Global.exec_files = [
 #     'mycode.py',
 #     'fancy.ipy'
@@ -71,9 +71,9 b' c.Global.log_level = 20'
 
 # c.InteractiveShell.logstart = True
 
-# c.InteractiveShell.logfile = 'ipython_log.py'
+# c.InteractiveShell.logfile = u'ipython_log.py'
 
-# c.InteractiveShell.logappend = 'mylog.py'
+# c.InteractiveShell.logappend = u'mylog.py'
 
 # c.InteractiveShell.object_info_string_level = 0
 

IPython/config/loader.py

@@ -1,10 +1,10 b''
-#!/usr/bin/env python
-# encoding: utf-8
+# coding: utf-8
 """A simple configuration system.
 
-Authors:
-
+Authors
+-------
 * Brian Granger
+* Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
@@ -37,6 +37,24 b' class ConfigError(Exception):'
 class ConfigLoaderError(ConfigError):
     pass
 
+#-----------------------------------------------------------------------------
+# Argparse fix
+#-----------------------------------------------------------------------------
+# Unfortunately argparse by default prints help messages to stderr instead of
+# stdout.  This makes it annoying to capture long help screens at the command
+# line, since one must know how to pipe stderr, which many users don't know how
+# to do.  So we override the print_help method with one that defaults to
+# stdout and use our class instead.
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Simple argparse subclass that prints help to stdout by default."""
+    
+    def print_help(self, file=None):
+        if file is None:
+            file = sys.stdout
+        return super(ArgumentParser, self).print_help(file)
+    
+    print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
 
 #-----------------------------------------------------------------------------
 # Config class for holding config information
@@ -244,7 +262,13 b' class PyFileConfigLoader(FileConfigLoader):'
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
+            try:
             sub_config = loader.load_config()
+            except IOError:
+                # Pass silently if the sub config is not there. This happens
+                # when a user us using a profile, but not the default config.
+                pass
+            else:
             self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
@@ -268,26 +292,55 b' class CommandLineConfigLoader(ConfigLoader):'
     """
 
 
-class NoConfigDefault(object): pass
-NoConfigDefault = NoConfigDefault()
+class __NoConfigDefault(object): pass
+NoConfigDefault = __NoConfigDefault()
 
-class ArgParseConfigLoader(CommandLineConfigLoader):
     
-    # arguments = [(('-f','--file'),dict(type=str,dest='file'))]
-    arguments = ()
+class ArgParseConfigLoader(CommandLineConfigLoader):
+    #: Global default for arguments (see argparse docs for details)
+    argument_default = NoConfigDefault
 
-    def __init__(self, *args, **kw):
+    def __init__(self, argv=None, arguments=(), *args, **kw):
         """Create a config loader for use with argparse.
 
-        The args and kwargs arguments here are passed onto the constructor
-        of :class:`argparse.ArgumentParser`.
+        With the exception of ``argv`` and ``arguments``, other args and kwargs
+        arguments here are passed onto the constructor of
+        :class:`argparse.ArgumentParser`.
+
+        Parameters
+        ----------
+
+        argv : optional, list
+          If given, used to read command-line arguments from, otherwise
+          sys.argv[1:] is used.
+
+        arguments : optional, tuple
+          Description of valid command-line arguments, to be called in sequence
+          with parser.add_argument() to configure the parser.
         """
         super(CommandLineConfigLoader, self).__init__()
+        if argv == None:
+            argv = sys.argv[1:]
+        self.argv = argv
+        self.arguments = arguments
         self.args = args
-        self.kw = kw
+        kwargs = dict(argument_default=self.argument_default)
+        kwargs.update(kw)
+        self.kw = kwargs
 
     def load_config(self, args=None):
-        """Parse command line arguments and return as a Struct."""
+        """Parse command line arguments and return as a Struct.
+
+        Parameters
+        ----------
+
+        args : optional, list
+          If given, a list with the structure of sys.argv[1:] to parse arguments
+          from.  If not given, the instance's self.argv attribute (given at
+          construction time) is used."""
+        
+        if args is None:
+            args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
@@ -300,24 +353,20 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
             return []
 
     def _create_parser(self):
-        self.parser = argparse.ArgumentParser(*self.args, **self.kw)
+        self.parser = ArgumentParser(*self.args, **self.kw)
         self._add_arguments()
         self._add_other_arguments()
 
-    def _add_other_arguments(self):
-        pass
-
     def _add_arguments(self):
         for argument in self.arguments:
-            if not argument[1].has_key('default'):
-                argument[1]['default'] = NoConfigDefault
             self.parser.add_argument(*argument[0],**argument[1])
 
-    def _parse_args(self, args=None):
+    def _add_other_arguments(self):
+        """Meant for subclasses to add their own arguments."""
+        pass
+
+    def _parse_args(self, args):
         """self.parser->self.parsed_data"""
-        if args is None:
-            self.parsed_data, self.extra_args = self.parser.parse_known_args()
-        else:
             self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
@@ -326,4 +375,3 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
             if v is not NoConfigDefault:
                 exec_str = 'self.config.' + k + '= v'
                 exec exec_str in locals(), globals()
-

IPython/config/tests/test_loader.py

@@ -37,17 +37,18 b' from IPython.config.loader import ('
 
 
 pyfile = """
-a = 10
-b = 20
-Foo.Bar.value = 10
-Foo.Bam.value = range(10)
-D.C.value = 'hi there'
+c = get_config()
+c.a = 10
+c.b = 20
+c.Foo.Bar.value = 10
+c.Foo.Bam.value = range(10)
+c.D.C.value = 'hi there'
 """
 
 class TestPyFileCL(TestCase):
 
     def test_basic(self):
-        fd, fname = mkstemp()
+        fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
@@ -65,15 +66,13 b' class TestArgParseCL(TestCase):'
 
     def test_basic(self):
 
-        class MyLoader(ArgParseConfigLoader):
             arguments = (
                 (('-f','--foo'), dict(dest='Global.foo', type=str)),
                 (('-b',), dict(dest='MyClass.bar', type=int)),
                 (('-n',), dict(dest='n', action='store_true')),
                 (('Global.bam',), dict(type=str))
             )
-
-        cl = MyLoader()
+        cl = ArgParseConfigLoader(arguments=arguments)
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEquals(config.Global.foo, 'hi')
         self.assertEquals(config.MyClass.bar, 10)

IPython/core/application.py

@@ -1,7 +1,12 b''
-#!/usr/bin/env python
 # encoding: utf-8
 """
-An application for IPython
+An application for IPython.
+
+All top-level applications should use the classes in this module for
+handling configuration and creating componenets.
+
+The job of an :class:`Application` is to create the master configuration 
+object and then create the components, passing the config to them.
 
 Authors:
 
@@ -26,62 +31,131 b' Notes'
 import logging
 import os
 import sys
-import traceback
-from copy import deepcopy
 
-from IPython.utils.genutils import get_ipython_dir, filefind
+from IPython.core import release, crashhandler
+from IPython.utils.genutils import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
-    NoConfigDefault
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-
-class IPythonArgParseConfigLoader(ArgParseConfigLoader):
-    """Default command line options for IPython based applications."""
-
-    def _add_other_arguments(self):
-        self.parser.add_argument('-ipythondir',dest='Global.ipythondir',type=str,
-            help='Set to override default location of Global.ipythondir.',
-            default=NoConfigDefault,
-            metavar='Global.ipythondir')
-        self.parser.add_argument('-p','-profile',dest='Global.profile',type=str,
-            help='The string name of the ipython profile to be used.',
-            default=NoConfigDefault,
-            metavar='Global.profile')
-        self.parser.add_argument('-log_level',dest="Global.log_level",type=int,
-            help='Set the log level (0,10,20,30,40,50).  Default is 30.',
-            default=NoConfigDefault)
-        self.parser.add_argument('-config_file',dest='Global.config_file',type=str,
-            help='Set the config file name to override default.',
-            default=NoConfigDefault,
-            metavar='Global.config_file')
-
-
 class ApplicationError(Exception):
     pass
 
 
+app_cl_args = (
+    (('--ipython-dir', ), dict(
+        dest='Global.ipython_dir',type=unicode,
+        help=
+        """Set to override default location of the IPython directory
+        IPYTHON_DIR, stored as Global.ipython_dir.  This can also be specified
+        through the environment variable IPYTHON_DIR.""",
+        metavar='Global.ipython_dir') ),
+    (('-p', '--profile',), dict(
+        dest='Global.profile',type=unicode,
+        help=
+        """The string name of the ipython profile to be used.  Assume that your
+        config file is ipython_config-<name>.py (looks in current dir first,
+        then in IPYTHON_DIR). This is a quick way to keep and load multiple
+        config files for different tasks, especially if include your basic one
+        in your more specialized ones.  You can keep a basic
+        IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
+        include this one and load extra things for particular tasks.""",
+        metavar='Global.profile') ),
+    (('--log-level',), dict(
+        dest="Global.log_level",type=int,
+        help='Set the log level (0,10,20,30,40,50).  Default is 30.',
+        metavar='Global.log_level')),
+    (('--config-file',), dict(
+        dest='Global.config_file',type=unicode,
+        help=
+        """Set the config file name to override default.  Normally IPython
+        loads ipython_config.py (from current directory) or
+        IPYTHON_DIR/ipython_config.py.  If the loading of your config file
+        fails, IPython starts with a bare bones configuration (no modules
+        loaded at all).""",
+        metavar='Global.config_file')),
+    )
+
 class Application(object):
-    """Load a config, construct an app and run it.
+    """Load a config, construct components and set them running.
+
+    The configuration of an application can be done via four different Config
+    objects, which are loaded and ultimately merged into a single one used from
+    that point on by the app.  These are:
+
+    1. default_config: internal defaults, implemented in code.
+    2. file_config: read from the filesystem.
+    3. command_line_config: read from the system's command line flags.
+    4. constructor_config: passed parametrically to the constructor.
+
+    During initialization, 3 is actually read before 2, since at the
+    command-line one may override the location of the file to be read.  But the
+    above is the order in which the merge is made.
+
+    There is a final config object can be created and passed to the
+    constructor: override_config.  If it exists, this completely overrides the
+    configs 2-4 above (the default is still used to ensure that all needed
+    fields at least are created).  This makes it easier to create
+    parametrically (e.g. in testing or sphinx plugins) objects with a known
+    configuration, that are unaffected by whatever arguments may be present in
+    sys.argv or files in the user's various directories.
     """
 
-    config_file_name = 'ipython_config.py'
-    name = 'ipython'
-
-    def __init__(self):
+    name = u'ipython'
+    description = 'IPython: an enhanced interactive Python shell.'
+    #: usage message printed by argparse. If None, auto-generate
+    usage = None
+    config_file_name = u'ipython_config.py'
+    #: Track the default and actual separately because some messages are
+    #: only printed if we aren't using the default.
+    default_config_file_name = config_file_name
+    default_log_level = logging.WARN
+    #: Set by --profile option
+    profile_name = None
+    #: User's ipython directory, typically ~/.ipython/
+    ipython_dir = None
+    #: internal defaults, implemented in code.
+    default_config = None
+    #: read from the filesystem
+    file_config = None
+    #: read from the system's command line flags
+    command_line_config = None
+    #: passed parametrically to the constructor.
+    constructor_config = None
+    #: final override, if given supercedes file/command/constructor configs
+    override_config = None
+    #: A reference to the argv to be used (typically ends up being sys.argv[1:])
+    argv = None
+    #: Default command line arguments.  Subclasses should create a new tuple
+    #: that *includes* these.
+    cl_arguments = app_cl_args
+
+    #: extra arguments computed by the command-line loader
+    extra_args = None
+
+    # Private attributes
+    _exiting = False
+    _initialized = False
+
+    # Class choices for things that will be instantiated at runtime.
+    _CrashHandler = crashhandler.CrashHandler
+
+    def __init__(self, argv=None, constructor_config=None, override_config=None):
+        self.argv = sys.argv[1:] if argv is None else argv
+        self.constructor_config = constructor_config
+        self.override_config = override_config
         self.init_logger()
-        self.default_config_file_name = self.config_file_name
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
-        self.log.setLevel(logging.WARN)
+        self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
@@ -95,28 +169,80 b' class Application(object):'
 
     log_level = property(_get_log_level, _set_log_level)
 
+    def initialize(self):
+        """Initialize the application.
+
+        Loads all configuration information and sets all application state, but
+        does not start any relevant processing (typically some kind of event
+        loop).
+
+        Once this method has been called, the application is flagged as
+        initialized and the method becomes a no-op."""
+        
+        if self._initialized:
+            return
+
+        # The first part is protected with an 'attempt' wrapper, that will log
+        # failures with the basic system traceback machinery.  Once our crash
+        # handler is in place, we can let any subsequent exception propagate,
+        # as our handler will log it with much better detail than the default.
+        self.attempt(self.create_crash_handler)
+
+        # Configuration phase
+        # Default config (internally hardwired in application code)
+        self.create_default_config()
+        self.log_default_config()
+        self.set_default_config_log_level()
+
+        if self.override_config is None:
+            # Command-line config
+            self.pre_load_command_line_config()
+            self.load_command_line_config()
+            self.set_command_line_config_log_level()
+            self.post_load_command_line_config()
+            self.log_command_line_config()
+
+        # Find resources needed for filesystem access, using information from
+        # the above two
+        self.find_ipython_dir()
+        self.find_resources()
+        self.find_config_file_name()
+        self.find_config_file_paths()
+
+        if self.override_config is None:
+            # File-based config
+            self.pre_load_file_config()
+            self.load_file_config()
+            self.set_file_config_log_level()
+            self.post_load_file_config()
+            self.log_file_config()
+
+        # Merge all config objects into a single one the app can then use
+        self.merge_configs()
+        self.log_master_config()
+
+        # Construction phase
+        self.pre_construct()
+        self.construct()
+        self.post_construct()
+
+        # Done, flag as such and
+        self._initialized = True
+
     def start(self):
         """Start the application."""
-        self.attempt(self.create_default_config)
-        self.attempt(self.pre_load_command_line_config)
-        self.attempt(self.load_command_line_config, action='abort')
-        self.attempt(self.post_load_command_line_config)
-        self.attempt(self.find_ipythondir)
-        self.attempt(self.find_config_file_name)
-        self.attempt(self.find_config_file_paths)
-        self.attempt(self.pre_load_file_config)
-        self.attempt(self.load_file_config)
-        self.attempt(self.post_load_file_config)
-        self.attempt(self.merge_configs)
-        self.attempt(self.pre_construct)
-        self.attempt(self.construct)
-        self.attempt(self.post_construct)
-        self.attempt(self.start_app)
+        self.initialize()
+        self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
+    def create_crash_handler(self):
+        """Create a crash handler, typically setting sys.excepthook to it."""
+        self.crash_handler = self._CrashHandler(self, self.name)
+        sys.excepthook = self.crash_handler
+
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
@@ -126,66 +252,91 b' class Application(object):'
         we set them here.  The Global section is for variables like this that
         don't belong to a particular component.
         """
-        self.default_config = Config()
-        self.default_config.Global.ipythondir = get_ipython_dir()
+        c = Config()
+        c.Global.ipython_dir = get_ipython_dir()
+        c.Global.log_level = self.log_level
+        self.default_config = c
+
+    def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
+    def set_default_config_log_level(self):
+        try:
+            self.log_level = self.default_config.Global.log_level
+        except AttributeError:
+            # Fallback to the default_log_level class attribute
+            pass
+
     def create_command_line_config(self):
         """Create and return a command line config loader."""
-        return IPythonArgParseConfigLoader(description=self.name)
+        return ArgParseConfigLoader(self.argv, self.cl_arguments,
+                                    description=self.description, 
+                                    version=release.version,
+                                    usage=self.usage,
+                                    )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
-        """Load the command line config.
-
-        This method also sets ``self.debug``.
-        """
-
+        """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
+    def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
-            pass # Use existing value which is set in Application.init_logger.
-        self.log.debug("Command line config loaded:")
-        self.log.debug(repr(self.command_line_config))
+            pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
-    def find_ipythondir(self):
+    def log_command_line_config(self):
+        self.log.debug("Command line config loaded:")
+        self.log.debug(repr(self.command_line_config))
+
+    def find_ipython_dir(self):
         """Set the IPython directory.
 
-        This sets ``self.ipythondir``, but the actual value that is passed
-        to the application is kept in either ``self.default_config`` or
-        ``self.command_line_config``.  This also added ``self.ipythondir`` to
-        ``sys.path`` so config files there can be references by other config
+        This sets ``self.ipython_dir``, but the actual value that is passed to
+        the application is kept in either ``self.default_config`` or
+        ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
+        ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
-            self.ipythondir = self.command_line_config.Global.ipythondir
+            self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
-            self.ipythondir = self.default_config.Global.ipythondir
-        sys.path.append(os.path.abspath(self.ipythondir))
-        if not os.path.isdir(self.ipythondir):
-            os.makedirs(self.ipythondir, mode = 0777)
-        self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
+            self.ipython_dir = self.default_config.Global.ipython_dir
+        sys.path.append(os.path.abspath(self.ipython_dir))
+        if not os.path.isdir(self.ipython_dir):
+            os.makedirs(self.ipython_dir, mode=0777)
+        self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
+
+    def find_resources(self):
+        """Find other resources that need to be in place.
+
+        Things like cluster directories need to be in place to find the
+        config file.  These happen right after the IPython directory has
+        been set.
+        """
+        pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
-        If a profile has been set at the command line, this will resolve
-        it.  The search paths for the config file are set in
-        :meth:`find_config_file_paths` and then passed to the config file
-        loader where they are resolved to an absolute path.
+        This must set ``self.config_file_name`` to the filename of the
+        config file to use (just the filename). The search paths for the
+        config file are set in :meth:`find_config_file_paths` and then passed
+        to the config file loader where they are resolved to an absolute path.
+
+        If a profile has been set at the command line, this will resolve it.
         """
 
         try:
@@ -195,15 +346,24 b' class Application(object):'
 
         try:
             self.profile_name = self.command_line_config.Global.profile
-            name_parts = self.config_file_name.split('.')
-            name_parts.insert(1, '_' + self.profile_name + '.')
-            self.config_file_name = ''.join(name_parts)
         except AttributeError:
             pass
+        else:
+            name_parts = self.config_file_name.split('.')
+            name_parts.insert(1, u'_' + self.profile_name + u'.')
+            self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
-        """Set the search paths for resolving the config file."""
-        self.config_file_paths = (os.getcwd(), self.ipythondir)
+        """Set the search paths for resolving the config file.
+
+        This must set ``self.config_file_paths`` to a sequence of search
+        paths to pass to the config file loader.
+        """
+        # Include our own profiles directory last, so that users can still find
+        # our shipped copies of builtin profiles even if they don't have them
+        # in their local ipython directory.
+        prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
+        self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
@@ -216,7 +376,8 b' class Application(object):'
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
         """
-        self.log.debug("Attempting to load config file: <%s>" % self.config_file_name)
+        self.log.debug("Attempting to load config file: %s" %
+                       self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
@@ -225,19 +386,18 b' class Application(object):'
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
-                self.log.warn("Config file not found, skipping: <%s>" % \
+                self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
-            self.log.warn("Error loading config file: <%s>" % \
+            self.log.warn("Error loading config file: %s" %
                            self.config_file_name, exc_info=True)
             self.file_config = Config()
-        else:
-            self.log.debug("Config file loaded: <%s>" % loader.full_filename)
-            self.log.debug(repr(self.file_config))
+
+    def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
-        # line.
+        # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
@@ -248,13 +408,31 b' class Application(object):'
         """Do actions after the config file is loaded."""
         pass
 
+    def log_file_config(self):
+        if hasattr(self.file_config.Global, 'config_file'):
+            self.log.debug("Config file loaded: %s" %
+                           self.file_config.Global.config_file)
+            self.log.debug(repr(self.file_config))
+
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
+        if self.override_config is None:
         config._merge(self.file_config)
         config._merge(self.command_line_config)
+            if self.constructor_config is not None:
+                config._merge(self.constructor_config)
+        else:
+            config._merge(self.override_config)
+        # XXX fperez - propose to Brian we rename master_config to simply
+        # config, I think this is going to be heavily used in examples and
+        # application code and the name is shorter/easier to find/remember.
+        # For now, just alias it...
         self.master_config = config
+        self.config = config
+
+    def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
@@ -280,21 +458,31 b' class Application(object):'
 
     def abort(self):
         """Abort the starting of the application."""
+        if self._exiting:
+            pass
+        else:
         self.log.critical("Aborting application: %s" % self.name, exc_info=True)
+            self._exiting = True
         sys.exit(1)
 
-    def exit(self):
-        self.log.critical("Aborting application: %s" % self.name)
-        sys.exit(1)
+    def exit(self, exit_status=0):
+        if self._exiting:
+            pass
+        else:
+            self.log.debug("Exiting application: %s" % self.name)
+            self._exiting = True
+            sys.exit(exit_status)
 
     def attempt(self, func, action='abort'):
         try:
             func()
         except SystemExit:
-            self.exit()
+            raise
         except:
             if action == 'abort':
+                self.log.critical("Aborting application: %s" % self.name,
+                                  exc_info=True)
                 self.abort()
+                raise
             elif action == 'exit':
-                self.exit()
-      
+                self.exit(0)

IPython/core/builtin_trap.py

@@ -31,8 +31,8 b' from IPython.utils.autoattr import auto_attr'
 #-----------------------------------------------------------------------------
 
 
-class BuiltinUndefined(object): pass
-BuiltinUndefined = BuiltinUndefined()
+class __BuiltinUndefined(object): pass
+BuiltinUndefined = __BuiltinUndefined()
 
 
 class BuiltinTrap(Component):
@@ -86,6 +86,7 b' class BuiltinTrap(Component):'
         """Store ipython references in the __builtin__ namespace."""
         self.add_builtin('exit', Quitter(self.shell, 'exit'))
         self.add_builtin('quit', Quitter(self.shell, 'quit'))
+        self.add_builtin('get_ipython', self.shell.get_ipython)
 
         # Recursive reload function
         try:

IPython/core/completer.py

@@ -44,7 +44,6 b' its input.'
 
 - When the original stdin is not a tty device, GNU readline is never
 used, and this module (and the readline module) are silently inactive.
-
 """
 
 #*****************************************************************************
@@ -54,14 +53,19 b' used, and this module (and the readline module) are silently inactive.'
 # proper procedure is to maintain its copyright as belonging to the Python
 # Software Foundation (in addition to my own, for all new code).
 #
+#       Copyright (C) 2008-2010 IPython Development Team
+#       Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
-#       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 import __builtin__
 import __main__
 import glob
@@ -73,23 +77,57 b' import shlex'
 import sys
 import types
 
+import IPython.utils.rlineimpl as readline
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
-
-import IPython.utils.rlineimpl as readline
-from IPython.utils.ipstruct import Struct
 from IPython.utils import generics
-
-# Python 2.4 offers sets as a builtin
-try:
-    set()
-except NameError:
-    from sets import Set as set
-
 from IPython.utils.genutils import debugx, dir2
 
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Public API
 __all__ = ['Completer','IPCompleter']
 
+if sys.platform == 'win32':
+    PROTECTABLES = ' '
+else:
+    PROTECTABLES = ' ()'
+
+#-----------------------------------------------------------------------------
+# Main functions and classes
+#-----------------------------------------------------------------------------
+
+def protect_filename(s):
+    """Escape a string to protect certain characters."""
+    
+    return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
+                    for ch in s])
+
+
+def single_dir_expand(matches):
+    "Recursively expand match lists containing a single dir."
+
+    if len(matches) == 1 and os.path.isdir(matches[0]):
+        # Takes care of links to directories also.  Use '/'
+        # explicitly, even under Windows, so that name completions
+        # don't end up escaped.
+        d = matches[0]
+        if d[-1] in ['/','\\']:
+            d = d[:-1]
+
+        subdirs = os.listdir(d)
+        if subdirs:
+            matches = [ (d + '/' + p) for p in subdirs]
+            return single_dir_expand(matches)
+        else:
+            return matches
+    else:
+        return matches
+
+class Bunch: pass
+
 class Completer:
     def __init__(self,namespace=None,global_namespace=None):
         """Create a new completer for the command line.
@@ -152,6 +190,7 b' class Completer:'
         defined in self.namespace or self.global_namespace that match.
 
         """
+        #print 'Completer->global_matches, txt=%r' % text # dbg
         matches = []
         match_append = matches.append
         n = len(text)
@@ -179,6 +218,7 b' class Completer:'
         """
         import re
 
+        #print 'Completer->attr_matches, txt=%r' % text # dbg
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
@@ -205,6 +245,7 b' class Completer:'
         res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
         return res
 
+
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
 
@@ -235,7 +276,7 b' class IPCompleter(Completer):'
         to complete. """
 
         Completer.__init__(self,namespace,global_namespace)
-        self.magic_prefix = shell.name+'.magic_'
+
         self.magic_escape = ESC_MAGIC
         self.readline = readline
         delims = self.readline.get_completer_delims()
@@ -245,6 +286,7 b' class IPCompleter(Completer):'
         self.get_endidx = self.readline.get_endidx
         self.omit__names = omit__names
         self.merge_completions = shell.readline_merge_completions        
+        self.shell = shell.shell
         if alias_table is None:
             alias_table = {}
         self.alias_table = alias_table
@@ -263,12 +305,14 b' class IPCompleter(Completer):'
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
+
+        # All active matcher routines for completion
         self.matchers = [self.python_matches,
                          self.file_matches,
+                         self.magic_matches,
                          self.alias_matches,
                          self.python_func_kw_matches]
 
-    
     # Code contributed by Alex Schmolck, for ipython/emacs integration
     def all_completions(self, text):
         """Return all possible completions for the benefit of emacs."""
@@ -278,9 +322,8 b' class IPCompleter(Completer):'
         try:
             for i in xrange(sys.maxint):
                 res = self.complete(text, i)
-
-                if not res: break
-
+                if not res:
+                    break
                 comp_append(res)
         #XXX workaround for ``notDefined.<tab>``
         except NameError:
@@ -316,41 +359,12 b' class IPCompleter(Completer):'
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
 
-        if sys.platform == 'win32':
-            protectables = ' '
-        else:
-            protectables = ' ()'
-
         if text.startswith('!'):
             text = text[1:]
             text_prefix = '!'
         else:
             text_prefix = ''
             
-        def protect_filename(s):
-            return "".join([(ch in protectables and '\\' + ch or ch)
-                            for ch in s])
-
-        def single_dir_expand(matches):
-            "Recursively expand match lists containing a single dir."
-            
-            if len(matches) == 1 and os.path.isdir(matches[0]):
-                # Takes care of links to directories also.  Use '/'
-                # explicitly, even under Windows, so that name completions
-                # don't end up escaped.
-                d = matches[0]
-                if d[-1] in ['/','\\']:
-                    d = d[:-1]
-
-                subdirs = os.listdir(d)
-                if subdirs:
-                    matches = [ (d + '/' + p) for p in subdirs]
-                    return single_dir_expand(matches)
-                else:
-                    return matches
-            else:
-                return matches
-        
         lbuf = self.lbuf
         open_quotes = 0  # track strings with open quotes
         try:
@@ -402,13 +416,24 b' class IPCompleter(Completer):'
         #print 'mm',matches  # dbg
         return single_dir_expand(matches)
 
+    def magic_matches(self, text):
+        """Match magics"""
+        #print 'Completer->magic_matches:',text,'lb',self.lbuf # dbg
+        # Get all shell magics now rather than statically, so magics loaded at
+        # runtime show up too
+        magics = self.shell.lsmagic()
+        pre = self.magic_escape
+        baretext = text.lstrip(pre)
+        return [ pre+m for m in magics if m.startswith(baretext)]
+
     def alias_matches(self, text):
         """Match internal system aliases"""        
         #print 'Completer->alias_matches:',text,'lb',self.lbuf # dbg
         
         # if we are not in the first 'item', alias matching 
         # doesn't make sense - unless we are starting with 'sudo' command.
-        if ' ' in self.lbuf.lstrip() and not self.lbuf.lstrip().startswith('sudo'):
+        if ' ' in self.lbuf.lstrip() and \
+               not self.lbuf.lstrip().startswith('sudo'):
             return []
         text = os.path.expanduser(text)
         aliases =  self.alias_table.keys()
@@ -420,7 +445,7 b' class IPCompleter(Completer):'
     def python_matches(self,text):
         """Match attributes or global python names"""
 
-        #print 'Completer->python_matches, txt=<%s>' % text # dbg
+        #print 'Completer->python_matches, txt=%r' % text # dbg
         if "." in text:
             try:
                 matches = self.attr_matches(text)
@@ -439,11 +464,7 b' class IPCompleter(Completer):'
                 matches = []
         else:
             matches = self.global_matches(text)
-            # this is so completion finds magics when automagic is on:
-            if (matches == [] and 
-                 not text.startswith(os.sep) and
-                 not ' ' in self.lbuf):
-                matches = self.attr_matches(self.magic_prefix+text)
+
         return matches
 
     def _default_arguments(self, obj):
@@ -514,9 +535,11 b' class IPCompleter(Completer):'
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
-            try: namedArgs = self._default_arguments(eval(callableMatch,
+            try:
+                namedArgs = self._default_arguments(eval(callableMatch,
                                                          self.namespace))
-            except: continue
+            except:
+                continue
             for namedArg in namedArgs:
                 if namedArg.startswith(text):
                     argMatches.append("%s=" %namedArg)
@@ -528,7 +551,7 b' class IPCompleter(Completer):'
         if not line.strip():
             return None
 
-        event = Struct()
+        event = Bunch()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
@@ -542,9 +565,7 b' class IPCompleter(Completer):'
         else:
             try_magic = []
         
-        
-        for c in itertools.chain(
-                                 self.custom_completers.s_matches(cmd),
+        for c in itertools.chain(self.custom_completers.s_matches(cmd),
                                  try_magic,
                                  self.custom_completers.flat_matches(self.lbuf)):
             #print "try",c # dbg
@@ -555,7 +576,8 b' class IPCompleter(Completer):'
                 if withcase:
                     return withcase
                 # if none, then case insensitive ones are ok too
-                return [r for r in res if r.lower().startswith(text.lower())]
+                text_low = text.lower()
+                return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
             
@@ -598,14 +620,11 b' class IPCompleter(Completer):'
             return None
         
         magic_escape = self.magic_escape
-        magic_prefix = self.magic_prefix
 
         self.lbuf = self.full_lbuf[:self.get_endidx()]
 
         try:
-            if text.startswith(magic_escape):
-                text = text.replace(magic_escape,magic_prefix)
-            elif text.startswith('~'):
+            if text.startswith('~'):
                 text = os.path.expanduser(text)
             if state == 0:
                 custom_res = self.dispatch_custom_completer(text)
@@ -625,13 +644,10 b' class IPCompleter(Completer):'
                             self.matches = matcher(text)
                             if self.matches:
                                 break
-                    def uniq(alist):
-                        set = {}
-                        return [set.setdefault(e,e) for e in alist if e not in set]
-                    self.matches = uniq(self.matches)                
+                    self.matches = list(set(self.matches))
             try:
-                ret = self.matches[state].replace(magic_prefix,magic_escape)                
-                return ret
+                #print "MATCH: %r" % self.matches[state] # dbg
+                return self.matches[state]
             except IndexError:
                 return None
         except:

IPython/core/component.py

@@ -157,7 +157,7 b' def masquerade_as(instance, cls):'
     cls.register_instance(instance)
 
 
-class ComponentNameGenerator(object):
+class __ComponentNameGenerator(object):
     """A Singleton to generate unique component names."""
 
     def __init__(self, prefix):
@@ -170,7 +170,7 b' class ComponentNameGenerator(object):'
         return "%s%s" % (self.prefix, count)
 
 
-ComponentNameGenerator = ComponentNameGenerator('ipython.component')
+ComponentNameGenerator = __ComponentNameGenerator('ipython.component')
 
 
 class MetaComponent(MetaHasTraits, MetaComponentTracker):
@@ -237,14 +237,20 b' class Component(HasTraits):'
             self.config = config
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
-            # will share config.
+            # will share config. In fact, I did find such a side effect in
+            # _config_changed below. If a config attribute value was a mutable type
+            # all instances of a component were getting the same copy, effectively
+            # making that a class attribute.
             # self.config = deepcopy(config)
         else:
             if self.parent is not None:
                 self.config = self.parent.config
                 # We used to deepcopy, but for now we are trying to just save
                 # by reference.  This *could* have side effects as all components
-                # will share config.
+                # will share config. In fact, I did find such a side effect in
+                # _config_changed below. If a config attribute value was a mutable type
+                # all instances of a component were getting the same copy, effectively
+                # making that a class attribute.
                 # self.config = deepcopy(self.parent.config)
 
         self.created = datetime.datetime.now()
@@ -296,14 +302,29 b' class Component(HasTraits):'
             if new._has_section(sname):
                 my_config = new[sname]
                 for k, v in traits.items():
+                    # Don't allow traitlets with config=True to start with
+                    # uppercase.  Otherwise, they are confused with Config
+                    # subsections.  But, developers shouldn't have uppercase
+                    # attributes anyways! (PEP 6)
+                    if k[0].upper()==k[0] and not k.startswith('_'):
+                        raise ComponentError('Component traitlets with '
+                        'config=True must start with a lowercase so they are '
+                        'not confused with Config subsections: %s.%s' % \
+                        (self.__class__.__name__, k))
                     try:
+                        # Here we grab the value from the config
+                        # If k has the naming convention of a config
+                        # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass
                     else:
                         # print "Setting %s.%s from %s.%s=%r" % \
                         #     (self.__class__.__name__,k,sname,k,config_value)
-                        setattr(self, k, config_value)
+                        # We have to do a deepcopy here if we don't deepcopy the entire
+                        # config object. If we don't, a mutable config_value will be
+                        # shared by all instances, effectively making it a class attribute.
+                        setattr(self, k, deepcopy(config_value))
 
     @property
     def children(self):

IPython/core/crashhandler.py

@@ -28,10 +28,8 b' from IPython.core import release'
 from IPython.core import ultratb
 from IPython.external.Itpl import itpl
 
-from IPython.utils.genutils import *
-
 #****************************************************************************
-class CrashHandler:
+class CrashHandler(object):
     """Customizable crash handlers for IPython-based systems.
 
     Instances of this class provide a __call__ method which can be used as a
@@ -41,15 +39,15 b' class CrashHandler:'
 
     """
 
-    def __init__(self,IP,app_name,contact_name,contact_email,
-                 bug_tracker,crash_report_fname,
-                 show_crash_traceback=True):
+    def __init__(self,app, app_name, contact_name=None, contact_email=None, 
+                 bug_tracker=None, crash_report_fname='CrashReport.txt', 
+                 show_crash_traceback=True, call_pdb=False):
         """New crash handler.
 
         Inputs:
 
-        - IP: a running IPython instance, which will be queried at crash time
-        for internal information.
+        - app: a running application instance, which will be queried at crash
+        time for internal information.
 
         - app_name: a string containing the name of your application.
 
@@ -77,13 +75,16 b' class CrashHandler:'
         """
 
         # apply args into instance
-        self.IP = IP  # IPython instance
+        self.app = app
         self.app_name = app_name
         self.contact_name = contact_name
         self.contact_email = contact_email
         self.bug_tracker = bug_tracker
         self.crash_report_fname = crash_report_fname
         self.show_crash_traceback = show_crash_traceback
+        self.section_sep = '\n\n'+'*'*75+'\n\n'
+        self.call_pdb = call_pdb
+        #self.call_pdb = True # dbg
         
         # Hardcoded defaults, which can be overridden either by subclasses or
         # at runtime for the instance.
@@ -124,7 +125,7 b' $self.bug_tracker'
         #color_scheme = 'Linux'   # dbg
         
         try:
-            rptdir = self.IP.config.IPYTHONDIR
+            rptdir = self.app.ipython_dir
         except:
             rptdir = os.getcwd()
         if not os.path.isdir(rptdir):
@@ -134,7 +135,13 b' $self.bug_tracker'
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         TBhandler = ultratb.VerboseTB(color_scheme=color_scheme,
-                                           long_header=1)
+                                      long_header=1,
+                                      call_pdb=self.call_pdb,
+                                      )
+        if self.call_pdb:
+            TBhandler(etype,evalue,etb)
+            return
+        else:
         traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
@@ -155,74 +162,66 b' $self.bug_tracker'
         # Construct report on disk
         report.write(self.make_report(traceback))
         report.close()
-        raw_input("Press enter to exit:")
+        raw_input("Hit <Enter> to quit this message (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
+        import platform
 
-        sec_sep = '\n\n'+'*'*75+'\n\n'
+        sec_sep = self.section_sep
 
         report = []
         rpt_add = report.append
         
         rpt_add('*'*75+'\n\n'+'IPython post-mortem report\n\n')
-        rpt_add('IPython version: %s \n\n' % release.version)
-        rpt_add('BZR revision   : %s \n\n' % release.revision)
-        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s' %
+        rpt_add('IPython version: %s \n' % release.version)
+        rpt_add('BZR revision   : %s \n' % release.revision)
+        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s\n' %
                      (os.name,sys.platform) )
-        rpt_add(sec_sep+'Current user configuration structure:\n\n')
-        rpt_add(pformat(self.IP.dict()))
-        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
+        rpt_add('               : %s\n' % platform.platform())
+        rpt_add('Python info    : %s\n' % sys.version)	     
+
         try:
-            rpt_add(sec_sep+"History of session input:")
-            for line in self.IP.user_ns['_ih']:
-                rpt_add(line)
-            rpt_add('\n*** Last line of input (may not be in above history):\n')
-            rpt_add(self.IP._last_input_line+'\n')
+            config = pformat(self.app.config)
+            rpt_add(sec_sep+'Current user configuration structure:\n\n')
+            rpt_add(config)
         except:
             pass
+        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 
+
 class IPythonCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
     
-    def __init__(self,IP):
+    def __init__(self, app, app_name='IPython'):
 
         # Set here which of the IPython authors should be listed as contact
         AUTHOR_CONTACT = 'Fernando'
         
         # Set argument defaults
-        app_name = 'IPython'
         bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
         contact_name,contact_email = release.authors[AUTHOR_CONTACT][:2]
         crash_report_fname = 'IPython_crash_report.txt'
         # Call parent constructor
-        CrashHandler.__init__(self,IP,app_name,contact_name,contact_email,
+        CrashHandler.__init__(self,app,app_name,contact_name,contact_email,
                               bug_tracker,crash_report_fname)
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
-        sec_sep = '\n\n'+'*'*75+'\n\n'
-
-        report = []
+        sec_sep = self.section_sep
+        # Start with parent report
+        report = [super(IPythonCrashHandler, self).make_report(traceback)]
+        # Add interactive-specific info we may have
         rpt_add = report.append
-        
-        rpt_add('*'*75+'\n\n'+'IPython post-mortem report\n\n')
-        rpt_add('IPython version: %s \n\n' % release.version)
-        rpt_add('BZR revision   : %s \n\n' % release.revision)
-        rpt_add('Platform info  : os.name -> %s, sys.platform -> %s' %
-                     (os.name,sys.platform) )
-        rpt_add(sec_sep+'Current user configuration structure:\n\n')
-        # rpt_add(pformat(self.IP.dict()))
-        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
         try:
             rpt_add(sec_sep+"History of session input:")
-            for line in self.IP.user_ns['_ih']:
+            for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
-            rpt_add(self.IP._last_input_line+'\n')
+            rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 

IPython/core/debugger.py

@@ -70,6 +70,7 b' def BdbQuit_excepthook(et,ev,tb):'
 def BdbQuit_IPython_excepthook(self,et,ev,tb):
     print 'Exiting Debugger.'
 
+
 class Tracer(object):
     """Class for local debugging, similar to pdb.set_trace.
 
@@ -105,12 +106,10 b' class Tracer(object):'
         from the Python standard library for usage details.
         """
 
-        global __IPYTHON__
         try:
-            __IPYTHON__
-        except NameError:
+            ip = ipapi.get()
+        except:
             # Outside of ipython, we set our own exception hook manually
-            __IPYTHON__ = ipapi.get()
             BdbQuit_excepthook.excepthook_ori = sys.excepthook
             sys.excepthook = BdbQuit_excepthook
             def_colors = 'NoColor'
@@ -122,7 +121,6 b' class Tracer(object):'
                 pass
         else:
             # In ipython, we use its custom exception handler mechanism
-            ip = ipapi.get()
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,),BdbQuit_IPython_excepthook)
 
@@ -138,6 +136,7 b' class Tracer(object):'
         
         self.debugger.set_trace(sys._getframe().f_back)
 
+
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the do_... commands that hook into the help system.
@@ -149,6 +148,7 b' def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):'
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
+
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
@@ -164,10 +164,10 b' def _file_lines(fname):'
         outfile.close()
         return out
 
+
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline."""
 
-    if sys.version[:3] >= '2.5' or has_pydb:
         def __init__(self,color_scheme='NoColor',completekey=None,
                      stdin=None, stdout=None):
 
@@ -182,6 +182,8 b' class Pdb(OldPdb):'
             # IPython changes...
             self.is_pydb = has_pydb
 
+        self.shell = ipapi.get()
+
             if self.is_pydb:
 
                 # iplib.py's ipalias seems to want pdb's checkline
@@ -193,8 +195,8 b' class Pdb(OldPdb):'
                 self.curframe = None
                 self.do_restart = self.new_do_restart
 
-                self.old_all_completions = __IPYTHON__.Completer.all_completions
-                __IPYTHON__.Completer.all_completions=self.all_completions
+            self.old_all_completions = self.shell.Completer.all_completions
+            self.shell.Completer.all_completions=self.all_completions
 
                 self.do_list = decorate_fn_with_doc(self.list_command_pydb,
                                                     OldPdb.do_list)
@@ -227,80 +229,33 b' class Pdb(OldPdb):'
             # debugging.
             self.parser = PyColorize.Parser()
 
-
-    else:
-        # Ugly hack: for Python 2.3-2.4, we can't call the parent constructor,
-        # because it binds readline and breaks tab-completion.  This means we
-        # have to COPY the constructor here.
-        def __init__(self,color_scheme='NoColor'):
-            bdb.Bdb.__init__(self)
-            cmd.Cmd.__init__(self,completekey=None) # don't load readline
-            self.prompt = 'ipdb> ' # The default prompt is '(Pdb)'
-            self.aliases = {}
-
-            # These two lines are part of the py2.4 constructor, let's put them
-            # unconditionally here as they won't cause any problems in 2.3.
-            self.mainpyfile = ''
-            self._wait_for_mainpyfile = 0
-
-            # Read $HOME/.pdbrc and ./.pdbrc
-            try:
-                self.rcLines = _file_lines(os.path.join(os.environ['HOME'],
-                                                        ".pdbrc"))
-            except KeyError:
-                self.rcLines = []
-            self.rcLines.extend(_file_lines(".pdbrc"))
-
-            # Create color table: we copy the default one from the traceback
-            # module and add a few attributes needed for debugging
-            self.color_scheme_table = exception_colors()
-
-            # shorthands 
-            C = coloransi.TermColors
-            cst = self.color_scheme_table
-
-            cst['NoColor'].colors.breakpoint_enabled = C.NoColor
-            cst['NoColor'].colors.breakpoint_disabled = C.NoColor
-
-            cst['Linux'].colors.breakpoint_enabled = C.LightRed
-            cst['Linux'].colors.breakpoint_disabled = C.Red
-
-            cst['LightBG'].colors.breakpoint_enabled = C.LightRed
-            cst['LightBG'].colors.breakpoint_disabled = C.Red
-
-            self.set_colors(color_scheme)
-
-            # Add a python parser so we can syntax highlight source while
-            # debugging.
-            self.parser = PyColorize.Parser()
-        
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
 
     def interaction(self, frame, traceback):
-        __IPYTHON__.set_completer_frame(frame)
+        self.shell.set_completer_frame(frame)
         OldPdb.interaction(self, frame, traceback)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
-        __IPYTHON__.set_completer_frame(self.curframe)
+        self.shell.set_completer_frame(self.curframe)
 
     def new_do_quit(self, arg):
         
         if hasattr(self, 'old_all_completions'):
-            __IPYTHON__.Completer.all_completions=self.old_all_completions
+            self.shell.Completer.all_completions=self.old_all_completions
         
         
         return OldPdb.do_quit(self, arg)
@@ -314,7 +269,7 b' class Pdb(OldPdb):'
         return self.do_quit(arg)
 
     def postloop(self):
-        __IPYTHON__.set_completer_frame(None)
+        self.shell.set_completer_frame(None)
 
     def print_stack_trace(self):
         try:
@@ -331,7 +286,7 b' class Pdb(OldPdb):'
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
-        __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context = 3):
@@ -500,7 +455,7 b' class Pdb(OldPdb):'
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
-        __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
@@ -509,16 +464,49 b' class Pdb(OldPdb):'
         """The debugger interface to magic_pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pdef(arg, namespaces=namespaces)
+        self.shell.magic_pdef(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """The debugger interface to magic_pdoc"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pdoc(arg, namespaces=namespaces)
+        self.shell.magic_pdoc(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """The debugger equivalant of ?obj"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
-        __IPYTHON__.magic_pinfo("pinfo %s" % arg, namespaces=namespaces)
+        self.shell.magic_pinfo("pinfo %s" % arg, namespaces=namespaces)
+
+    def checkline(self, filename, lineno):
+        """Check whether specified line seems to be executable.
+
+        Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
+        line or EOF). Warning: testing is not comprehensive.
+        """
+        #######################################################################
+        # XXX Hack!  Use python-2.5 compatible code for this call, because with
+        # all of our changes, we've drifted from the pdb api in 2.6.  For now,
+        # changing:
+        #
+        #line = linecache.getline(filename, lineno, self.curframe.f_globals)
+        # to:
+        #
+        line = linecache.getline(filename, lineno)
+        #
+        # does the trick.  But in reality, we need to fix this by reconciling
+        # our updates with the new Pdb APIs in Python 2.6.
+        #
+        # End hack.  The rest of this method is copied verbatim from 2.6 pdb.py
+        #######################################################################
+        
+        if not line:
+            print >>self.stdout, 'End of file'
+            return 0
+        line = line.strip()
+        # Don't allow setting breakpoint at a blank line
+        if (not line or (line[0] == '#') or
+             (line[:3] == '"""') or line[:3] == "'''"):
+            print >>self.stdout, '*** Blank or comment'
+            return 0
+        return lineno

IPython/core/display_trap.py

@@ -46,11 +46,11 b' class DisplayTrap(Component):'
         # Only turn off the trap when the outermost call to __exit__ is made.
         self._nested_level = 0
 
-    @auto_attr
-    def shell(self):
-        return Component.get_instances(
-            root=self.root,
-            klass='IPython.core.iplib.InteractiveShell')[0]
+    # @auto_attr
+    # def shell(self):
+    #     return Component.get_instances(
+    #         root=self.root,
+    #         klass='IPython.core.iplib.InteractiveShell')[0]
 
     def __enter__(self):
         if self._nested_level == 0:

IPython/core/embed.py

@@ -68,7 +68,7 b' class InteractiveShellEmbed(InteractiveShell):'
     # is True by default.
     display_banner = CBool(True)
 
-    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
+    def __init__(self, parent=None, config=None, ipython_dir=None, usage=None,
                  user_ns=None, user_global_ns=None,
                  banner1=None, banner2=None, display_banner=None,
                  custom_exceptions=((),None), exit_msg=''):
@@ -76,7 +76,7 b' class InteractiveShellEmbed(InteractiveShell):'
         self.save_sys_ipcompleter()
 
         super(InteractiveShellEmbed,self).__init__(
-            parent=parent, config=config, ipythondir=ipythondir, usage=usage, 
+            parent=parent, config=config, ipython_dir=ipython_dir, usage=usage, 
             user_ns=user_ns, user_global_ns=user_global_ns,
             banner1=banner1, banner2=banner2, display_banner=display_banner,
             custom_exceptions=custom_exceptions)
@@ -233,14 +233,6 b' class InteractiveShellEmbed(InteractiveShell):'
             for var in local_varnames:
                 delvar(var,None)
 
-    def set_completer_frame(self, frame=None):
-        if frame:
-            self.Completer.namespace = frame.f_locals
-            self.Completer.global_namespace = frame.f_globals
-        else:
-            self.Completer.namespace = self.user_ns
-            self.Completer.global_namespace = self.user_global_ns
-
 
 _embedded_shell = None
 

IPython/core/history.py

@@ -15,19 +15,24 b" def magic_history(self, parameter_s = ''):"
     %history n     -> print at most n inputs\\
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
     
-    Each input's number <n> is shown, and is accessible as the
-    automatically generated variable _i<n>.  Multi-line statements are
-    printed starting at a new line for easy copy/paste.
+    By default, input history is printed without line numbers so it can be
+    directly pasted into an editor.
     
+    With -n, each input's number <n> is shown, and is accessible as the
+    automatically generated variable _i<n> as well as In[<n>].  Multi-line
+    statements are printed starting at a new line for easy copy/paste.
 
     Options:
 
-      -n: do NOT print line numbers. This is useful if you want to get a
-      printout of many lines which can be directly pasted into a text
-      editor.
-
+      -n: print line numbers for each input.
       This feature is only available if numbered prompts are in use.
 
+      -o: also print outputs for each input.
+
+      -p: print classic '>>>' python prompts before each input.  This is useful
+       for making documentation, and in conjunction with -o, for producing
+       doctest-ready output.
+
       -t: (default) print the 'translated' history, as IPython understands it.
       IPython filters your input and converts it all into valid Python source
       before executing it (things like magics or aliases are turned into
@@ -50,7 +55,7 b" def magic_history(self, parameter_s = ''):"
     if not self.outputcache.do_full_cache:
         print 'This feature is only available if numbered prompts are in use.'
         return
-    opts,args = self.parse_options(parameter_s,'gntsrf:',mode='list')
+    opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
 
     # Check if output to specific file was requested.
     try:
@@ -97,9 +102,12 b" def magic_history(self, parameter_s = ''):"
         warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
         print self.magic_hist.__doc__
         return
+    
     width = len(str(final))
     line_sep = ['','\n']
-    print_nums = not opts.has_key('n')
+    print_nums = 'n' in opts
+    print_outputs = 'o' in opts
+    pyprompts = 'p' in opts
     
     found = False
     if pattern is not None:
@@ -123,7 +131,19 b" def magic_history(self, parameter_s = ''):"
         if print_nums:
             print >> outfile, \
                   '%s:%s' % (str(in_num).ljust(width),line_sep[multiline]),
+        if pyprompts:
+            print >> outfile, '>>>',
+            if multiline:
+                lines = inline.splitlines()
+                print >> outfile, '\n... '.join(lines)
+                print >> outfile, '... '
+            else:
+                print >> outfile, inline,
+        else:
         print >> outfile, inline,
+        output = self.shell.user_ns['Out'].get(in_num)
+        if output is not None:
+            print repr(output)
 
     if close_at_end:
         outfile.close()
@@ -245,10 +265,10 b' class ShadowHist(object):'
 
 
 def init_ipython(ip):
-    import ipy_completers
-
     ip.define_magic("rep",rep_f)        
     ip.define_magic("hist",magic_hist)            
     ip.define_magic("history",magic_history)
 
-    ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
+    # XXX - ipy_completers are in quarantine, need to be updated to new apis
+    #import ipy_completers
+    #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')

IPython/core/hooks.py

@@ -137,8 +137,7 b' class CommandChainDispatcher:'
         for prio,cmd in self.chain:
             #print "prio",prio,"cmd",cmd #dbg
             try:
-                ret = cmd(*args, **kw)
-                return ret
+                return cmd(*args, **kw)
             except TryNext, exc:
                 if exc.args or exc.kwargs:
                     args = exc.args

IPython/core/ipapi.py

@@ -18,16 +18,19 b' has been made into a component, this module will be sent to deathrow.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.core.error import TryNext, UsageError
+from IPython.core.error import TryNext, UsageError, IPythonCoreError
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
+
 def get():
     """Get the most recently created InteractiveShell instance."""
     from IPython.core.iplib import InteractiveShell
     insts = InteractiveShell.get_instances()
+    if len(insts)==0:
+        return None
     most_recent = insts[0]
     for inst in insts[1:]:
         if inst.created > most_recent.created:

IPython/core/ipapp.py

@@ -1,19 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
-The main IPython application object
+The :class:`~IPython.core.application.Application` object for the command
+line :command:`ipython` program.
 
-Authors:
+Authors
+-------
 
 * Brian Granger
 * Fernando Perez
-
-Notes
------
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2009  The IPython Development Team
+#  Copyright (C) 2008-2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -22,321 +21,405 b' Notes'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import absolute_import
 
 import logging
 import os
 import sys
-import warnings
 
-from IPython.core.application import Application, IPythonArgParseConfigLoader
-from IPython.core import release
+from IPython.core import crashhandler
+from IPython.core.application import Application
 from IPython.core.iplib import InteractiveShell
 from IPython.config.loader import (
-    NoConfigDefault, 
     Config,
-    ConfigError,
-    PyFileConfigLoader
+    PyFileConfigLoader,
+#    NoConfigDefault,
 )
-
 from IPython.lib import inputhook
-
-from IPython.utils.ipstruct import Struct
 from IPython.utils.genutils import filefind, get_ipython_dir
+from . import usage
 
 #-----------------------------------------------------------------------------
-# Utilities and helpers
+# Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
-
-ipython_desc = """
-A Python shell with automatic history (input and output), dynamic object
-introspection, easier configuration, command completion, access to the system
-shell and more.
-"""
-
-def pylab_warning():
-    msg = """
-
-IPython's -pylab mode has been disabled until matplotlib supports this version
-of IPython.  This version of IPython has greatly improved GUI integration that 
-matplotlib will soon be able to take advantage of.  This will eventually
-result in greater stability and a richer API for matplotlib under IPython.
-However during this transition, you will either need to use an older version
-of IPython, or do the following to use matplotlib interactively::
-
-    import matplotlib
-    matplotlib.interactive(True)
-    matplotlib.use('wxagg')  # adjust for your backend
-    %gui -a wx               # adjust for your GUI
-    from matplotlib import pyplot as plt
-
-See the %gui magic for information on the new interface.
-"""
-    warnings.warn(msg, category=DeprecationWarning, stacklevel=1)
-
-
-#-----------------------------------------------------------------------------
-# Main classes and functions
-#-----------------------------------------------------------------------------
+default_config_file_name = u'ipython_config.py'
 
 cl_args = (
-    (('-autocall',), dict(
-        type=int, dest='InteractiveShell.autocall', default=NoConfigDefault,
-        help='Set the autocall value (0,1,2).',
+    (('--autocall',), dict(
+        type=int, dest='InteractiveShell.autocall',
+        help=
+        """Make IPython automatically call any callable object even if you
+        didn't type explicit parentheses. For example, 'str 43' becomes
+        'str(43)' automatically.  The value can be '0' to disable the feature,
+        '1' for 'smart' autocall, where it is not applied if there are no more
+        arguments on the line, and '2' for 'full' autocall, where all callable
+        objects are automatically called (even if no arguments are present).
+        The default is '1'.""",
         metavar='InteractiveShell.autocall')
     ),
-    (('-autoindent',), dict(
-        action='store_true', dest='InteractiveShell.autoindent', default=NoConfigDefault,
+    (('--autoindent',), dict(
+        action='store_true', dest='InteractiveShell.autoindent',
         help='Turn on autoindenting.')
     ),
-    (('-noautoindent',), dict(
-        action='store_false', dest='InteractiveShell.autoindent', default=NoConfigDefault,
+    (('--no-autoindent',), dict(
+        action='store_false', dest='InteractiveShell.autoindent',
         help='Turn off autoindenting.')
     ),
-    (('-automagic',), dict(
-        action='store_true', dest='InteractiveShell.automagic', default=NoConfigDefault,
-        help='Turn on the auto calling of magic commands.')
+    (('--automagic',), dict(
+        action='store_true', dest='InteractiveShell.automagic',
+        help='Turn on the auto calling of magic commands.'
+        'Type %%magic at  the  IPython  prompt  for  more information.')
     ),
-    (('-noautomagic',), dict(
-        action='store_false', dest='InteractiveShell.automagic', default=NoConfigDefault,
+    (('--no-automagic',), dict(
+        action='store_false', dest='InteractiveShell.automagic',
         help='Turn off the auto calling of magic commands.')
     ),
-    (('-autoedit_syntax',), dict(
-        action='store_true', dest='InteractiveShell.autoedit_syntax', default=NoConfigDefault,
+    (('--autoedit-syntax',), dict(
+        action='store_true', dest='InteractiveShell.autoedit_syntax',
         help='Turn on auto editing of files with syntax errors.')
     ),
-    (('-noautoedit_syntax',), dict(
-        action='store_false', dest='InteractiveShell.autoedit_syntax', default=NoConfigDefault,
+    (('--no-autoedit-syntax',), dict(
+        action='store_false', dest='InteractiveShell.autoedit_syntax',
         help='Turn off auto editing of files with syntax errors.')
     ),
-    (('-banner',), dict(
-        action='store_true', dest='Global.display_banner', default=NoConfigDefault,
+    (('--banner',), dict(
+        action='store_true', dest='Global.display_banner',
         help='Display a banner upon starting IPython.')
     ),
-    (('-nobanner',), dict(
-        action='store_false', dest='Global.display_banner', default=NoConfigDefault,
+    (('--no-banner',), dict(
+        action='store_false', dest='Global.display_banner',
         help="Don't display a banner upon starting IPython.")
     ),
-    (('-cache_size',), dict(
-        type=int, dest='InteractiveShell.cache_size', default=NoConfigDefault,
-        help="Set the size of the output cache.",
+    (('--cache-size',), dict(
+        type=int, dest='InteractiveShell.cache_size',
+        help=
+        """Set the size of the output cache.  The default is 1000, you can
+        change it permanently in your config file.  Setting it to 0 completely
+        disables the caching system, and the minimum value accepted is 20 (if
+        you provide a value less than 20, it is reset to 0 and a warning is
+        issued).  This limit is defined because otherwise you'll spend more
+        time re-flushing a too small cache than working.
+        """,
         metavar='InteractiveShell.cache_size')
     ),
-    (('-classic',), dict(
-        action='store_true', dest='Global.classic', default=NoConfigDefault,
+    (('--classic',), dict(
+        action='store_true', dest='Global.classic',
         help="Gives IPython a similar feel to the classic Python prompt.")
     ),
-    (('-colors',), dict(
-        type=str, dest='InteractiveShell.colors', default=NoConfigDefault,
+    (('--colors',), dict(
+        type=str, dest='InteractiveShell.colors',
         help="Set the color scheme (NoColor, Linux, and LightBG).",
         metavar='InteractiveShell.colors')
     ),
-    (('-color_info',), dict(
-        action='store_true', dest='InteractiveShell.color_info', default=NoConfigDefault,
-        help="Enable using colors for info related things.")
+    (('--color-info',), dict(
+        action='store_true', dest='InteractiveShell.color_info',
+        help=
+        """IPython can display information about objects via a set of func-
+        tions, and optionally can use colors for this, syntax highlighting
+        source code and various other elements.  However, because this
+        information is passed through a pager (like 'less') and many pagers get
+        confused with color codes, this option is off by default.  You can test
+        it and turn it on permanently in your ipython_config.py file if it
+        works for you.  Test it and turn it on permanently if it works with
+        your system.  The magic function %%color_info allows you to toggle this
+        inter- actively for testing."""
+        )
     ),
-    (('-nocolor_info',), dict(
-        action='store_false', dest='InteractiveShell.color_info', default=NoConfigDefault,
+    (('--no-color-info',), dict(
+        action='store_false', dest='InteractiveShell.color_info',
         help="Disable using colors for info related things.")
     ),
-    (('-confirm_exit',), dict(
-        action='store_true', dest='InteractiveShell.confirm_exit', default=NoConfigDefault,
-        help="Prompt the user when existing.")
-    ),
-    (('-noconfirm_exit',), dict(
-        action='store_false', dest='InteractiveShell.confirm_exit', default=NoConfigDefault,
-        help="Don't prompt the user when existing.")
-    ),
-    (('-deep_reload',), dict(
-        action='store_true', dest='InteractiveShell.deep_reload', default=NoConfigDefault,
-        help="Enable deep (recursive) reloading by default.")
+    (('--confirm-exit',), dict(
+        action='store_true', dest='InteractiveShell.confirm_exit',
+        help=
+        """Set to confirm when you try to exit IPython with an EOF (Control-D
+        in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
+        '%%Exit', you can force a direct exit without any confirmation.
+        """
+        )
     ),
-    (('-nodeep_reload',), dict(
-        action='store_false', dest='InteractiveShell.deep_reload', default=NoConfigDefault,
+    (('--no-confirm-exit',), dict(
+        action='store_false', dest='InteractiveShell.confirm_exit',
+        help="Don't prompt the user when exiting.")
+    ),
+    (('--deep-reload',), dict(
+        action='store_true', dest='InteractiveShell.deep_reload',
+        help=
+        """Enable deep (recursive) reloading by default. IPython can use the
+        deep_reload module which reloads changes in modules recursively (it
+        replaces the reload() function, so you don't need to change anything to
+        use it). deep_reload() forces a full reload of modules whose code may
+        have changed, which the default reload() function does not.  When
+        deep_reload is off, IPython will use the normal reload(), but
+        deep_reload will still be available as dreload(). This fea- ture is off
+        by default [which means that you have both normal reload() and
+        dreload()].""")
+    ),
+    (('--no-deep-reload',), dict(
+        action='store_false', dest='InteractiveShell.deep_reload',
         help="Disable deep (recursive) reloading by default.")
     ),
-    (('-editor',), dict(
-        type=str, dest='InteractiveShell.editor', default=NoConfigDefault,
+    (('--editor',), dict(
+        type=str, dest='InteractiveShell.editor',
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
         metavar='InteractiveShell.editor')
     ),
-    (('-log','-l'), dict(
-        action='store_true', dest='InteractiveShell.logstart', default=NoConfigDefault,
-        help="Start logging to the default file (./ipython_log.py).")
+    (('--log','-l'), dict(
+        action='store_true', dest='InteractiveShell.logstart',
+        help="Start logging to the default log file (./ipython_log.py).")
     ),
-    (('-logfile','-lf'), dict(
-        type=str, dest='InteractiveShell.logfile', default=NoConfigDefault,
-        help="Start logging to logfile.",
+    (('--logfile','-lf'), dict(
+        type=unicode, dest='InteractiveShell.logfile',
+        help="Start logging to logfile with this name.",
         metavar='InteractiveShell.logfile')
     ),
-    (('-logappend','-la'), dict(
-        type=str, dest='InteractiveShell.logappend', default=NoConfigDefault,
-        help="Start logging to logappend in append mode.",
+    (('--log-append','-la'), dict(
+        type=unicode, dest='InteractiveShell.logappend',
+        help="Start logging to the given file in append mode.",
         metavar='InteractiveShell.logfile')
     ),
-    (('-pdb',), dict(
-        action='store_true', dest='InteractiveShell.pdb', default=NoConfigDefault,
+    (('--pdb',), dict(
+        action='store_true', dest='InteractiveShell.pdb',
         help="Enable auto calling the pdb debugger after every exception.")
     ),
-    (('-nopdb',), dict(
-        action='store_false', dest='InteractiveShell.pdb', default=NoConfigDefault,
+    (('--no-pdb',), dict(
+        action='store_false', dest='InteractiveShell.pdb',
         help="Disable auto calling the pdb debugger after every exception.")
     ),
-    (('-pprint',), dict(
-        action='store_true', dest='InteractiveShell.pprint', default=NoConfigDefault,
+    (('--pprint',), dict(
+        action='store_true', dest='InteractiveShell.pprint',
         help="Enable auto pretty printing of results.")
     ),
-    (('-nopprint',), dict(
-        action='store_false', dest='InteractiveShell.pprint', default=NoConfigDefault,
+    (('--no-pprint',), dict(
+        action='store_false', dest='InteractiveShell.pprint',
         help="Disable auto auto pretty printing of results.")
     ),
-    (('-prompt_in1','-pi1'), dict(
-        type=str, dest='InteractiveShell.prompt_in1', default=NoConfigDefault,
-        help="Set the main input prompt ('In [\#]: ')",
+    (('--prompt-in1','-pi1'), dict(
+        type=str, dest='InteractiveShell.prompt_in1',
+        help=
+        """Set the main input prompt ('In [\#]: ').  Note that if you are using
+        numbered prompts, the number is represented with a '\#' in the string.
+        Don't forget to quote strings with spaces embedded in them. Most
+        bash-like escapes can be used to customize IPython's prompts, as well
+        as a few additional ones which are IPython-spe- cific.  All valid
+        prompt escapes are described in detail in the Customization section of
+        the IPython manual.""",
         metavar='InteractiveShell.prompt_in1')
     ),
-    (('-prompt_in2','-pi2'), dict(
-        type=str, dest='InteractiveShell.prompt_in2', default=NoConfigDefault,
-        help="Set the secondary input prompt (' .\D.: ')",
+    (('--prompt-in2','-pi2'), dict(
+        type=str, dest='InteractiveShell.prompt_in2',
+        help=
+        """Set the secondary input prompt (' .\D.: ').  Similar to the previous
+        option, but used for the continuation prompts. The special sequence
+        '\D' is similar to '\#', but with all digits replaced by dots (so you
+        can have your continuation prompt aligned with your input prompt).
+        Default: ' .\D.: ' (note three spaces at the start for alignment with
+        'In [\#]')""",
         metavar='InteractiveShell.prompt_in2')
     ),
-    (('-prompt_out','-po'), dict(
-        type=str, dest='InteractiveShell.prompt_out', default=NoConfigDefault,
+    (('--prompt-out','-po'), dict(
+        type=str, dest='InteractiveShell.prompt_out',
         help="Set the output prompt ('Out[\#]:')",
         metavar='InteractiveShell.prompt_out')
     ),
-    (('-quick',), dict(
-        action='store_true', dest='Global.quick', default=NoConfigDefault,
+    (('--quick',), dict(
+        action='store_true', dest='Global.quick',
         help="Enable quick startup with no config files.")
     ),
-    (('-readline',), dict(
-        action='store_true', dest='InteractiveShell.readline_use', default=NoConfigDefault,
+    (('--readline',), dict(
+        action='store_true', dest='InteractiveShell.readline_use',
         help="Enable readline for command line usage.")
     ),
-    (('-noreadline',), dict(
-        action='store_false', dest='InteractiveShell.readline_use', default=NoConfigDefault,
+    (('--no-readline',), dict(
+        action='store_false', dest='InteractiveShell.readline_use',
         help="Disable readline for command line usage.")
     ),
-    (('-screen_length','-sl'), dict(
-        type=int, dest='InteractiveShell.screen_length', default=NoConfigDefault,
-        help='Number of lines on screen, used to control printing of long strings.',
+    (('--screen-length','-sl'), dict(
+        type=int, dest='InteractiveShell.screen_length',
+        help=
+        """Number of lines of your screen, used to control printing of very
+        long strings.  Strings longer than this number of lines will be sent
+        through a pager instead of directly printed.  The default value for
+        this is 0, which means IPython will auto-detect your screen size every
+        time it needs to print certain potentially long strings (this doesn't
+        change the behavior of the 'print' keyword, it's only triggered
+        internally). If for some reason this isn't working well (it needs
+        curses support), specify it yourself. Otherwise don't change the
+        default.""",
         metavar='InteractiveShell.screen_length')
     ),
-    (('-separate_in','-si'), dict(
-        type=str, dest='InteractiveShell.separate_in', default=NoConfigDefault,
-        help="Separator before input prompts.  Default '\n'.",
+    (('--separate-in','-si'), dict(
+        type=str, dest='InteractiveShell.separate_in',
+        help="Separator before input prompts.  Default '\\n'.",
         metavar='InteractiveShell.separate_in')
     ),
-    (('-separate_out','-so'), dict(
-        type=str, dest='InteractiveShell.separate_out', default=NoConfigDefault,
+    (('--separate-out','-so'), dict(
+        type=str, dest='InteractiveShell.separate_out',
         help="Separator before output prompts.  Default 0 (nothing).",
         metavar='InteractiveShell.separate_out')
     ),
-    (('-separate_out2','-so2'), dict(
-        type=str, dest='InteractiveShell.separate_out2', default=NoConfigDefault,
+    (('--separate-out2','-so2'), dict(
+        type=str, dest='InteractiveShell.separate_out2',
         help="Separator after output prompts.  Default 0 (nonight).",
         metavar='InteractiveShell.separate_out2')
     ),
-    (('-nosep',), dict(
-        action='store_true', dest='Global.nosep', default=NoConfigDefault,
+    (('-no-sep',), dict(
+        action='store_true', dest='Global.nosep',
         help="Eliminate all spacing between prompts.")
     ),
-    (('-term_title',), dict(
-        action='store_true', dest='InteractiveShell.term_title', default=NoConfigDefault,
+    (('--term-title',), dict(
+        action='store_true', dest='InteractiveShell.term_title',
         help="Enable auto setting the terminal title.")
     ),
-    (('-noterm_title',), dict(
-        action='store_false', dest='InteractiveShell.term_title', default=NoConfigDefault,
+    (('--no-term-title',), dict(
+        action='store_false', dest='InteractiveShell.term_title',
         help="Disable auto setting the terminal title.")
     ),
-    (('-xmode',), dict(
-        type=str, dest='InteractiveShell.xmode', default=NoConfigDefault,
-        help="Exception mode ('Plain','Context','Verbose')",
+    (('--xmode',), dict(
+        type=str, dest='InteractiveShell.xmode',
+        help=
+        """Exception reporting mode ('Plain','Context','Verbose').  Plain:
+        similar to python's normal traceback printing.  Context: prints 5 lines
+        of context source code around each line in the traceback.  Verbose:
+        similar to Context, but additionally prints the variables currently
+        visible where the exception happened (shortening their strings if too
+        long).  This can potentially be very slow, if you happen to have a huge
+        data structure whose string representation is complex to compute.
+        Your computer may appear to freeze for a while with cpu usage at 100%%.
+        If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
+        it more than once).
+        """,
         metavar='InteractiveShell.xmode')
     ),
-    (('-ext',), dict(
-        type=str, dest='Global.extra_extension', default=NoConfigDefault,
+    (('--ext',), dict(
+        type=str, dest='Global.extra_extension',
         help="The dotted module name of an IPython extension to load.",
         metavar='Global.extra_extension')
     ),
     (('-c',), dict(
-        type=str, dest='Global.code_to_run', default=NoConfigDefault,
+        type=str, dest='Global.code_to_run',
         help="Execute the given command string.",
         metavar='Global.code_to_run')
     ),
     (('-i',), dict(
-        action='store_true', dest='Global.force_interact', default=NoConfigDefault,
-        help="If running code from the command line, become interactive afterwards.")
+        action='store_true', dest='Global.force_interact',
+        help=
+        "If running code from the command line, become interactive afterwards."
+        )
+    ),
+
+    # Options to start with GUI control enabled from the beginning
+    (('--gui',), dict(
+        type=str, dest='Global.gui',
+        help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
+        metavar='gui-mode')
     ),
-    (('-wthread',), dict(
-        action='store_true', dest='Global.wthread', default=NoConfigDefault,
-        help="Enable wxPython event loop integration.")
+
+    (('--pylab','-pylab'), dict(
+        type=str, dest='Global.pylab',
+        nargs='?', const='auto', metavar='gui-mode',
+        help="Pre-load matplotlib and numpy for interactive use. "+
+        "If no value is given, the gui backend is matplotlib's, else use "+
+        "one of:  ['tk', 'qt', 'wx', 'gtk'].")
     ),
-    (('-q4thread','-qthread'), dict(
-        action='store_true', dest='Global.q4thread', default=NoConfigDefault,
-        help="Enable Qt4 event loop integration. Qt3 is no longer supported.")
+    
+    # Legacy GUI options.  Leave them in for backwards compatibility, but the
+    # 'thread' names are really a misnomer now.
+    (('--wthread','-wthread'), dict(
+        action='store_true', dest='Global.wthread',
+        help="Enable wxPython event loop integration "+
+             "(DEPRECATED, use --gui wx)")
     ),
-    (('-gthread',), dict(
-        action='store_true', dest='Global.gthread', default=NoConfigDefault,
-        help="Enable GTK event loop integration.")
+    (('--q4thread','--qthread','-q4thread','-qthread'), dict(
+        action='store_true', dest='Global.q4thread',
+        help="Enable Qt4 event loop integration. Qt3 is no longer supported. "+
+             "(DEPRECATED, use --gui qt)")
+    ),
+    (('--gthread','-gthread'), dict(
+        action='store_true', dest='Global.gthread',
+        help="Enable GTK event loop integration. "+
+             "(DEPRECATED, use --gui gtk)")
     ),
-    # # These are only here to get the proper deprecation warnings
-    (('-pylab',), dict(
-        action='store_true', dest='Global.pylab', default=NoConfigDefault,
-        help="Disabled.  Pylab has been disabled until matplotlib supports this version of IPython.")
-    )
 )
 
+#-----------------------------------------------------------------------------
+# Main classes and functions
+#-----------------------------------------------------------------------------
 
-class IPythonAppCLConfigLoader(IPythonArgParseConfigLoader):
+class IPythonApp(Application):
+    name = u'ipython'
+    #: argparse formats better the 'usage' than the 'description' field
+    description = None
+    #: usage message printed by argparse. If None, auto-generate
+    usage = usage.cl_usage
 
-    arguments = cl_args
+    config_file_name = default_config_file_name
 
+    cl_arguments = Application.cl_arguments + cl_args
 
-_default_config_file_name = 'ipython_config.py'
+    # Private and configuration attributes
+    _CrashHandler = crashhandler.IPythonCrashHandler
 
-class IPythonApp(Application):
-    name = 'ipython'
-    config_file_name = _default_config_file_name
+    def __init__(self, argv=None,
+                 constructor_config=None, override_config=None,
+                 **shell_params):
+        """Create a new IPythonApp.
+
+        See the parent class for details on how configuration is handled.
+
+        Parameters
+        ----------
+        argv : optional, list
+          If given, used as the command-line argv environment to read arguments
+          from.
+
+        constructor_config : optional, Config
+          If given, additional config that is merged last, after internal
+          defaults, command-line and file-based configs.
+
+        override_config : optional, Config
+          If given, config that overrides all others unconditionally (except
+          for internal defaults, which ensure that all parameters exist).
+
+        shell_params : optional, dict
+          All other keywords are passed to the :class:`iplib.InteractiveShell`
+          constructor. 
+        """
+        super(IPythonApp, self).__init__(argv, constructor_config,
+                                         override_config)
+        self.shell_params = shell_params
 
     def create_default_config(self):
         super(IPythonApp, self).create_default_config()
-        self.default_config.Global.display_banner = True
+        # Eliminate multiple lookups
+        Global = self.default_config.Global
+
+        # Set all default values
+        Global.display_banner = True
         
         # If the -c flag is given or a file is given to run at the cmd line
         # like "ipython foo.py", normally we exit without starting the main
         # loop.  The force_interact config variable allows a user to override
         # this and interact.  It is also set by the -i cmd line flag, just
         # like Python.
-        self.default_config.Global.force_interact = False
+        Global.force_interact = False
 
         # By default always interact by starting the IPython mainloop.
-        self.default_config.Global.interact = True
-
-        # Let the parent class set the default, but each time log_level
-        # changes from config, we need to update self.log_level as that is
-        # what updates the actual log level in self.log.
-        self.default_config.Global.log_level = self.log_level
+        Global.interact = True
 
         # No GUI integration by default
-        self.default_config.Global.wthread = False
-        self.default_config.Global.q4thread = False
-        self.default_config.Global.gthread = False
-
-    def create_command_line_config(self):
-        """Create and return a command line config loader."""
-        return IPythonAppCLConfigLoader(
-            description=ipython_desc,
-            version=release.version)
-
-    def post_load_command_line_config(self):
-        """Do actions after loading cl config."""
-        clc = self.command_line_config
-
-        # Display the deprecation warnings about threaded shells
-        if hasattr(clc.Global, 'pylab'):
-            pylab_warning()
-            del clc.Global['pylab']
+        Global.gui = False
+        # Pylab off by default
+        Global.pylab = False
+
+        # Deprecated versions of gui support that used threading, we support
+        # them just for bacwards compatibility as an alternate spelling for
+        # '--gui X'
+        Global.qthread = False
+        Global.q4thread = False
+        Global.wthread = False
+        Global.gthread = False
 
     def load_file_config(self):
         if hasattr(self.command_line_config.Global, 'quick'):
@@ -379,8 +462,7 b' class IPythonApp(Application):'
         # unless the -i flag (Global.force_interact) is true.
         code_to_run = config.Global.get('code_to_run','')
         file_to_run = False
-        if len(self.extra_args)>=1:
-            if self.extra_args[0]:
+        if self.extra_args and self.extra_args[0]:
                 file_to_run = True
         if file_to_run or code_to_run:
             if not config.Global.force_interact:
@@ -392,10 +474,8 b' class IPythonApp(Application):'
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance
-        self.shell = InteractiveShell(
-            parent=None,
-            config=self.master_config
-        )
+        self.shell = InteractiveShell(None, self.master_config,
+                                      **self.shell_params )
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
@@ -414,26 +494,49 b' class IPythonApp(Application):'
         if self.log_level <= logging.INFO: print
 
         # Now a variety of things that happen after the banner is printed.
-        self._enable_gui()
+        self._enable_gui_pylab()
         self._load_extensions()
         self._run_exec_lines()
         self._run_exec_files()
         self._run_cmd_line_code()
+        self._configure_xmode()
+
+    def _enable_gui_pylab(self):
+        """Enable GUI event loop integration, taking pylab into account."""
+        Global = self.master_config.Global
+
+        # Select which gui to use
+        if Global.gui:
+            gui = Global.gui
+        # The following are deprecated, but there's likely to be a lot of use
+        # of this form out there, so we might as well support it for now.  But
+        # the --gui option above takes precedence.
+        elif Global.wthread:
+            gui = inputhook.GUI_WX
+        elif Global.qthread:
+            gui = inputhook.GUI_QT
+        elif Global.gthread:
+            gui = inputhook.GUI_GTK
+        else:
+            gui = None
+
+        # Using --pylab will also require gui activation, though which toolkit
+        # to use may be chosen automatically based on mpl configuration.
+        if Global.pylab:
+            activate = self.shell.enable_pylab
+            if Global.pylab == 'auto':
+                gui = None
+            else:
+                gui = Global.pylab
+        else:
+            # Enable only GUI integration, no pylab
+            activate = inputhook.enable_gui
 
-    def _enable_gui(self):
-        """Enable GUI event loop integration."""
-        config = self.master_config
+        if gui or Global.pylab:
         try:
-            # Enable GUI integration
-            if config.Global.wthread:
-                self.log.info("Enabling wx GUI event loop integration")
-                inputhook.enable_wx(app=True)
-            elif config.Global.q4thread:
-                self.log.info("Enabling Qt4 GUI event loop integration")
-                inputhook.enable_qt4(app=True)
-            elif config.Global.gthread:
-                self.log.info("Enabling GTK GUI event loop integration")
-                inputhook.enable_gtk(app=True)
+                self.log.info("Enabling GUI event loop integration, "
+                              "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
+                activate(gui)
         except:
             self.log.warn("Error in enabling GUI event loop integration:")
             self.shell.showtraceback()
@@ -477,9 +580,9 b' class IPythonApp(Application):'
             self.shell.showtraceback()
 
     def _exec_file(self, fname):
-        full_filename = filefind(fname, ['.', self.ipythondir])
+        full_filename = filefind(fname, [u'.', self.ipython_dir])
         if os.path.isfile(full_filename):
-            if full_filename.endswith('.py'):
+            if full_filename.endswith(u'.py'):
                 self.log.info("Running file in user namespace: %s" % full_filename)
                 self.shell.safe_execfile(full_filename, self.shell.user_ns)
             elif full_filename.endswith('.ipy'):
@@ -521,26 +624,32 b' class IPythonApp(Application):'
                 self.log.warn("Error in executing file in user namespace: %s" % fname)
                 self.shell.showtraceback()
 
+    def _configure_xmode(self):
+        # XXX - shouldn't this be read from the config?  I'm still a little
+        # lost with all the details of handling the new config guys...
+        self.shell.InteractiveTB.set_mode(mode=self.shell.xmode)
+        
     def start_app(self):
         if self.master_config.Global.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
+        else:
+            self.log.debug("IPython not interactive, start_app is no-op...")
 
 
-def load_default_config(ipythondir=None):
-    """Load the default config file from the default ipythondir.
+def load_default_config(ipython_dir=None):
+    """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
-    if ipythondir is None:
-        ipythondir = get_ipython_dir()
-    cl = PyFileConfigLoader(_default_config_file_name, ipythondir)
+    if ipython_dir is None:
+        ipython_dir = get_ipython_dir()
+    cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
     config = cl.load_config()
     return config
 
 
 def launch_new_instance():
-    """Create a run a full blown IPython instance"""
+    """Create and run a full blown IPython instance"""
     app = IPythonApp()
     app.start()
-

IPython/core/iplib.py

@@ -17,6 +17,7 b' Main IPython Component'
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
+from __future__ import absolute_import
 
 import __builtin__
 import StringIO
@@ -31,34 +32,37 b' import sys'
 import tempfile
 from contextlib import nested
 
-from IPython.core import ultratb
 from IPython.core import debugger, oinspect
-from IPython.core import shadowns
 from IPython.core import history as ipcorehist
 from IPython.core import prefilter
+from IPython.core import shadowns
+from IPython.core import ultratb
 from IPython.core.alias import AliasManager
 from IPython.core.builtin_trap import BuiltinTrap
+from IPython.core.component import Component
 from IPython.core.display_trap import DisplayTrap
+from IPython.core.error import TryNext, UsageError
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
-from IPython.core.prompts import CachedOutput
 from IPython.core.prefilter import PrefilterManager
-from IPython.core.component import Component
+from IPython.core.prompts import CachedOutput
+from IPython.core.pylabtools import pylab_activate
 from IPython.core.usage import interactive_usage, default_banner
-from IPython.core.error import TryNext, UsageError
-
-from IPython.utils import pickleshare
 from IPython.external.Itpl import ItplNS
+from IPython.lib.inputhook import enable_gui
 from IPython.lib.backgroundjobs import BackgroundJobManager
-from IPython.utils.ipstruct import Struct
 from IPython.utils import PyColorize
-from IPython.utils.genutils import *
+from IPython.utils import pickleshare
 from IPython.utils.genutils import get_ipython_dir
+from IPython.utils.ipstruct import Struct
 from IPython.utils.platutils import toggle_set_term_title, set_term_title
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 
+# XXX - need to clean up this import * line
+from IPython.utils.genutils import *
+
 # from IPython.utils import growl
 # growl.start("IPython")
 
@@ -70,7 +74,6 b' from IPython.utils.traitlets import ('
 # Globals
 #-----------------------------------------------------------------------------
 
-
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
@@ -78,12 +81,10 b' raw_input_original = raw_input'
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
-
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
-
 ini_spaces_re = re.compile(r'^(\s+)')
 
 
@@ -113,6 +114,8 b' def softspace(file, newvalue):'
     return oldvalue
 
 
+def no_op(*a, **kw): pass
+
 class SpaceInInput(exceptions.Exception): pass
 
 class Bunch: pass
@@ -162,6 +165,15 b' def get_default_editor():'
     return ed
 
 
+def get_default_colors():
+    if sys.platform=='darwin':
+        return "LightBG"
+    elif os.name=='nt':
+        return 'Linux'
+    else:
+        return 'Linux'
+
+
 class SeparateStr(Str):
     """A Str subclass to validate separate_in, separate_out, etc.
 
@@ -174,6 +186,57 b' class SeparateStr(Str):'
         return super(SeparateStr, self).validate(obj, value)
 
 
+def make_user_namespaces(user_ns=None, user_global_ns=None):
+    """Return a valid local and global user interactive namespaces.
+
+    This builds a dict with the minimal information needed to operate as a
+    valid IPython user namespace, which you can pass to the various
+    embedding classes in ipython. The default implementation returns the
+    same dict for both the locals and the globals to allow functions to
+    refer to variables in the namespace. Customized implementations can
+    return different dicts. The locals dictionary can actually be anything
+    following the basic mapping protocol of a dict, but the globals dict
+    must be a true dict, not even a subclass. It is recommended that any
+    custom object for the locals namespace synchronize with the globals
+    dict somehow.
+
+    Raises TypeError if the provided globals namespace is not a true dict.
+
+    Parameters
+    ----------
+    user_ns : dict-like, optional
+        The current user namespace. The items in this namespace should
+        be included in the output. If None, an appropriate blank
+        namespace should be created.
+    user_global_ns : dict, optional
+        The current user global namespace. The items in this namespace
+        should be included in the output. If None, an appropriate
+        blank namespace should be created.
+
+    Returns
+    -------
+        A pair of dictionary-like object to be used as the local namespace
+        of the interpreter and a dict to be used as the global namespace.
+    """
+
+    if user_ns is None:
+        # Set __name__ to __main__ to better match the behavior of the
+        # normal interpreter.
+        user_ns = {'__name__'     :'__main__',
+                   '__builtins__' : __builtin__,
+                  }
+    else:
+        user_ns.setdefault('__name__','__main__')
+        user_ns.setdefault('__builtins__',__builtin__)
+
+    if user_global_ns is None:
+        user_global_ns = user_ns
+    if type(user_global_ns) is not dict:
+        raise TypeError("user_global_ns must be a true dict; got %r"
+            % type(user_global_ns))
+
+    return user_ns, user_global_ns
+
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
@@ -182,7 +245,7 b' class SeparateStr(Str):'
 class InteractiveShell(Component, Magic):
     """An enhanced, interactive shell for Python."""
 
-    autocall = Enum((0,1,2), config=True)
+    autocall = Enum((0,1,2), default_value=1, config=True)
     autoedit_syntax = CBool(False, config=True)
     autoindent = CBool(True, config=True)
     automagic = CBool(True, config=True)
@@ -192,7 +255,7 b' class InteractiveShell(Component, Magic):'
     cache_size = Int(1000, config=True)
     color_info = CBool(True, config=True)
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'), 
-                             default_value='LightBG', config=True)
+                             default_value=get_default_colors(), config=True)
     confirm_exit = CBool(True, config=True)
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True)
@@ -206,7 +269,7 b' class InteractiveShell(Component, Magic):'
     embedded_active = CBool(False)
     editor = Str(get_default_editor(), config=True)
     filename = Str("<ipython console>")
-    ipythondir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
+    ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
     logstart = CBool(False, config=True)
     logfile = Str('', config=True)
     logappend = Str('', config=True)
@@ -264,7 +327,7 b' class InteractiveShell(Component, Magic):'
     # Subclasses with thread support should override this as needed.
     isthreaded = False
 
-    def __init__(self, parent=None, config=None, ipythondir=None, usage=None,
+    def __init__(self, parent=None, config=None, ipython_dir=None, usage=None,
                  user_ns=None, user_global_ns=None,
                  banner1=None, banner2=None, display_banner=None,
                  custom_exceptions=((),None)):
@@ -274,7 +337,7 b' class InteractiveShell(Component, Magic):'
         super(InteractiveShell, self).__init__(parent, config=config)
 
         # These are relatively independent and stateless
-        self.init_ipythondir(ipythondir)
+        self.init_ipython_dir(ipython_dir)
         self.init_instance_attrs()
         self.init_term_title()
         self.init_usage(usage)
@@ -320,6 +383,7 b' class InteractiveShell(Component, Magic):'
         self.hooks.late_startup_hook()
 
     def get_ipython(self):
+        """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
@@ -332,7 +396,7 b' class InteractiveShell(Component, Magic):'
     def _banner2_changed(self):
         self.compute_banner()
 
-    def _ipythondir_changed(self, name, new):
+    def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0777)
         if not os.path.isdir(self.ipython_extension_dir):
@@ -340,7 +404,7 b' class InteractiveShell(Component, Magic):'
 
     @property
     def ipython_extension_dir(self):
-        return os.path.join(self.ipythondir, 'extensions')
+        return os.path.join(self.ipython_dir, 'extensions')
 
     @property
     def usable_screen_length(self):
@@ -372,19 +436,19 b' class InteractiveShell(Component, Magic):'
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
-    def init_ipythondir(self, ipythondir):
-        if ipythondir is not None:
-            self.ipythondir = ipythondir
-            self.config.Global.ipythondir = self.ipythondir
+    def init_ipython_dir(self, ipython_dir):
+        if ipython_dir is not None:
+            self.ipython_dir = ipython_dir
+            self.config.Global.ipython_dir = self.ipython_dir
             return
 
-        if hasattr(self.config.Global, 'ipythondir'):
-            self.ipythondir = self.config.Global.ipythondir
+        if hasattr(self.config.Global, 'ipython_dir'):
+            self.ipython_dir = self.config.Global.ipython_dir
         else:
-            self.ipythondir = get_ipython_dir()
+            self.ipython_dir = get_ipython_dir()
 
         # All children can just read this
-        self.config.Global.ipythondir = self.ipythondir
+        self.config.Global.ipython_dir = self.ipython_dir
 
     def init_instance_attrs(self):
         self.jobs = BackgroundJobManager()
@@ -805,8 +869,7 b' class InteractiveShell(Component, Magic):'
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
-        user_ns, user_global_ns = self.make_user_namespaces(user_ns,
-            user_global_ns)
+        user_ns, user_global_ns = make_user_namespaces(user_ns, user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
@@ -816,7 +879,7 b' class InteractiveShell(Component, Magic):'
         # An auxiliary namespace that checks what parts of the user_ns were
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
-        # doesn't need to be seaparately tracked in the ns_table
+        # doesn't need to be separately tracked in the ns_table.
         self.user_config_ns = {}
 
         # A namespace to keep track of internal data structures to prevent
@@ -891,55 +954,6 b' class InteractiveShell(Component, Magic):'
         else:
             sys.modules[main_name] = FakeModule(self.user_ns)
 
-    def make_user_namespaces(self, user_ns=None, user_global_ns=None):
-        """Return a valid local and global user interactive namespaces.
-
-        This builds a dict with the minimal information needed to operate as a
-        valid IPython user namespace, which you can pass to the various
-        embedding classes in ipython. The default implementation returns the
-        same dict for both the locals and the globals to allow functions to
-        refer to variables in the namespace. Customized implementations can
-        return different dicts. The locals dictionary can actually be anything
-        following the basic mapping protocol of a dict, but the globals dict
-        must be a true dict, not even a subclass. It is recommended that any
-        custom object for the locals namespace synchronize with the globals
-        dict somehow.
-
-        Raises TypeError if the provided globals namespace is not a true dict.
-
-        :Parameters:
-            user_ns : dict-like, optional
-                The current user namespace. The items in this namespace should
-                be included in the output. If None, an appropriate blank
-                namespace should be created.
-            user_global_ns : dict, optional
-                The current user global namespace. The items in this namespace
-                should be included in the output. If None, an appropriate
-                blank namespace should be created.
-
-        :Returns:
-            A tuple pair of dictionary-like object to be used as the local namespace
-            of the interpreter and a dict to be used as the global namespace.
-        """
-
-        if user_ns is None:
-            # Set __name__ to __main__ to better match the behavior of the
-            # normal interpreter.
-            user_ns = {'__name__'     :'__main__',
-                       '__builtins__' : __builtin__,
-                      }
-        else:
-            user_ns.setdefault('__name__','__main__')
-            user_ns.setdefault('__builtins__',__builtin__)
-
-        if user_global_ns is None:
-            user_global_ns = user_ns
-        if type(user_global_ns) is not dict:
-            raise TypeError("user_global_ns must be a true dict; got %r"
-                % type(user_global_ns))
-
-        return user_ns, user_global_ns
-
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
@@ -952,27 +966,44 b' class InteractiveShell(Component, Magic):'
         method.  If they were not empty before, data will simply be added to
         therm.
         """
-        # Store myself as the public api!!!
-        self.user_ns['get_ipython'] = self.get_ipython
-
-        # make global variables for user access to the histories
-        self.user_ns['_ih'] = self.input_hist
-        self.user_ns['_oh'] = self.output_hist
-        self.user_ns['_dh'] = self.dir_hist
-
-        # user aliases to input and output histories
-        self.user_ns['In']  = self.input_hist
-        self.user_ns['Out'] = self.output_hist
-
-        self.user_ns['_sh'] = shadowns
+        # This function works in two parts: first we put a few things in
+        # user_ns, and we sync that contents into user_config_ns so that these
+        # initial variables aren't shown by %who.  After the sync, we add the
+        # rest of what we *do* want the user to see with %who even on a new
+        # session.
+        ns = {}
 
         # Put 'help' in the user namespace
         try:
             from site import _Helper
-            self.user_ns['help'] = _Helper()
+            ns['help'] = _Helper()
         except ImportError:
             warn('help() not available - check site.py')
 
+        # make global variables for user access to the histories
+        ns['_ih'] = self.input_hist
+        ns['_oh'] = self.output_hist
+        ns['_dh'] = self.dir_hist
+
+        ns['_sh'] = shadowns
+
+        # Sync what we've added so far to user_config_ns so these aren't seen
+        # by %who
+        self.user_config_ns.update(ns)
+
+        # Now, continue adding more contents
+
+        # user aliases to input and output histories
+        ns['In']  = self.input_hist
+        ns['Out'] = self.output_hist
+
+        # Store myself as the public api!!!
+        ns['get_ipython'] = self.get_ipython
+        
+        # And update the real user's namespace
+        self.user_ns.update(ns)
+
+
     def reset(self):
         """Clear all internal namespaces.
 
@@ -1070,7 +1101,7 b' class InteractiveShell(Component, Magic):'
             histfname = 'history-%s' % self.profile
         else:
             histfname = 'history'
-        self.histfile = os.path.join(self.ipythondir, histfname)
+        self.histfile = os.path.join(self.ipython_dir, histfname)
 
         # Fill the history zero entry, user counter starts at 1
         self.input_hist.append('\n')
@@ -1078,21 +1109,18 b' class InteractiveShell(Component, Magic):'
 
     def init_shadow_hist(self):
         try:
-            self.db = pickleshare.PickleShareDB(self.ipythondir + "/db")
+            self.db = pickleshare.PickleShareDB(self.ipython_dir + "/db")
         except exceptions.UnicodeDecodeError:
-            print "Your ipythondir can't be decoded to unicode!"
+            print "Your ipython_dir can't be decoded to unicode!"
             print "Please set HOME environment variable to something that"
             print r"only has ASCII characters, e.g. c:\home"
-            print "Now it is", self.ipythondir
+            print "Now it is", self.ipython_dir
             sys.exit()
         self.shadowhist = ipcorehist.ShadowHist(self.db)
 
     def savehist(self):
         """Save input history to a file (via readline library)."""
 
-        if not self.has_readline:
-            return
-        
         try:
             self.readline.write_history_file(self.histfile)
         except:
@@ -1102,7 +1130,6 b' class InteractiveShell(Component, Magic):'
     def reloadhist(self):
         """Reload the input history from disk file."""
 
-        if self.has_readline:
             try:
                 self.readline.clear_history()
                 self.readline.read_history_file(self.shell.histfile)
@@ -1141,37 +1168,14 b' class InteractiveShell(Component, Magic):'
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
-        # IPython itself shouldn't crash. This will produce a detailed
-        # post-mortem if it does.  But we only install the crash handler for
-        # non-threaded shells, the threaded ones use a normal verbose reporter
-        # and lose the crash handler.  This is because exceptions in the main
-        # thread (such as in GUI code) propagate directly to sys.excepthook,
-        # and there's no point in printing crash dumps for every user exception.
-        if self.isthreaded:
-            ipCrashHandler = ultratb.FormattedTB()
-        else:
-            from IPython.core import crashhandler
-            ipCrashHandler = crashhandler.IPythonCrashHandler(self)
-        self.set_crash_handler(ipCrashHandler)
+        # The instance will store a pointer to the system-wide exception hook,
+        # so that runtime code (such as magics) can access it.  This is because
+        # during the read-eval loop, it may get temporarily overwritten.
+        self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
-    def set_crash_handler(self, crashHandler):
-        """Set the IPython crash handler.
-
-        This must be a callable with a signature suitable for use as
-        sys.excepthook."""
-
-        # Install the given crash handler as the Python exception hook
-        sys.excepthook = crashHandler
-        
-        # The instance will store a pointer to this, so that runtime code
-        # (such as magics) can access it.  This is because during the
-        # read-eval loop, it gets temporarily overwritten (to deal with GUI
-        # frameworks).
-        self.sys_excepthook = sys.excepthook
-
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
@@ -1239,7 +1243,8 b' class InteractiveShell(Component, Magic):'
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
-    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None):
+    def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
+                      exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
@@ -1251,17 +1256,23 b' class InteractiveShell(Component, Magic):'
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
-        
-        # Though this won't be called by syntax errors in the input line,
-        # there may be SyntaxError cases whith imported code.
-        
         try:
             if exc_tuple is None:
                 etype, value, tb = sys.exc_info()
             else:
                 etype, value, tb = exc_tuple
     
+            if etype is None:
+                if hasattr(sys, 'last_type'):
+                    etype, value, tb = sys.last_type, sys.last_value, \
+                                       sys.last_traceback
+                else:
+                    self.write('No traceback available to show.\n')
+                    return
+    
             if etype is SyntaxError:
+                # Though this won't be called by syntax errors in the input
+                # line, there may be SyntaxError cases whith imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 print "UsageError:", value
@@ -1277,13 +1288,21 b' class InteractiveShell(Component, Magic):'
                 if etype in self.custom_exceptions:
                     self.CustomTB(etype,value,tb)
                 else:
+                    if exception_only:
+                        m = ('An exception has occurred, use %tb to see the '
+                             'full traceback.')
+                        print m
+                        self.InteractiveTB.show_exception_only(etype, value)
+                    else:
                     self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
-                    if self.InteractiveTB.call_pdb and self.has_readline:
+                        if self.InteractiveTB.call_pdb:
                         # pdb mucks up readline, fix it back
                         self.set_completer()
+                        
         except KeyboardInterrupt:
             self.write("\nKeyboardInterrupt\n")
 
+
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
@@ -1295,7 +1314,7 b' class InteractiveShell(Component, Magic):'
         """
         etype, value, last_traceback = sys.exc_info()
 
-        # See note about these variables in showtraceback() below
+        # See note about these variables in showtraceback() above
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = last_traceback
@@ -1426,9 +1445,7 b' class InteractiveShell(Component, Magic):'
             return outcomps
 
     def set_custom_completer(self,completer,pos=0):
-        """set_custom_completer(completer,pos=0)
-
-        Adds a new custom completer function.
+        """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
@@ -1438,9 +1455,18 b' class InteractiveShell(Component, Magic):'
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_completer(self):
-        """reset readline's completer to be our own."""
+        """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.complete)
 
+    def set_completer_frame(self, frame=None):
+        """Set the frame of the completer."""
+        if frame:
+            self.Completer.namespace = frame.f_locals
+            self.Completer.global_namespace = frame.f_globals
+        else:
+            self.Completer.namespace = self.user_ns
+            self.Completer.global_namespace = self.user_global_ns
+
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
@@ -1448,20 +1474,25 b' class InteractiveShell(Component, Magic):'
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
+        if self.readline_use:
+            import IPython.utils.rlineimpl as readline
+                  
         self.rl_next_input = None
         self.rl_do_indent = False
 
-        if not self.readline_use:
-            return
-
-        import IPython.utils.rlineimpl as readline
-                  
-        if not readline.have_readline:
-            self.has_readline = 0
+        if not self.readline_use or not readline.have_readline:
+            self.has_readline = False
             self.readline = None
-            # no point in bugging windows users with this every time:
-            warn('Readline services not available on this platform.')
+            # Set a number of methods that depend on readline to be no-op
+            self.savehist = no_op
+            self.reloadhist = no_op
+            self.set_completer = no_op
+            self.set_custom_completer = no_op
+            self.set_completer_frame = no_op
+            warn('Readline services not available or not loaded.')
         else:
+            self.has_readline = True
+            self.readline = readline
             sys.modules['readline'] = readline
             import atexit
             from IPython.core.completer import IPCompleter
@@ -1496,8 +1527,6 b' class InteractiveShell(Component, Magic):'
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
-            self.has_readline = 1
-            self.readline = readline
             # save this in sys so embedded copies can restore it properly
             sys.ipcompleter = self.Completer.complete
             self.set_completer()
@@ -1569,6 +1598,9 b' class InteractiveShell(Component, Magic):'
         # Set user colors (don't do it in the constructor above so that it
         # doesn't crash if colors option is invalid)
         self.magic_colors(self.colors)
+        # History was moved to a separate module
+        from . import history
+        history.init_ipython(self)
 
     def magic(self,arg_s):
         """Call a magic function by name.
@@ -1587,7 +1619,6 b' class InteractiveShell(Component, Magic):'
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
-
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
@@ -1826,6 +1857,7 b' class InteractiveShell(Component, Magic):'
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
+                    if self.has_readline:
                     self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
@@ -1847,6 +1879,9 b' class InteractiveShell(Component, Magic):'
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
+        # Turn off the exit flag, so the mainloop can be restarted if desired
+        self.exit_now = False
+
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
@@ -1862,7 +1897,8 b' class InteractiveShell(Component, Magic):'
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
-            If True, then don't print errors for non-zero exit statuses.
+            If True, then silence SystemExit for non-zero status (it is always
+            silenced for zero status, as it is so common).
         """
         kw.setdefault('exit_ignore', False)
 
@@ -1887,40 +1923,21 b' class InteractiveShell(Component, Magic):'
 
         with prepended_to_syspath(dname):
             try:
-                if sys.platform == 'win32' and sys.version_info < (2,5,1):
-                    # Work around a bug in Python for Windows.  The bug was
-                    # fixed in in Python 2.5 r54159 and 54158, but that's still
-                    # SVN Python as of March/07.  For details, see:
-                    # http://projects.scipy.org/ipython/ipython/ticket/123
-                    try:
-                        globs,locs = where[0:2]
-                    except:
-                        try:
-                            globs = locs = where[0]
-                        except:
-                            globs = locs = globals()
-                    exec file(fname) in globs,locs
-                else:
                     execfile(fname,*where)
-            except SyntaxError:
-                self.showsyntaxerror()
-                warn('Failure executing file: <%s>' % fname)
             except SystemExit, status:
-                # Code that correctly sets the exit status flag to success (0)
-                # shouldn't be bothered with a traceback.  Note that a plain
-                # sys.exit() does NOT set the message to 0 (it's empty) so that
-                # will still get a traceback.  Note that the structure of the
-                # SystemExit exception changed between Python 2.4 and 2.5, so
-                # the checks must be done in a version-dependent way.
-                show = False
-                if status.message!=0 and not kw['exit_ignore']:
-                    show = True
-                if show:
-                    self.showtraceback()
-                    warn('Failure executing file: <%s>' % fname)
+                # If the call was made with 0 or None exit status (sys.exit(0)
+                # or sys.exit() ), don't bother showing a traceback, as both of
+                # these are considered normal by the OS:
+                # > python -c'import sys;sys.exit(0)'; echo $?
+                # 0
+                # > python -c'import sys;sys.exit()'; echo $?
+                # 0
+                # For other exit status, we show the exception unless
+                # explicitly silenced, but only in short form.
+                if status.code not in (0, None) and not kw['exit_ignore']:
+                    self.showtraceback(exception_only=True)
             except:
                 self.showtraceback()
-                warn('Failure executing file: <%s>' % fname)
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
@@ -2134,9 +2151,8 b' class InteractiveShell(Component, Magic):'
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
-            self.showtraceback()
-            warn("Type %exit or %quit to exit IPython "
-                 "(%Exit or %Quit do so unconditionally).",level=1)
+            self.showtraceback(exception_only=True)
+            warn("To exit: use any of 'exit', 'quit', %Exit or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
@@ -2278,6 +2294,8 b' class InteractiveShell(Component, Magic):'
     def get_component(self, name=None, klass=None):
         """Fetch a component by name and klass in my tree."""
         c = Component.get_instances(root=self, name=name, klass=klass)
+        if len(c) == 0:
+            return None
         if len(c) == 1:
             return c[0]
         else:
@@ -2309,7 +2327,7 b' class InteractiveShell(Component, Magic):'
         You can put your extension modules anywhere you want, as long as
         they can be imported by Python's standard import mechanism.  However,
         to make it easy to write extensions, you can also put your extensions
-        in ``os.path.join(self.ipythondir, 'extensions')``.  This directory
+        in ``os.path.join(self.ipython_dir, 'extensions')``.  This directory
         is added to ``sys.path`` automatically.
         """
         from IPython.utils.syspathcontext import prepended_to_syspath
@@ -2318,7 +2336,7 b' class InteractiveShell(Component, Magic):'
             with prepended_to_syspath(self.ipython_extension_dir):
                 __import__(module_str)
         mod = sys.modules[module_str]
-        self._call_load_ipython_extension(mod)
+        return self._call_load_ipython_extension(mod)
 
     def unload_extension(self, module_str):
         """Unload an IPython extension by its module name.
@@ -2350,11 +2368,11 b' class InteractiveShell(Component, Magic):'
 
     def _call_load_ipython_extension(self, mod):
         if hasattr(mod, 'load_ipython_extension'):
-            mod.load_ipython_extension(self)
+            return mod.load_ipython_extension(self)
 
     def _call_unload_ipython_extension(self, mod):
         if hasattr(mod, 'unload_ipython_extension'):
-            mod.unload_ipython_extension(self)
+            return mod.unload_ipython_extension(self)
 
     #-------------------------------------------------------------------------
     # Things related to the prefilter
@@ -2362,6 +2380,10 b' class InteractiveShell(Component, Magic):'
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(self, config=self.config)
+        # Ultimately this will be refactored in the new interpreter code, but
+        # for now, we should expose the main prefilter method (there's legacy
+        # code out there that may rely on this).
+        self.prefilter = self.prefilter_manager.prefilter_lines
 
     #-------------------------------------------------------------------------
     # Utilities
@@ -2427,11 +2449,46 b' class InteractiveShell(Component, Magic):'
         return ask_yes_no(prompt,default)
 
     #-------------------------------------------------------------------------
+    # Things related to GUI support and pylab
+    #-------------------------------------------------------------------------
+
+    def enable_pylab(self, gui=None):
+        """Activate pylab support at runtime.
+
+        This turns on support for matplotlib, preloads into the interactive
+        namespace all of numpy and pylab, and configures IPython to correcdtly
+        interact with the GUI event loop.  The GUI backend to be used can be
+        optionally selected with the optional :param:`gui` argument.
+
+        Parameters
+        ----------
+        gui : optional, string
+
+          If given, dictates the choice of matplotlib GUI backend to use
+          (should be one of IPython's supported backends, 'tk', 'qt', 'wx' or
+          'gtk'), otherwise we use the default chosen by matplotlib (as
+          dictated by the matplotlib build-time options plus the user's
+          matplotlibrc configuration file).
+        """
+        # We want to prevent the loading of pylab to pollute the user's
+        # namespace as shown by the %who* magics, so we execute the activation
+        # code in an empty namespace, and we update *both* user_ns and
+        # user_config_ns with this information.
+        ns = {}
+        gui = pylab_activate(ns, gui)
+        self.user_ns.update(ns)
+        self.user_config_ns.update(ns)
+        # Now we must activate the gui pylab wants to use, and fix %run to take
+        # plot updates into account
+        enable_gui(gui)
+        self.magic_run = self._pylab_magic_run
+
+    #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
 
     def ask_exit(self):
-        """ Call for exiting. Can be overiden and used as a callback. """
+        """ Ask the shell to exit. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):

IPython/core/magic.py

@@ -21,6 +21,7 b' import os'
 import pdb
 import pydoc
 import sys
+import shutil
 import re
 import tempfile
 import time
@@ -43,21 +44,26 b' except ImportError:'
 
 # Homebrewed
 import IPython
-from IPython.utils import wildcard
+import IPython.utils.generics
+
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
+from IPython.core.error import UsageError
 from IPython.core.fakemodule import FakeModule
+from IPython.core.macro import Macro
+from IPython.core.page import page
 from IPython.core.prefilter import ESC_MAGIC
+from IPython.core.pylabtools import mpl_runner
+from IPython.lib.inputhook import enable_gui
 from IPython.external.Itpl import Itpl, itpl, printpl,itplns
+from IPython.testing import decorators as testdec
+from IPython.utils import platutils
+from IPython.utils import wildcard
 from IPython.utils.PyColorize import Parser
 from IPython.utils.ipstruct import Struct
-from IPython.core.macro import Macro
+
+# XXX - We need to switch to explicit imports here with genutils
 from IPython.utils.genutils import *
-from IPython.core.page import page
-from IPython.utils import platutils
-import IPython.utils.generics
-from IPython.core.error import UsageError
-from IPython.testing import decorators as testdec
 
 #***************************************************************************
 # Utility functions
@@ -83,6 +89,12 b' def compress_dhist(dh):'
 
 #***************************************************************************
 # Main class implementing Magic functionality
+
+# XXX - for some odd reason, if Magic is made a new-style class, we get errors
+# on construction of the main InteractiveShell object.  Something odd is going
+# on with super() calls, Component and the MRO... For now leave it as-is, but
+# eventually this needs to be clarified.
+
 class Magic:
     """Magic functions for InteractiveShell.
 
@@ -334,7 +346,7 b' python-profiler package from non-free.""")'
             raise ValueError,'incorrect mode given: %s' % mode
         # Get options
         list_all = kw.get('list_all',0)
-        posix = kw.get('posix',True)
+        posix = kw.get('posix', os.name == 'posix')
 
         # Check if we have more than one argument to warrant extra processing:
         odict = {}  # Dictionary with options
@@ -873,17 +885,15 b' Currently the magic system has the following functions:\\n"""'
         user_ns = self.shell.user_ns
         internal_ns = self.shell.internal_ns
         user_config_ns = self.shell.user_config_ns
-        out = []
-        typelist = parameter_s.split()
+        out = [ i for i in user_ns
+                if not i.startswith('_') \
+                and not (i in internal_ns or i in user_config_ns) ]
 
-        for i in user_ns:
-            if not (i.startswith('_') or i.startswith('_i')) \
-                   and not (i in internal_ns or i in user_config_ns):
+        typelist = parameter_s.split()
                 if typelist:
-                    if type(user_ns[i]).__name__ in typelist:
-                        out.append(i)
-                else:
-                    out.append(i)
+            typeset = set(typelist)
+            out = [i for i in out if type(i).__name__ in typeset]
+
         out.sort()
         return out
         
@@ -1268,7 +1278,6 b' Currently the magic system has the following functions:\\n"""'
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
-        
         self.shell.debugger(force=True)
 
     @testdec.skip_doctest
@@ -1571,7 +1580,7 b' Currently the magic system has the following functions:\\n"""'
             return
 
         if filename.lower().endswith('.ipy'):
-            self.safe_execfile_ipy(filename)
+            self.shell.safe_execfile_ipy(filename)
             return
         
         # Control the response to exit() calls made by the script being run
@@ -2523,19 +2532,6 b' Defaulting color scheme to \'NoColor\'"""'
         print 'Pretty printing has been turned', \
               ['OFF','ON'][self.shell.pprint]
         
-    def magic_exit(self, parameter_s=''):
-        """Exit IPython, confirming if configured to do so.
-
-        You can configure whether IPython asks for confirmation upon exit by
-        setting the confirm_exit flag in the ipythonrc file."""
-
-        self.shell.exit()
-
-    def magic_quit(self, parameter_s=''):
-        """Exit IPython, confirming if configured to do so (like %exit)"""
-
-        self.shell.exit()
-        
     def magic_Exit(self, parameter_s=''):
         """Exit IPython without confirmation."""
 
@@ -2685,11 +2681,12 b' Defaulting color scheme to \'NoColor\'"""'
                             else:
                                 syscmdlist.append(ff)
             else:
+                no_alias = self.shell.alias_manager.no_alias
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
-                        if isexec(ff) and base.lower() not in self.shell.no_alias:
+                        if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
@@ -3365,7 +3362,7 b' Defaulting color scheme to \'NoColor\'"""'
         # By default, echo back to terminal unless quiet mode is requested
         if not opts.has_key('q'):
             write = self.shell.write
-            write(block)
+            write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
@@ -3379,34 +3376,6 b' Defaulting color scheme to \'NoColor\'"""'
         
         page(qr)
         
-    def magic_upgrade(self,arg):
-        """ Upgrade your IPython installation
-        
-        This will copy the config files that don't yet exist in your 
-        ipython dir from the system config dir. Use this after upgrading 
-        IPython if you don't wish to delete your .ipython dir.
-
-        Call with -nolegacy to get rid of ipythonrc* files (recommended for
-        new users)
-
-        """
-        ip = self.getapi()
-        ipinstallation = path(IPython.__file__).dirname()
-        upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'utils' / 'upgradedir.py')
-        src_config = ipinstallation / 'config' / 'userconfig'
-        userdir = path(ip.config.IPYTHONDIR)
-        cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
-        print ">",cmd
-        shell(cmd)
-        if arg == '-nolegacy':
-            legacy = userdir.files('ipythonrc*')
-            print "Nuking legacy files:",legacy
-            
-            [p.remove() for p in legacy]
-            suffix = (sys.platform == 'win32' and '.ini' or '')
-            (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
-
-
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
@@ -3427,8 +3396,6 b' Defaulting color scheme to \'NoColor\'"""'
         your existing IPython session.
         """
 
-        # XXX - Fix this to have cleaner activate/deactivate calls.
-        from IPython.extensions import InterpreterPasteInput as ipaste
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
@@ -3451,8 +3418,6 b' Defaulting color scheme to \'NoColor\'"""'
 
         if mode == False:
             # turn on
-            ipaste.activate_prefilter()
-
             oc.prompt1.p_template = '>>> '
             oc.prompt2.p_template = '... '
             oc.prompt_out.p_template = ''
@@ -3471,8 +3436,6 b' Defaulting color scheme to \'NoColor\'"""'
 
         else:
             # turn off
-            ipaste.deactivate_prefilter()
-
             oc.prompt1.p_template = shell.prompt_in1
             oc.prompt2.p_template = shell.prompt_in2
             oc.prompt_out.p_template = shell.prompt_out
@@ -3485,7 +3448,7 b' Defaulting color scheme to \'NoColor\'"""'
             oc.prompt1.pad_left = oc.prompt2.pad_left = \
                          oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
 
-            rc.pprint = dstore.rc_pprint
+            shell.pprint = dstore.rc_pprint
 
             shell.magic_xmode(dstore.xmode)
 
@@ -3503,7 +3466,7 b' Defaulting color scheme to \'NoColor\'"""'
         using the (pylab/wthread/etc.) command line flags.  GUI toolkits
         can now be enabled, disabled and swtiched at runtime and keyboard
         interrupts should work without any problems.  The following toolkits
-        are supports:  wxPython, PyQt4, PyGTK, and Tk::
+        are supported:  wxPython, PyQt4, PyGTK, and Tk::
 
             %gui wx      # enable wxPython event loop integration
             %gui qt4|qt  # enable PyQt4 event loop integration
@@ -3522,25 +3485,13 b' Defaulting color scheme to \'NoColor\'"""'
 
         This is highly recommended for most users.
         """
-        from IPython.lib import inputhook
-        if "-a" in parameter_s:
-            app = True
-        else:
-            app = False
-        if not parameter_s:
-            inputhook.clear_inputhook()
-        elif 'wx' in parameter_s:
-            return inputhook.enable_wx(app)
-        elif ('qt4' in parameter_s) or ('qt' in parameter_s):
-            return inputhook.enable_qt4(app)
-        elif 'gtk' in parameter_s:
-            return inputhook.enable_gtk(app)
-        elif 'tk' in parameter_s:
-            return inputhook.enable_tk(app)
+        opts, arg = self.parse_options(parameter_s,'a')
+        if arg=='': arg = None
+        return enable_gui(arg, 'a' in opts)
 
     def magic_load_ext(self, module_str):
         """Load an IPython extension by its module name."""
-        self.load_extension(module_str)
+        return self.load_extension(module_str)
 
     def magic_unload_ext(self, module_str):
         """Unload an IPython extension by its module name."""
@@ -3550,4 +3501,113 b' Defaulting color scheme to \'NoColor\'"""'
         """Reload an IPython extension by its module name."""
         self.reload_extension(module_str)
 
+    @testdec.skip_doctest
+    def magic_install_profiles(self, s):
+        """Install the default IPython profiles into the .ipython dir.
+
+        If the default profiles have already been installed, they will not
+        be overwritten. You can force overwriting them by using the ``-o``
+        option::
+
+            In [1]: %install_profiles -o
+        """
+        if '-o' in s:
+            overwrite = True
+        else:
+            overwrite = False
+        from IPython.config import profile
+        profile_dir = os.path.split(profile.__file__)[0]
+        ipython_dir = self.ipython_dir
+        files = os.listdir(profile_dir)
+
+        to_install = []
+        for f in files:
+            if f.startswith('ipython_config'):
+                src = os.path.join(profile_dir, f)
+                dst = os.path.join(ipython_dir, f)
+                if (not os.path.isfile(dst)) or overwrite:
+                    to_install.append((f, src, dst))
+        if len(to_install)>0:
+            print "Installing profiles to: ", ipython_dir
+            for (f, src, dst) in to_install:
+                shutil.copy(src, dst)
+                print "    %s" % f
+
+    def magic_install_default_config(self, s):
+        """Install IPython's default config file into the .ipython dir.
+
+        If the default config file (:file:`ipython_config.py`) is already
+        installed, it will not be overwritten. You can force overwriting
+        by using the ``-o`` option::
+
+            In [1]: %install_default_config
+        """
+        if '-o' in s:
+            overwrite = True
+        else:
+            overwrite = False
+        from IPython.config import default
+        config_dir = os.path.split(default.__file__)[0]
+        ipython_dir = self.ipython_dir
+        default_config_file_name = 'ipython_config.py'
+        src = os.path.join(config_dir, default_config_file_name)
+        dst = os.path.join(ipython_dir, default_config_file_name)
+        if (not os.path.isfile(dst)) or overwrite:
+            shutil.copy(src, dst)
+            print "Installing default config file: %s" % dst
+
+    # Pylab support: simple wrappers that activate pylab, load gui input
+    # handling and modify slightly %run
+
+    @testdec.skip_doctest
+    def _pylab_magic_run(self, parameter_s=''):
+        Magic.magic_run(self, parameter_s,
+                        runner=mpl_runner(self.shell.safe_execfile))
+
+    _pylab_magic_run.__doc__ = magic_run.__doc__
+
+    @testdec.skip_doctest
+    def magic_pylab(self, s):
+        """Load numpy and matplotlib to work interactively.
+
+        %pylab [GUINAME]
+
+        This function lets you activate pylab (matplotlib, numpy and
+        interactive support) at any point during an IPython session.
+
+        It will import at the top level numpy as np, pyplot as plt, matplotlib,
+        pylab and mlab, as well as all names from numpy and pylab.
+
+        Parameters
+        ----------
+        guiname : optional
+          One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk' or
+          'tk').  If given, the corresponding Matplotlib backend is used,
+          otherwise matplotlib's default (which you can override in your
+          matplotlib config file) is used.
+
+        Examples
+        --------
+        In this case, where the MPL default is TkAgg:
+        In [2]: %pylab
+
+        Welcome to pylab, a matplotlib-based Python environment.
+        Backend in use: TkAgg
+        For more information, type 'help(pylab)'.
+
+        But you can explicitly request a different backend:
+        In [3]: %pylab qt
+
+        Welcome to pylab, a matplotlib-based Python environment.
+        Backend in use: Qt4Agg
+        For more information, type 'help(pylab)'.
+        """
+        self.shell.enable_pylab(s)
+
+    def magic_tb(self, s):
+        """Print the last traceback with the currently active exception mode.
+
+        See %xmode for changing exception reporting modes."""
+        self.shell.showtraceback()
+
 # end Magic

IPython/core/prefilter.py

@@ -39,7 +39,7 b' from IPython.core.splitinput import split_user_input'
 from IPython.core.page import page
 
 from IPython.utils.traitlets import List, Int, Any, Str, CBool, Bool
-from IPython.utils.genutils import make_quoted_expr
+from IPython.utils.genutils import make_quoted_expr, Term
 from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
@@ -158,7 +158,7 b' class LineInfo(object):'
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
-            self._oinfo = ip._ofind(self.ifun)
+            self._oinfo = ip.shell._ofind(self.ifun)
         return self._oinfo
 
     def __str__(self):                                                         
@@ -362,7 +362,7 b' class PrefilterManager(Component):'
                 line = transformer.transform(line, continue_prompt)
         return line
 
-    def prefilter_line(self, line, continue_prompt):
+    def prefilter_line(self, line, continue_prompt=False):
         """Prefilter a single input line as text.
 
         This method prefilters a single line of text by calling the
@@ -416,7 +416,7 b' class PrefilterManager(Component):'
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
-    def prefilter_lines(self, lines, continue_prompt):
+    def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of
@@ -427,11 +427,19 b' class PrefilterManager(Component):'
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
-        out = []
-        for line in lines.rstrip('\n').split('\n'):
-            out.append(self.prefilter_line(line, continue_prompt))
-        return '\n'.join(out)
+        llines = lines.rstrip('\n').split('\n')
+        # We can get multiple lines in one shot, where multiline input 'blends'
+        # into one line, in cases like recalling from the readline history
+        # buffer.  We need to make sure that in such cases, we correctly
+        # communicate downstream which line is first and which are continuation
+        # ones.
+        if len(llines) > 1:
+            out = '\n'.join([self.prefilter_line(line, lnum>0)
+                             for lnum, line in enumerate(llines) ])
+        else:
+            out = self.prefilter_line(llines[0], continue_prompt)
 
+        return out
 
 #-----------------------------------------------------------------------------
 # Prefilter transformers
@@ -508,6 +516,47 b' class AssignMagicTransformer(PrefilterTransformer):'
         return line
 
 
+_classic_prompt_re = re.compile(r'(^[ \t]*>>> |^[ \t]*\.\.\. )')
+
+class PyPromptTransformer(PrefilterTransformer):
+    """Handle inputs that start with '>>> ' syntax."""
+
+    priority = Int(50, config=True)
+
+    def transform(self, line, continue_prompt):
+
+        if not line or line.isspace() or line.strip() == '...':
+            # This allows us to recognize multiple input prompts separated by
+            # blank lines and pasted in a single chunk, very common when
+            # pasting doctests or long tutorial passages.
+            return ''
+        m = _classic_prompt_re.match(line)
+        if m:
+            return line[len(m.group(0)):]
+        else:
+            return line
+
+
+_ipy_prompt_re = re.compile(r'(^[ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
+
+class IPyPromptTransformer(PrefilterTransformer):
+    """Handle inputs that start classic IPython prompt syntax."""
+
+    priority = Int(50, config=True)
+
+    def transform(self, line, continue_prompt):
+
+        if not line or line.isspace() or line.strip() == '...':
+            # This allows us to recognize multiple input prompts separated by
+            # blank lines and pasted in a single chunk, very common when
+            # pasting doctests or long tutorial passages.
+            return ''
+        m = _ipy_prompt_re.match(line)
+        if m:
+            return line[len(m.group(0)):]
+        else:
+            return line
+
 #-----------------------------------------------------------------------------
 # Prefilter checkers
 #-----------------------------------------------------------------------------
@@ -755,9 +804,17 b' class PrefilterHandler(Component):'
         line = line_info.line
         continue_prompt = line_info.continue_prompt
 
-        if (continue_prompt and self.shell.autoindent and line.isspace() and
-            (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2 or
-             (self.shell.buffer[-1]).isspace() )):
+        if (continue_prompt and
+            self.shell.autoindent and
+            line.isspace() and
+            
+            (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2
+             or
+             not self.shell.buffer
+             or
+             (self.shell.buffer[-1]).isspace()
+             )
+            ):
             line = ''
 
         self.shell.log(line, line, continue_prompt)
@@ -845,12 +902,11 b' class AutoHandler(PrefilterHandler):'
         pre     = line_info.pre
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
-
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
-            self.log(line,line,continue_prompt)
+            self.shell.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
@@ -967,7 +1023,9 b' class EmacsHandler(PrefilterHandler):'
 
 _default_transformers = [
     AssignSystemTransformer,
-    AssignMagicTransformer
+    AssignMagicTransformer,
+    PyPromptTransformer,
+    IPyPromptTransformer,
 ]
 
 _default_checkers = [
@@ -992,4 +1050,3 b' _default_handlers = ['
     HelpHandler,
     EmacsHandler
 ]
-

IPython/core/prompts.py

@@ -549,15 +549,20 b' class CachedOutput:'
             # print "Got prompt: ", outprompt
             if self.do_full_cache:
                 cout_write(outprompt)
-            else:
-                print "self.do_full_cache = False"
 
-            # and now call a possibly user-defined print mechanism
+            # and now call a possibly user-defined print mechanism. Note that
+            # self.display typically prints as a side-effect, we don't do any
+            # printing to stdout here.
+            try:
             manipulated_val = self.display(arg)
+            except TypeError:
+                # If the user's display hook didn't return a string we can
+                # print, we're done.  Happens commonly if they return None
+                cout_write('\n')
+                return
             
             # user display hooks can change the variable to be stored in
             # output history
-            
             if manipulated_val is not None:
                 arg = manipulated_val
                 

IPython/core/quitter.py

@@ -1,10 +1,10 b''
-#!/usr/bin/env python
-# encoding: utf-8
+# coding: utf-8
 """
 A simple class for quitting IPython.
 
-Authors:
-
+Authors
+-------
+* Fernando Perez
 * Brian Granger
 """
 
@@ -19,6 +19,7 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
+import sys
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
@@ -30,9 +31,13 b' class Quitter(object):'
         self.shell = shell
         self.name = name
         
-    def __repr__(self):
+    def __str__(self):
         return 'Type %s() to exit.' % self.name
-    __str__ = __repr__
 
     def __call__(self):
-        self.shell.exit()
  No newline at end of file
+        self.shell.ask_exit()
+
+    # Repr MUST return a string, else display like pprint hooks get confused
+    def __repr__(self):
+        self.shell.ask_exit()
+        return 'Bye.'

IPython/core/release.py

@@ -23,7 +23,7 b" name = 'ipython'"
 development = True    # change this to False to do a release
 version_base = '0.11'
 branch = 'ipython'
-revision = '1205'
+revision = '1321'
 
 if development:
     if branch == 'ipython':

IPython/core/tests/tclass.py

@@ -1,16 +1,12 b''
-"""Simple script to instantiate a class for testing %run"""
+"""Simple script to be run *twice*, to check reference counting bugs.
 
-import sys
-
-# An external test will check that calls to f() work after %run
-class foo: pass
+See test_run for details."""
 
-def f():
-    return foo()
+import sys
 
-# We also want to ensure that while objects remain available for immediate
-# access, objects from *previous* runs of the same script get collected, to
-# avoid accumulating massive amounts of old references.
+# We want to ensure that while objects remain available for immediate access,
+# objects from *previous* runs of the same script get collected, to avoid
+# accumulating massive amounts of old references.
 class C(object):
     def __init__(self,name):
         self.name = name
@@ -18,6 +14,7 b' class C(object):'
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
 
+
 try:
     name = sys.argv[1]
 except IndexError:
@@ -25,3 +22,9 b' except IndexError:'
 else:
     if name.startswith('C'):
         c = C(name)
+
+#print >> sys.stderr, "ARGV:", sys.argv  # dbg
+
+# This next print statement is NOT debugging, we're making the check on a
+# completely separate process so we verify by capturing stdout:
+print 'ARGV 1-:', sys.argv[1:]

IPython/core/tests/test_iplib.py

@@ -13,68 +13,233 b' import tempfile'
 import nose.tools as nt
 
 # our own packages
-from IPython.core import iplib
-from IPython.core import ipapi
-from IPython.core.oldusersetup import user_setup
+from IPython.testing import decorators as dec
+from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
-# Useful global ipapi object and main IPython one.  Unfortunately we have a
-# long precedent of carrying the 'ipapi' global object which is injected into
-# the system namespace as _ip, but that keeps a pointer to the actual IPython
-# InteractiveShell instance, which is named IP.  Since in testing we do need
-# access to the real thing (we want to probe beyond what ipapi exposes), make
-# here a global reference to each.  In general, things that are exposed by the
-# ipapi instance should be read from there, but we also will often need to use
-# the actual IPython one.
-
-# Get the public instance of IPython, and if it's None, make one so we can use
-# it for testing
-ip = ipapi.get()
-if ip is None:
-    # IPython not running yet,  make one from the testing machinery for
-    # consistency when the test suite is being run via iptest
-    from IPython.testing.plugin import ipdoctest
-    ip = ipapi.get()
+# Get the public instance of IPython
+ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_reset():
     """reset must clear most namespaces."""
-    ip.reset()  # first, it should run without error
-    # Then, check that most namespaces end up empty
+    # The number of variables in the private user_config_ns is not zero, but it
+    # should be constant regardless of what we do
+    nvars_config_ns = len(ip.user_config_ns)
+
+    # Check that reset runs without error
+    ip.reset()
+
+    # Once we've reset it (to clear of any junk that might have been there from
+    # other tests, we can count how many variables are in the user's namespace
+    nvars_user_ns = len(ip.user_ns)
+
+    # Now add a few variables to user_ns, and check that reset clears them
+    ip.user_ns['x'] = 1
+    ip.user_ns['y'] = 1
+    ip.reset()
+    
+    # Finally, check that all namespaces have only as many variables as we
+    # expect to find in them:
     for ns in ip.ns_refs_table:
         if ns is ip.user_ns:
-            # The user namespace is reset with some data, so we can't check for
-            # it being empty
-            continue
-        nt.assert_equals(len(ns),0)
-
-
-# make sure that user_setup can be run re-entrantly in 'install' mode.
-def test_user_setup():
-    # use a lambda to pass kwargs to the generator
-    user_setup = lambda a,k: user_setup(*a,**k)
-    kw = dict(mode='install', interactive=False)
-
-    # Call the user setup and verify that the directory exists
-    yield user_setup, (ip.config.IPYTHONDIR,''), kw
-    yield os.path.isdir, ip.config.IPYTHONDIR
-
-    # Now repeat the operation with a non-existent directory. Check both that
-    # the call succeeds and that the directory is created.
-    tmpdir = tempfile.mktemp(prefix='ipython-test-')
-    # Use a try with an empty except because try/finally doesn't work with a 
-    # yield in Python 2.4.
-    try:
-        yield user_setup, (tmpdir,''), kw
-        yield os.path.isdir, tmpdir
-    except:
-        pass
-    # Clean up the temp dir once done
-    shutil.rmtree(tmpdir)
+            nvars_expected = nvars_user_ns
+        elif ns is ip.user_config_ns:
+            nvars_expected = nvars_config_ns
+        else:
+            nvars_expected = 0
+            
+        yield nt.assert_equals(len(ns), nvars_expected)
+
+
+# Tests for reporting of exceptions in various modes, handling of SystemExit,
+# and %tb functionality.  This is really a mix of testing ultraTB and iplib.
+
+def doctest_tb_plain():
+    """
+In [18]: xmode plain
+Exception reporting mode: Plain
+
+In [19]: run simpleerr.py
+Traceback (most recent call last):
+  ...line 32, in <module>
+    bar(mode)
+  ...line 16, in bar
+    div0()
+  ...line 8, in div0
+    x/y
+ZeroDivisionError: integer division or modulo by zero
+    """
+
+
+def doctest_tb_context():
+    """
+In [3]: xmode context
+Exception reporting mode: Context
+
+In [4]: run simpleerr.py
+---------------------------------------------------------------------------
+ZeroDivisionError                         Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode)
+     14     "bar"
+     15     if mode=='div':
+---> 16         div0()
+     17     elif mode=='exit':
+     18         try:
+<BLANKLINE>
+... in div0()
+      6     x = 1
+      7     y = 0
+----> 8     x/y
+      9 
+     10 def sysexit(stat, mode):
+<BLANKLINE>
+ZeroDivisionError: integer division or modulo by zero
+"""
     
+
+def doctest_tb_verbose():
+    """
+In [5]: xmode verbose
+Exception reporting mode: Verbose
+
+In [6]: run simpleerr.py
+---------------------------------------------------------------------------
+ZeroDivisionError                         Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+        global bar = <function bar at ...>
+        global mode = 'div'
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode='div')
+     14     "bar"
+     15     if mode=='div':
+---> 16         div0()
+        global div0 = <function div0 at ...>
+     17     elif mode=='exit':
+     18         try:
+<BLANKLINE>
+... in div0()
+      6     x = 1
+      7     y = 0
+----> 8     x/y
+        x = 1
+        y = 0
+      9 
+     10 def sysexit(stat, mode):
+<BLANKLINE>
+ZeroDivisionError: integer division or modulo by zero
+      """
+
+
+def doctest_tb_sysexit():
+    """
+In [17]: %xmode plain
+Exception reporting mode: Plain
+
+In [18]: %run simpleerr.py exit
+An exception has occurred, use %tb to see the full traceback.
+SystemExit: (1, 'Mode = exit')
+
+In [19]: %run simpleerr.py exit 2
+An exception has occurred, use %tb to see the full traceback.
+SystemExit: (2, 'Mode = exit')
+
+In [20]: %tb
+Traceback (most recent call last):
+  File ... in <module>
+    bar(mode)
+  File ... line 22, in bar
+    sysexit(stat, mode)
+  File ... line 11, in sysexit
+    raise SystemExit(stat, 'Mode = %s' % mode)
+SystemExit: (2, 'Mode = exit')
+
+In [21]: %xmode context
+Exception reporting mode: Context
+
+In [22]: %tb
+---------------------------------------------------------------------------
+SystemExit                                Traceback (most recent call last)
+<BLANKLINE>
+...<module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+     33 
+     34 
+<BLANKLINE>
+...bar(mode)
+     20         except:
+     21             stat = 1
+---> 22         sysexit(stat, mode)
+     23     else:
+     24         raise ValueError('Unknown mode')
+<BLANKLINE>
+...sysexit(stat, mode)
+      9 
+     10 def sysexit(stat, mode):
+---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
+     12 
+     13 def bar(mode):
+<BLANKLINE>
+SystemExit: (2, 'Mode = exit')
+
+In [23]: %xmode verbose
+Exception reporting mode: Verbose
+
+In [24]: %tb
+---------------------------------------------------------------------------
+SystemExit                                Traceback (most recent call last)
+<BLANKLINE>
+... in <module>()
+     30         mode = 'div'
+     31 
+---> 32     bar(mode)
+        global bar = <function bar at ...>
+        global mode = 'exit'
+     33 
+     34 
+<BLANKLINE>
+... in bar(mode='exit')
+     20         except:
+     21             stat = 1
+---> 22         sysexit(stat, mode)
+        global sysexit = <function sysexit at ...>
+        stat = 2
+        mode = 'exit'
+     23     else:
+     24         raise ValueError('Unknown mode')
+<BLANKLINE>
+... in sysexit(stat=2, mode='exit')
+      9 
+     10 def sysexit(stat, mode):
+---> 11     raise SystemExit(stat, 'Mode = %s' % mode)
+        global SystemExit = undefined
+        stat = 2
+        mode = 'exit'
+     12 
+     13 def bar(mode):
+<BLANKLINE>
+SystemExit: (2, 'Mode = exit')
+    """

IPython/core/tests/test_magic.py

@@ -2,22 +2,31 b''
 
 Needs to be run by nose (to make ipython session available).
 """
+from __future__ import absolute_import
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
+# third-party
 import nose.tools as nt
 
+# our own
+from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
-
+#-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
@@ -37,6 +46,19 b' def test_rehashx():'
     yield (nt.assert_true, len(scoms) > 10)
 
 
+def test_magic_parse_options():
+    """Test that we don't mangle paths when parsing magic options."""
+    ip = get_ipython()
+    path = 'c:\\x'
+    opts = ip.parse_options('-f %s' % path,'f:')[0]
+    # argv splitting is os-dependent
+    if os.name == 'posix':
+        expected = 'c:x'
+    else:
+        expected = path
+    nt.assert_equals(opts['f'], expected)
+
+    
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
@@ -45,35 +67,94 b' def doctest_hist_f():'
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
+
+    In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
-    XXX - This test is not recording the output correctly.  Not sure why...
+    XXX - This test is not recording the output correctly.  For some reason, in
+    testing mode the raw history isn't getting populated.  No idea why.
+    Disabling the output checking for now, though at least we do run it.
 
-    In [20]: 'hist' in _ip.lsmagic()
-    Out[20]: True
+    In [1]: 'hist' in _ip.lsmagic()
+    Out[1]: True
 
-    In [6]: x=1
+    In [2]: x=1
 
-    In [7]: %hist -n -r 2
+    In [3]: %hist -r 2
     x=1  # random
-    hist -n -r 2  # random
+    %hist -r 2
     """
 
-# This test is known to fail on win32.
-# See ticket https://bugs.launchpad.net/bugs/366334
-def test_obj_del():
-    _ip = get_ipython()
-    """Test that object's __del__ methods are called on exit."""
-    test_dir = os.path.dirname(__file__)
-    del_file = os.path.join(test_dir,'obj_del.py')
-    ipython_cmd = find_cmd('ipython')
-    out = _ip.getoutput('%s %s' % (ipython_cmd, del_file))
-    nt.assert_equals(out,'obj_del.py: object A deleted')
+def doctest_hist_op():
+    """Test %hist -op
+
+    In [1]: class b:
+       ...:         pass
+       ...: 
+
+    In [2]: class s(b):
+       ...:         def __str__(self):
+       ...:             return 's'
+       ...: 
+
+    In [3]: 
 
+    In [4]: class r(b):
+       ...:         def __repr__(self):
+       ...:             return 'r'
+       ...: 
+
+    In [5]: class sr(s,r): pass
+       ...: 
+
+    In [6]: 
+
+    In [7]: bb=b()
+
+    In [8]: ss=s()
+
+    In [9]: rr=r()
+
+    In [10]: ssrr=sr()
+
+    In [11]: bb
+    Out[11]: <...b instance at ...>
+
+    In [12]: ss
+    Out[12]: <...s instance at ...>
+
+    In [13]: 
+
+    In [14]: %hist -op
+    >>> class b:
+    ...     pass
+    ... 
+    >>> class s(b):
+    ...     def __str__(self):
+    ...         return 's'
+    ... 
+    >>> 
+    >>> class r(b):
+    ...     def __repr__(self):
+    ...         return 'r'
+    ... 
+    >>> class sr(s,r): pass
+    >>> 
+    >>> bb=b()
+    >>> ss=s()
+    >>> rr=r()
+    >>> ssrr=sr()
+    >>> bb
+    <...b instance at ...>
+    >>> ss
+    <...s instance at ...>
+    >>> 
+    >>> get_ipython().magic("hist -op")
+    """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
@@ -98,7 +179,11 b' def test_shist():'
     
     shutil.rmtree(tfile)
     
[e-mail]_not_numpy
+    
+# XXX failing for now, until we get clearcmd out of quarantine.  But we should
+# fix this and revert the skip to happen only if numpy is not around.
+#@dec.skipif_not_numpy
[e-mail]
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
@@ -109,162 +194,8 b' def test_numpy_clear_array_undec():'
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
[e-mail]()
-def test_fail_dec(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]('This one shouldn not run')
-def test_fail_dec2(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]
-def test_fail_dec3(*a,**k):
-    yield nt.assert_true, False
-
-
-def doctest_refbug():
-    """Very nasty problem with references held by multiple runs of a script.
-    See: https://bugs.launchpad.net/ipython/+bug/269966
-
-    In [1]: _ip.clear_main_mod_cache()
-    
-    In [2]: run refbug
-
-    In [3]: call_f()
-    lowercased: hello
-
-    In [4]: run refbug
-
-    In [5]: call_f()
-    lowercased: hello
-    lowercased: hello
-    """
-
-#-----------------------------------------------------------------------------
-# Tests for %run
-#-----------------------------------------------------------------------------
-
-# %run is critical enough that it's a good idea to have a solid collection of
-# tests for it, some as doctests and some as normal tests.
-
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass foo
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [4]: run tclass C-first_pass
-
-    In [5]: run tclass C-second_pass
-    tclass.py: deleting object: C-first_pass
-    """
-
-def doctest_run_builtins():
-    """Check that %run doesn't damage __builtins__ via a doctest.
-
-    This is similar to the test_run_builtins, but I want *both* forms of the
-    test to catch any possible glitches in our testing machinery, since that
-    modifies %run somewhat.  So for this, we have both a normal test (below)
-    and a doctest (this one).
-
-    In [1]: import tempfile
-
-    In [2]: bid1 = id(__builtins__)
-
-    In [3]: fname = tempfile.mkstemp()[1]
-
-    In [3]: f = open(fname,'w')
-
-    In [4]: f.write('pass\\n')
-
-    In [5]: f.flush()
-
-    In [6]: print type(__builtins__)
-    <type 'module'>
-
-    In [7]: %run "$fname"
-
-    In [7]: f.close()
-
-    In [8]: bid2 = id(__builtins__)
-
-    In [9]: print type(__builtins__)
-    <type 'module'>
-
-    In [10]: bid1 == bid2
-    Out[10]: True
-
-    In [12]: try:
-       ....:     os.unlink(fname)
-       ....: except:
-       ....:     pass
-       ....: 
-    """
-
-# For some tests, it will be handy to organize them in a class with a common
-# setup that makes a temp file
-
-class TestMagicRun(object):
-
-    def setup(self):
-        """Make a valid python temp file."""
-        fname = tempfile.mkstemp()[1]
-        f = open(fname,'w')
-        f.write('pass\n')
-        f.flush()
-        self.tmpfile = f
-        self.fname = fname
-
-    def run_tmpfile(self):
-        _ip = get_ipython()
-        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
-        # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run "%s"' % self.fname)
-
-    def test_builtins_id(self):
-        """Check that %run doesn't damage __builtins__ """
-        _ip = get_ipython()
-        # Test that the id of __builtins__ is not modified by %run
-        bid1 = id(_ip.user_ns['__builtins__'])
-        self.run_tmpfile()
-        bid2 = id(_ip.user_ns['__builtins__'])
-        tt.assert_equals(bid1, bid2)
-
-    def test_builtins_type(self):
-        """Check that the type of __builtins__ doesn't change with %run.
-        
-        However, the above could pass if __builtins__ was already modified to
-        be a dict (it should be a module) by a previous use of %run.  So we
-        also check explicitly that it really is a module:
-        """
-        _ip = get_ipython()
-        self.run_tmpfile()
-        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
-
-    def test_prompts(self):
-        """Test that prompts correctly generate after %run"""
-        self.run_tmpfile()
-        _ip = get_ipython()
-        p2 = str(_ip.outputcache.prompt2).strip()
-        nt.assert_equals(p2[:3], '...')
-
-    def teardown(self):
-        self.tmpfile.close()
-        try:
-            os.unlink(self.fname)
-        except:
-            # On Windows, even though we close the file, we still can't delete
-            # it.  I have no clue why
-            pass
-
 # Multiple tests for clipboard pasting
[e-mail]
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
@@ -286,11 +217,11 b' def test_paste():'
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
-        yield (nt.assert_equal, user_ns['x'], 1)
+        yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
-        yield (nt.assert_equal, user_ns['x'], 2)
+        yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
@@ -299,14 +230,14 b' def test_paste():'
         ...     y.append(i**2)
         ...
         """)
-        yield (nt.assert_equal, user_ns['x'], [1,2,3])
-        yield (nt.assert_equal, user_ns['y'], [1,4,9])
+        yield nt.assert_equal(user_ns['x'], [1,2,3])
+        yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
-        yield (nt.assert_false, 'x' in user_ns)
+        yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
-        yield (nt.assert_equal, user_ns['x'], [1,2,3])
+        yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
@@ -320,12 +251,29 b' def test_paste():'
             out = w.getvalue()
         finally:
             _ip.write = writer
-        yield (nt.assert_equal, user_ns['a'], 100)
-        yield (nt.assert_equal, user_ns['b'], 200)
-        yield (nt.assert_equal, out, code+"\n## -- End pasted text --\n")
+        yield nt.assert_equal(user_ns['a'], 100)
+        yield nt.assert_equal(user_ns['b'], 200)
+        yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
+
+def test_time():
+    _ip.magic('time None')
+
+
+def doctest_time():
+    """
+    In [10]: %time None
+    CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
+    Wall time: 0.00 s
+    """
+
+def test_doctest_mode():
+    "Toggle doctest_mode twice, it should be a no-op and run without error"
+    _ip.magic('doctest_mode')
+    _ip.magic('doctest_mode')
+    

IPython/core/ultratb.py

@@ -88,7 +88,6 b' import types'
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
-
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython.utils import PyColorize
@@ -263,7 +262,7 b' def _fixed_getinnerframes(etb, context=1,tb_offset=0):'
 
 _parser = PyColorize.Parser()
     
-def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):
+def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
@@ -313,6 +312,15 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
 class TBTools:
     """Basic tools used by all traceback printer classes."""
 
+    #: Default output stream, can be overridden at call time.  A special value
+    #: of 'stdout' *as a string* can be given to force extraction of sys.stdout
+    #: at runtime.  This allows testing exception printing with doctests, that
+    #: swap sys.stdout just at execution time.
+    #: Warning: be VERY careful to set this to one of the Term streams, NEVER
+    #: directly to sys.stdout/err, because under win32 the Term streams come from
+    #: pyreadline and know how to handle color correctly, whie stdout/err don't.
+    out_stream = Term.cerr
+
     def __init__(self,color_scheme = 'NoColor',call_pdb=False):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
@@ -376,16 +384,31 b' class ListTB(TBTools):'
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
-        print >> Term.cerr, self.text(etype,value,elist)
-        Term.cerr.flush()
+        Term.cerr.writeln(self.text(etype,value,elist))
 
     def text(self,etype, value, elist,context=5):
-        """Return a color formatted string with the traceback info."""
+        """Return a color formatted string with the traceback info.
+
+        Parameters
+        ----------
+        etype : exception type
+          Type of the exception raised.
+
+        value : object
+          Data stored in the exception
+
+        elist : list
+          List of frames, see class docstring for details.
+
+        Returns
+        -------
+        String with formatted exception.
+        """
 
         Colors = self.Colors
-        out_string = ['%s%s%s\n' % (Colors.topline,'-'*60,Colors.Normal)]
+        out_string = []
         if elist:
-            out_string.append('Traceback %s(most recent call last)%s:' % \
+            out_string.append('Traceback %s(most recent call last)%s:' %
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_string.extend(self._format_list(elist))
         lines = self._format_exception_only(etype, value)
@@ -492,15 +515,29 b' class ListTB(TBTools):'
             else:
                 list.append('%s\n' % str(stype))
 
-        # vds:>>
+        # sync with user hooks
         if have_filedata:
             ipinst = ipapi.get()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filename, lineno, 0)
-        # vds:<<
 
         return list
 
+    def show_exception_only(self, etype, value):
+        """Only print the exception type and message, without a traceback.
+        
+        Parameters
+        ----------
+        etype : exception type
+        value : exception value
+        """
+        # This method needs to use __call__ from *this* class, not the one from
+        # a subclass whose signature or behavior may be different
+        Term.cout.flush()
+        ostream = sys.stdout if self.out_stream == 'stdout' else Term.cerr
+        ostream.write(ListTB.text(self, etype, value, []))
+        ostream.flush()        
+
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
@@ -781,7 +818,7 b' class VerboseTB(TBTools):'
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
-                    _formatTracebackLines(lnum,index,lines,Colors,lvals,
+                    _format_traceback_lines(lnum,index,lines,Colors,lvals,
                                           col_scheme))))
 
         # Get (safely) a string form of the exception info
@@ -854,11 +891,11 b' class VerboseTB(TBTools):'
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
-                if hasattr(self,'tb'):
+                if hasattr(self,'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
-                while self.tb.tb_next is not None:
+                while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
@@ -872,8 +909,7 b' class VerboseTB(TBTools):'
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
-        print >> Term.cerr, self.text(etype, evalue, etb)
-        Term.cerr.flush()
+        Term.cerr.writeln(self.text(etype, evalue, etb))
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
@@ -980,6 +1016,7 b' class AutoFormattedTB(FormattedTB):'
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
+    
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
@@ -992,14 +1029,16 b' class AutoFormattedTB(FormattedTB):'
           given at initialization time.  """
         
         if out is None:
-            out = Term.cerr
+            out = sys.stdout if self.out_stream=='stdout' else self.out_stream
         Term.cout.flush()
         if tb_offset is not None:
             tb_offset, self.tb_offset = self.tb_offset, tb_offset
-            print >> out, self.text(etype, evalue, etb)
+            out.write(self.text(etype, evalue, etb))
+            out.write('\n')
             self.tb_offset = tb_offset
         else:
-            print >> out, self.text(etype, evalue, etb)
+            out.write(self.text(etype, evalue, etb))
+            out.write('\n')
         out.flush()
         try:
             self.debugger()

IPython/core/usage.py

@@ -1,338 +1,47 b''
 # -*- coding: utf-8 -*-
-#*****************************************************************************
-#       Copyright (C) 2001-2004 Fernando Perez. <[e-mail]>
+"""Usage information for the main IPython applications.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2010  The IPython Development Team
+#  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#*****************************************************************************
+#-----------------------------------------------------------------------------
 
 import sys
 from IPython.core import release
 
-__doc__ = """
-IPython -- An enhanced Interactive Python
-=========================================
+cl_usage = """\
+ipython [options] [files]
+
+IPython: an enhanced interactive Python shell.
 
 A Python shell with automatic history (input and output), dynamic object
-introspection, easier configuration, command completion, access to the system
-shell and more.
-
-IPython can also be embedded in running programs. See EMBEDDING below.
-
-
-USAGE
-       ipython [options] files
-
-       If invoked with no options, it executes all the files listed in
-       sequence and drops you into the interpreter while still acknowledging
-       any options you may have set in your ipythonrc file. This behavior is
-       different from standard Python, which when called as python -i will
-       only execute one file and will ignore your configuration setup.
-
-       Please note that some of the configuration options are not available at
-       the command line, simply because they are not practical here. Look into
-       your ipythonrc configuration file for details on those. This file
-       typically installed in the $HOME/.ipython directory.
-
-       For Windows users, $HOME resolves to C:\\Documents and
-       Settings\\YourUserName in most instances, and _ipython is used instead
-       of .ipython, since some Win32 programs have problems with dotted names
-       in directories.
-
-       In the rest of this text, we will refer to this directory as
-       IPYTHONDIR.
-
-REGULAR OPTIONS
-       After the above threading options have been given, regular options  can
-       follow  in any order.  All options can be abbreviated to their shortest
-       non-ambiguous form and are case-sensitive.  One or two  dashes  can  be
-       used.   Some options have an alternate short form, indicated after a |.
-
-       Most options can also be set from your  ipythonrc  configuration  file.
-       See the provided examples for assistance.  Options given on the comman-
-       dline override the values set in the ipythonrc file.
-
-       All options with a [no] prepended can be specified in negated form
-       (using -nooption instead of -option) to turn the feature off.
-
-       -h, --help
-              Show summary of options.
-
-       -autocall <val>
-              Make IPython automatically call any callable object even if you
-              didn't type explicit parentheses. For example, 'str 43' becomes
-              'str(43)' automatically.  The value can be '0' to disable the
-              feature, '1' for 'smart' autocall, where it is not applied if
-              there are no more arguments on the line, and '2' for 'full'
-              autocall, where all callable objects are automatically called
-              (even if no arguments are present).  The default is '1'.
-
-       -[no]autoindent
-              Turn automatic indentation on/off.
-
-       -[no]automagic
-              Make magic commands automatic (without needing their first char-
-              acter to be %).  Type %magic at  the  IPython  prompt  for  more
-              information.
-
-       -[no]autoedit_syntax
-              When a syntax error occurs after editing a file, automatically
-              open the file to the trouble causing line for convenient fixing.
-
-       -[no]banner
-              Print the intial information banner (default on).
-
-       -c <command>
-              Execute  the  given  command  string, and set sys.argv to ['c'].
-              This is similar to the -c option in  the  normal  Python  inter-
-              preter.
-
-       -cache_size|cs <n>
-              Size  of  the output cache (maximum number of entries to hold in
-              memory).  The default is 1000, you can change it permanently  in
-              your  config  file.   Setting  it  to  0 completely disables the
-              caching system, and the minimum value accepted  is  20  (if  you
-              provide  a value less than 20, it is reset to 0 and a warning is
-              issued).  This limit is defined because otherwise  you'll  spend
-              more time re-flushing a too small cache than working.
-
-       -classic|cl
-              Gives IPython a similar feel to the classic Python prompt.
-
-       -colors <scheme>
-              Color  scheme  for  prompts  and exception reporting.  Currently
-              implemented: NoColor, Linux, and LightBG.
-
-       -[no]color_info
-              IPython can display information about objects via a set of func-
-              tions, and optionally can use colors for this, syntax highlight-
-              ing source code and various other  elements.   However,  because
-              this  information  is  passed  through a pager (like 'less') and
-              many pagers get confused with color codes, this option is off by
-              default.   You  can  test  it and turn it on permanently in your
-              ipythonrc file if it works for you.  As a reference, the  'less'
-              pager  supplied  with  Mandrake 8.2 works ok, but that in RedHat
-              7.2 doesn't.
-
-              Test it and turn it on permanently if it works with your system.
-              The  magic function @color_info allows you to toggle this inter-
-              actively for testing.
-
-       -[no]confirm_exit
-              Set to confirm when you try to exit IPython with  an  EOF  (Con-
-              trol-D in Unix, Control-Z/Enter in Windows). Note that using the
-              magic functions @Exit or @Quit you  can  force  a  direct  exit,
-              bypassing any confirmation.
-
-       -[no]debug
-              Show  information  about the loading process. Very useful to pin
-              down problems with your configuration files or  to  get  details
-              about session restores.
-
-       -[no]deep_reload
-              IPython  can use the deep_reload module which reloads changes in
-              modules recursively (it replaces the reload() function,  so  you
-              don't need to change anything to use it). deep_reload() forces a
-              full reload of modules whose code may have  changed,  which  the
-              default reload() function does not.
-
-              When  deep_reload  is off, IPython will use the normal reload(),
-              but deep_reload will still be available as dreload(). This  fea-
-              ture  is  off  by default [which means that you have both normal
-              reload() and dreload()].
-
-       -editor <name>
-              Which editor to use with the @edit command. By default,  IPython
-              will  honor  your EDITOR environment variable (if not set, vi is
-              the Unix default and notepad the Windows one). Since this editor
-              is  invoked on the fly by IPython and is meant for editing small
-              code snippets, you may want to use a small,  lightweight  editor
-              here (in case your default EDITOR is something like Emacs).
-
-       -ipythondir <name>
-              The  name  of  your  IPython configuration directory IPYTHONDIR.
-              This can also be  specified  through  the  environment  variable
-              IPYTHONDIR.
-
-       -log|l Generate  a log file of all input. The file is named
-              ipython_log.py in your current directory (which prevents logs
-              from multiple IPython sessions from trampling each other). You
-              can use this to later restore a session by loading your logfile
-              as a file to be executed with option -logplay (see below).
-
-       -logfile|lf
-              Specify the name of your logfile.
-
-       -logplay|lp
-              Replay  a previous log. For restoring a session as close as pos-
-              sible to the state you left it in, use this option  (don't  just
-              run the logfile). With -logplay, IPython will try to reconstruct
-              the previous working environment in full, not just  execute  the
-              commands in the logfile.
-              When  a  session is restored, logging is automatically turned on
-              again with the name of the logfile it was invoked  with  (it  is
-              read  from the log header). So once you've turned logging on for
-              a session, you can quit IPython and reload it as many  times  as
-              you  want  and  it  will continue to log its history and restore
-              from the beginning every time.
-
-              Caveats: there are limitations in this option. The history vari-
-              ables  _i*,_* and _dh don't get restored properly. In the future
-              we will try to implement full  session  saving  by  writing  and
-              retrieving  a failed because of inherent limitations of Python's
-              Pickle module, so this may have to wait.
-
-       -[no]messages
-              Print messages which IPython collects about its startup  process
-              (default on).
-
-       -[no]pdb
-              Automatically  call the pdb debugger after every uncaught excep-
-              tion. If you are used to debugging  using  pdb,  this  puts  you
-              automatically  inside of it after any call (either in IPython or
-              in code called by it) which triggers  an  exception  which  goes
-              uncaught.
-
-       -[no]pprint
-              IPython  can  optionally  use the pprint (pretty printer) module
-              for displaying results. pprint tends to give a nicer display  of
-              nested  data structures. If you like it, you can turn it on per-
-              manently in your config file (default off).
-
-       -profile|p <name>
-              Assume that your config file is ipythonrc-<name> (looks in  cur-
-              rent dir first, then in IPYTHONDIR). This is a quick way to keep
-              and load multiple config files for different  tasks,  especially
-              if  you  use  the include option of config files. You can keep a
-              basic IPYTHONDIR/ipythonrc file and then have  other  'profiles'
-              which  include  this  one  and  load extra things for particular
-              tasks. For example:
-
-              1) $HOME/.ipython/ipythonrc : load basic things you always want.
-              2)  $HOME/.ipython/ipythonrc-math  :  load  (1)  and basic math-
-              related modules.
-              3) $HOME/.ipython/ipythonrc-numeric : load (1) and  Numeric  and
-              plotting modules.
-
-              Since  it is possible to create an endless loop by having circu-
-              lar file inclusions, IPython will stop if it reaches  15  recur-
-              sive inclusions.
-
-       -prompt_in1|pi1 <string>
-              Specify  the string used for input prompts. Note that if you are
-              using numbered prompts, the number is represented with a '\#' in
-              the  string.  Don't forget to quote strings with spaces embedded
-              in them. Default: 'In [\#]: '.
-
-              Most bash-like  escapes  can  be  used  to  customize  IPython's
-              prompts, as well as a few additional ones which are IPython-spe-
-              cific.  All valid prompt escapes are described in detail in  the
-              Customization section of the IPython HTML/PDF manual.
-
-       -prompt_in2|pi2 <string>
-              Similar to the previous option, but used for the continuation
-              prompts. The special sequence '\D' is similar to '\#', but with
-              all digits replaced dots (so you can have your continuation
-              prompt aligned with your input prompt).  Default: ' .\D.: '
-              (note three spaces at the start for alignment with 'In [\#]').
-
-       -prompt_out|po <string>
-              String   used   for  output  prompts,  also  uses  numbers  like
-              prompt_in1.  Default: 'Out[\#]:'.
-
-       -quick Start in bare bones mode (no config file loaded).
-
-       -rcfile <name>
-              Name of your  IPython  resource  configuration  file.   normally
-              IPython    loads   ipythonrc   (from   current   directory)   or
-              IPYTHONDIR/ipythonrc.  If the loading of your config file fails,
-              IPython  starts  with  a  bare  bones  configuration (no modules
-              loaded at all).
-
-       -[no]readline
-              Use the readline library, which is needed to support  name  com-
-              pletion  and  command history, among other things. It is enabled
-              by default, but may cause  problems  for  users  of  X/Emacs  in
-              Python comint or shell buffers.
-
-              Note  that  emacs 'eterm' buffers (opened with M-x term) support
-              IPython's readline and syntax coloring fine, only  'emacs'  (M-x
-              shell and C-c !)  buffers do not.
-
-       -screen_length|sl <n>
-              Number  of lines of your screen.  This is used to control print-
-              ing of very long strings.  Strings longer than  this  number  of
-              lines  will be sent through a pager instead of directly printed.
-
-              The default value for this is 0, which means IPython will  auto-
-              detect  your  screen  size  every time it needs to print certain
-              potentially long strings (this doesn't change  the  behavior  of
-              the  'print'  keyword,  it's  only triggered internally). If for
-              some reason this isn't working well (it needs  curses  support),
-              specify it yourself. Otherwise don't change the default.
-
-       -separate_in|si <string>
-              Separator before input prompts.  Default '0.
-
-       -separate_out|so <string>
-              Separator before output prompts.  Default: 0 (nothing).
-
-       -separate_out2|so2 <string>
-              Separator after output prompts.  Default: 0 (nothing).
-
-       -nosep Shorthand for '-separate_in 0 -separate_out 0 -separate_out2 0'.
-              Simply removes all input/output separators.
-
-       -upgrade
-              Allows you to upgrade your  IPYTHONDIR  configuration  when  you
-              install  a  new  version  of  IPython.   Since  new versions may
-              include new command lines options or example files, this  copies
-              updated ipythonrc-type files.  However, it backs up (with a .old
-              extension) all files which it overwrites so that you  can  merge
-              back any custimizations you might have in your personal files.
-
-       -Version
-              Print version information and exit.
-
-       -wxversion <string>
-              Select a specific version of wxPython (used in conjunction with
-              -wthread). Requires the wxversion module, part of recent
-              wxPython distributions.
-
-       -xmode <modename>
-              Mode  for  exception reporting.  The valid modes are Plain, Con-
-              text, and Verbose.
-
-              - Plain: similar to python's normal traceback printing.
-
-              - Context: prints 5 lines of context  source  code  around  each
-              line in the traceback.
-
-              - Verbose: similar to Context, but additionally prints the vari-
-              ables currently visible where the exception happened (shortening
-              their  strings if too long).  This can potentially be very slow,
-              if you happen to have a huge data structure whose string  repre-
-              sentation  is  complex  to compute.  Your computer may appear to
-              freeze for a while with cpu usage at 100%.  If this occurs,  you
-              can cancel the traceback with Ctrl-C (maybe hitting it more than
-              once).
-
-
-EMBEDDING
-       It is possible to start an IPython instance inside your own Python pro-
-       grams.  In the documentation example files there are some illustrations
-       on how to do this.
-
-       This feature allows you to evalutate  dynamically  the  state  of  your
-       code,  operate  with  your  variables, analyze them, etc.  Note however
-       that any changes you make to values while in the shell do NOT propagate
-       back  to  the running code, so it is safe to modify your values because
-       you won't break your code in bizarre ways by doing so.
-"""
+    introspection, easier configuration, command completion, access to the
+    system shell and more.  IPython can also be embedded in running programs.
 
-cmd_line_usage = __doc__
+    If invoked with no options, it executes all the files listed in sequence
+    and exits, use -i to enter interactive mode after running the files.  Files
+    ending in .py will be treated as normal Python, but files ending in .ipy
+    can contain special IPython syntax (magic commands, shell expansions, etc.)
+
+    Please note that some of the configuration options are not available at the
+    command line, simply because they are not practical here. Look into your
+    ipython_config.py configuration file for details on those.
+
+    This file typically installed in the $HOME/.ipython directory.  For Windows
+    users, $HOME resolves to C:\\Documents and Settings\\YourUserName in most
+    instances.
+
+    In IPython's documentation, we will refer to this directory as IPYTHON_DIR,
+    you can change its default location by setting any path you want in this
+    environment variable.
+
+    For more information, see the manual available in HTML and PDF in your
+    installation, or online at http://ipython.scipy.org.
+"""
 
-#---------------------------------------------------------------------------
 interactive_usage = """
 IPython -- An enhanced Interactive Python
 =========================================

IPython/extensions/pretty.py

@@ -128,13 +128,15 b' class PrettyResultDisplay(Component):'
 #-----------------------------------------------------------------------------
 
 
-def load_ipython_extension(ip):
+def load_ipython_extension(ip=None):
     """Load the extension in IPython as a hook."""
+    if ip is None: ip = get_ipython()
     global _loaded
     if not _loaded:
         prd = PrettyResultDisplay(ip, name='pretty_result_display')
         ip.set_hook('result_display', prd, priority=99)
         _loaded = True
+        return prd
 
 def unload_ipython_extension(ip):
     """Unload the extension."""
@@ -163,60 +165,3 b' def dtype_pprinter(obj, p, cycle):'
                     p.breakable()
                 p.pretty(field)
             p.end_group(7, '])')
-
-
-#-----------------------------------------------------------------------------
-# Tests
-#-----------------------------------------------------------------------------
-
-
-def test_pretty():
-    """
-    In [1]: from IPython.extensions import ipy_pretty
-
-    In [2]: ipy_pretty.activate()
-
-    In [3]: class A(object):
-       ...:     def __repr__(self):
-       ...:         return 'A()'
-       ...:     
-       ...:     
-
-    In [4]: a = A()
-
-    In [5]: a
-    Out[5]: A()
-
-    In [6]: def a_pretty_printer(obj, p, cycle):
-       ...:     p.text('<A>')
-       ...:     
-       ...:     
-
-    In [7]: ipy_pretty.for_type(A, a_pretty_printer)
-
-    In [8]: a
-    Out[8]: <A>
-
-    In [9]: class B(object):
-       ...:     def __repr__(self):
-       ...:         return 'B()'
-       ...:     
-       ...:     
-
-    In [10]: B.__module__, B.__name__
-    Out[10]: ('__main__', 'B')
-
-    In [11]: def b_pretty_printer(obj, p, cycle):
-       ....:     p.text('<B>')
-       ....:     
-       ....:     
-
-    In [12]: ipy_pretty.for_type_by_name('__main__', 'B', b_pretty_printer)
-
-    In [13]: b = B()
-
-    In [14]: b
-    Out[14]: <B>
-    """
-    assert False, "This should only be doctested, not run."
-

IPython/extensions/tests/test_pretty.py

@@ -15,21 +15,20 b' Simple tests for :mod:`IPython.extensions.pretty`.'
 # Imports
 #-----------------------------------------------------------------------------
 
-import sys
 from unittest import TestCase
 
 from IPython.core.component import Component, masquerade_as
 from IPython.core.iplib import InteractiveShell
 from IPython.extensions import pretty as pretty_ext
 from IPython.external import pretty
-
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
 from IPython.utils.traitlets import Bool
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class InteractiveShellStub(Component):
     pprint = Bool(True)
 
@@ -43,9 +42,11 b' class TestPrettyResultDisplay(TestCase):'
 
     def setUp(self):
         self.ip = InteractiveShellStub(None)
-        # This allows our stub to be retrieved instead of the real InteractiveShell
+        # This allows our stub to be retrieved instead of the real
+        # InteractiveShell
         masquerade_as(self.ip, InteractiveShell)
-        self.prd = pretty_ext.PrettyResultDisplay(self.ip, name='pretty_result_display')    
+        self.prd = pretty_ext.PrettyResultDisplay(self.ip,
+                                                  name='pretty_result_display')
 
     def test_for_type(self):
         self.prd.for_type(A, a_pprinter)
@@ -53,4 +54,48 b' class TestPrettyResultDisplay(TestCase):'
         result = pretty.pretty(a)
         self.assertEquals(result, "<A>")
 
+ipy_src = """
+class A(object):
+    def __repr__(self):
+        return 'A()'
+    
+class B(object):
+    def __repr__(self):
+        return 'B()'
+ 
+a = A()
+b = B()
+
+def a_pretty_printer(obj, p, cycle):
+    p.text('<A>')
+
+def b_pretty_printer(obj, p, cycle):
+    p.text('<B>')
+
+
+a
+b
+
+ip = get_ipython()
+prd = ip.load_extension('pretty')
+prd.for_type(A, a_pretty_printer)
+prd.for_type_by_name(B.__module__, B.__name__, b_pretty_printer)
+
+a
+b
+"""
+ipy_out = """
+A()
+B()
+<A>
+<B>
+"""
+
+class TestPrettyInteractively(tt.TempFileMixin):
 
+    # XXX Unfortunately, ipexec_validate fails under win32.  If someone helps
+    # us write a win32-compatible version, we can reactivate this test.
+    @dec.skip_win32
+    def test_printers(self):
+        self.mktmp(ipy_src, '.ipy')
+        tt.ipexec_validate(self.fname, ipy_out)

IPython/external/argparse.py

@@ -2,25 +2,17 b''
 
 # Copyright © 2006-2009 Steven J. Bethard <[e-mail]>.
 #
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
 #
-#  * Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
 
 """Command-line parsing library
 
@@ -83,7 +75,7 b' considered public as object names -- the API of the formatter objects is'
 still considered an implementation detail.)
 """
 
-__version__ = '1.0.1'
+__version__ = '1.1a1'
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -92,7 +84,7 b' __all__ = ['
     'FileType',
     'HelpFormatter',
     'RawDescriptionHelpFormatter',
-    'RawTextHelpFormatter'
+    'RawTextHelpFormatter',
     'ArgumentDefaultsHelpFormatter',
 ]
 
@@ -126,6 +118,10 b' except NameError:'
             result.reverse()
         return result
 
+
+def _callable(obj):
+    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')
+
 # silence Python 2.6 buggy warnings about Exception.message
 if _sys.version_info[:2] == (2, 6):
     import warnings
@@ -141,7 +137,8 b" SUPPRESS = '==SUPPRESS=='"
 OPTIONAL = '?'
 ZERO_OR_MORE = '*'
 ONE_OR_MORE = '+'
-PARSER = '==PARSER=='
+PARSER = 'A...'
+REMAINDER = '...'
 
 # =============================
 # Utility functions and classes
@@ -508,6 +505,8 b' class HelpFormatter(object):'
         return text
 
     def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
         text_width = self._width - self._current_indent
         indent = ' ' * self._current_indent
         return self._fill_text(text, text_width, indent) + '\n\n'
@@ -608,7 +607,9 b' class HelpFormatter(object):'
             result = '[%s [%s ...]]' % get_metavar(2)
         elif action.nargs == ONE_OR_MORE:
             result = '%s [%s ...]' % get_metavar(2)
-        elif action.nargs is PARSER:
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
             result = '%s ...' % get_metavar(1)
         else:
             formats = ['%s' for _ in range(action.nargs)]
@@ -724,6 +725,12 b' class ArgumentError(Exception):'
         return format % dict(message=self.message,
                              argument_name=self.argument_name)
 
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
 # ==============
 # Action classes
 # ==============
@@ -1018,6 +1025,7 b' class _VersionAction(Action):'
 
     def __init__(self,
                  option_strings,
+                 version=None,
                  dest=SUPPRESS,
                  default=SUPPRESS,
                  help=None):
@@ -1027,10 +1035,15 b' class _VersionAction(Action):'
             default=default,
             nargs=0,
             help=help)
+        self.version = version
 
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.print_version()
-        parser.exit()
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
 
 
 class _SubParsersAction(Action):
@@ -1156,8 +1169,7 b' class Namespace(_AttributeHolder):'
     """
 
     def __init__(self, **kwargs):
-        for name in kwargs:
-            setattr(self, name, kwargs[name])
+        self.__dict__.update(**kwargs)
 
     def __eq__(self, other):
         return vars(self) == vars(other)
@@ -1165,6 +1177,9 b' class Namespace(_AttributeHolder):'
     def __ne__(self, other):
         return not (self == other)
 
+    def __contains__(self, key):
+        return key in self.__dict__
+
 
 class _ActionsContainer(object):
 
@@ -1211,7 +1226,7 b' class _ActionsContainer(object):'
         self._defaults = {}
 
         # determines whether an "option" looks like a negative number
-        self._negative_number_matcher = _re.compile(r'^-\d+|-\d*.\d+$')
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
 
         # whether or not there are any optionals that look like negative
         # numbers -- uses a list so it can be shared and edited
@@ -1228,7 +1243,7 b' class _ActionsContainer(object):'
         return self._registries[registry_name].get(value, default)
 
     # ==================================
-    # Namespace default settings methods
+    # Namespace default accessor methods
     # ==================================
     def set_defaults(self, **kwargs):
         self._defaults.update(kwargs)
@@ -1239,6 +1254,13 b' class _ActionsContainer(object):'
             if action.dest in kwargs:
                 action.default = kwargs[action.dest]
 
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
     # =======================
     # Adding argument actions
     # =======================
@@ -1253,6 +1275,8 b' class _ActionsContainer(object):'
         # argument
         chars = self.prefix_chars
         if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
             kwargs = self._get_positional_kwargs(*args, **kwargs)
 
         # otherwise, we're adding an optional argument
@@ -1269,6 +1293,8 b' class _ActionsContainer(object):'
 
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
+        if not _callable(action_class):
+            raise ValueError('unknown action "%s"' % action_class)
         action = action_class(**kwargs)
         return self._add_action(action)
 
@@ -1578,6 +1604,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
         if self.version:
             self.add_argument(
                 '-v', '--version', action='version', default=SUPPRESS,
+                version=self.version,
                 help=_("show program's version number and exit"))
 
         # add parent arguments and defaults
@@ -2011,6 +2038,13 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
             action = self._option_string_actions[arg_string]
             return action, arg_string, None
 
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
         # search through all possible prefixes of the option string
         # and all actions in the parser for possible interpretations
         option_tuples = self._get_option_tuples(arg_string)
@@ -2108,8 +2142,12 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
         elif nargs == ONE_OR_MORE:
             nargs_pattern = '(-*A[A-]*)'
 
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
         # allow one argument followed by any number of options or arguments
-        elif nargs is PARSER:
+        elif nargs == PARSER:
             nargs_pattern = '(-*A[-AO]*)'
 
         # all others should be integers
@@ -2129,7 +2167,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # ========================
     def _get_values(self, action, arg_strings):
         # for everything but PARSER args, strip out '--'
-        if action.nargs is not PARSER:
+        if action.nargs not in [PARSER, REMAINDER]:
             arg_strings = [s for s in arg_strings if s != '--']
 
         # optional argument produces a default when not present
@@ -2158,8 +2196,12 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
             value = self._get_value(action, arg_string)
             self._check_value(action, value)
 
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
         # PARSER arguments convert all values, but check only the first
-        elif action.nargs is PARSER:
+        elif action.nargs == PARSER:
             value = [self._get_value(action, v) for v in arg_strings]
             self._check_value(action, value[0])
 
@@ -2174,8 +2216,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
 
     def _get_value(self, action, arg_string):
         type_func = self._registry_get('type', action.type, action.type)
-        if not hasattr(type_func, '__call__'):
-            if not hasattr(type_func, '__bases__'): # classic classes
+        if not _callable(type_func):
                 msg = _('%r is not callable')
                 raise ArgumentError(action, msg % type_func)
 
@@ -2183,7 +2224,13 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
         try:
             result = type_func(arg_string)
 
-        # TypeErrors or ValueErrors indicate errors
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
         except (TypeError, ValueError):
             name = getattr(action.type, '__name__', repr(action.type))
             msg = _('invalid %s value: %r')
@@ -2243,9 +2290,13 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # Help-printing methods
     # =====================
     def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
         self._print_message(self.format_usage(), file)
 
     def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
         self._print_message(self.format_help(), file)
 
     def print_version(self, file=None):
@@ -2262,7 +2313,7 b' class ArgumentParser(_AttributeHolder, _ActionsContainer):'
     # ===============
     def exit(self, status=0, message=None):
         if message:
-            _sys.stderr.write(message)
+            self._print_message(message, _sys.stderr)
         _sys.exit(status)
 
     def error(self, message):

IPython/frontend/prefilterfrontend.py

@@ -9,7 +9,6 b' functionnality is abstracted out of ipython0 in reusable functions and'
 is added on the interpreter. This class can be a used to guide this
 refactoring.
 """
-__docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
@@ -27,7 +26,7 b' import os'
 import re
 import __builtin__
 
-from IPython.core.ipmaker import make_IPython
+from IPython.core.ipapp import IPythonApp
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
@@ -36,6 +35,9 b' from IPython.utils.genutils import Term'
 
 from linefrontendbase import LineFrontEndBase, common_prefix
 
+#-----------------------------------------------------------------------------
+# Utility functions
+#-----------------------------------------------------------------------------
 
 def mk_system_call(system_call_function, command):
     """ given a os.system replacement, and a leading string command,
@@ -74,13 +76,7 b' class PrefilterFrontEnd(LineFrontEndBase):'
               Used as the instance's argv value.  If not given, [] is used.
         """
         if argv is None:
-            argv = []
-        # This is a hack to avoid the IPython exception hook to trigger
-        # on exceptions (https://bugs.launchpad.net/bugs/337105)
-        # XXX: This is horrible: module-leve monkey patching -> side
-        # effects.
-        from IPython.core import iplib
-        iplib.InteractiveShell.isthreaded = True
+            argv = ['--no-banner']
 
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
@@ -101,12 +97,15 b' class PrefilterFrontEnd(LineFrontEndBase):'
                 return '\n'
             old_rawinput = __builtin__.raw_input
             __builtin__.raw_input = my_rawinput
-            # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=argv, 
+            ipython0 = IPythonApp(argv=argv, 
                                     user_ns=self.shell.user_ns,
                                     user_global_ns=self.shell.user_global_ns)
+            ipython0.initialize()
             __builtin__.raw_input = old_rawinput
-        self.ipython0 = ipython0
+        # XXX This will need to be updated as we refactor things, but for now,
+        # the .shell attribute of the ipythonapp instance conforms to the old
+        # api.
+        self.ipython0 = ipython0.shell
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))
@@ -125,7 +124,7 b' class PrefilterFrontEnd(LineFrontEndBase):'
 
 
         if not 'banner' in kwargs and self.banner is None:
-            self.banner = self.ipython0.BANNER
+            self.banner = self.ipython0.banner
 
         # FIXME: __init__ and start should be two different steps
         self.start()
@@ -204,7 +203,6 b' class PrefilterFrontEnd(LineFrontEndBase):'
             line = line[:-len(word)] + prefix
         return line, completions 
  
-    
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface 
     #--------------------------------------------------------------------------
@@ -220,23 +218,11 b' class PrefilterFrontEnd(LineFrontEndBase):'
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
         
-        ## try:
-        ##     for line in input_string.split('\n'):
-        ##         filtered_lines.append(
-        ##                 self.ipython0.prefilter(line, False).rstrip())
-        ## except:
-        ##     # XXX: probably not the right thing to do.
-        ##     self.ipython0.showsyntaxerror()
-        ##     self.after_execute()
-        ## finally:
-        ##     self.release_output()
-
-
         try:
             try:
                 for line in input_string.split('\n'):
-                    filtered_lines.append(
-                            self.ipython0.prefilter(line, False).rstrip())
+                    pf = self.ipython0.prefilter_manager.prefilter_lines
+                    filtered_lines.append(pf(line, False).rstrip())
             except:
                 # XXX: probably not the right thing to do.
                 self.ipython0.showsyntaxerror()
@@ -244,13 +230,10 b' class PrefilterFrontEnd(LineFrontEndBase):'
         finally:
             self.release_output()
 
-
-
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string
 
-
     #--------------------------------------------------------------------------
     # PrefilterFrontEnd interface 
     #--------------------------------------------------------------------------
@@ -261,13 +244,11 b' class PrefilterFrontEnd(LineFrontEndBase):'
         """
         return os.system(command_string)
 
-
     def do_exit(self):
         """ Exit the shell, cleanup and save the history.
         """
         self.ipython0.atexit_operations()
 
-
     def _get_completion_text(self, line):
         """ Returns the text to be completed by breaking the line at specified
         delimiters.
@@ -281,4 +262,3 b' class PrefilterFrontEnd(LineFrontEndBase):'
         complete_sep = re.compile(expression)
         text = complete_sep.split(line)[-1]
         return text
-

IPython/frontend/tests/test_prefilterfrontend.py

@@ -21,8 +21,11 b' from nose.tools import assert_equal'
 
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 from IPython.core.ipapi import get as get_ipython0
-from IPython.testing.plugin.ipdoctest import default_argv
+from IPython.testing.tools import default_argv
 
+#-----------------------------------------------------------------------------
+# Support utilities
+#-----------------------------------------------------------------------------
 
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
@@ -85,14 +88,14 b' def isolate_ipython0(func):'
                 del user_ns[k]
             for k in new_globals:
                 del user_global_ns[k]
-        # Undo the hack at creation of PrefilterFrontEnd
-        from IPython.core import iplib
-        iplib.InteractiveShell.isthreaded = False
         return out
 
     my_func.__name__ = func.__name__
     return my_func
 
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
 
 @isolate_ipython0
 def test_execution():
@@ -166,7 +169,7 b' def test_magic():'
     f.input_buffer += '%who'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert_equal(out_value, 'Interactive namespace is empty.\n')
+    assert_equal(out_value, 'In\tOut\tget_ipython\t\n')
 
 
 @isolate_ipython0

IPython/frontend/wx/ipythonx.py

@@ -6,11 +6,10 b' ipython.'
 try:
     import wx
 except ImportError, e:
-    e.message = """%s
+    e.args[0] = """%s
 ________________________________________________________________________________
 You need wxPython to run this application.
-""" % e.message
-    e.args = (e.message, ) + e.args[1:]
+""" % e.args[0]
     raise e
 
 from wx_frontend import WxController

IPython/gui/wx/ipshell_nonblocking.py

@@ -23,13 +23,9 b' import os'
 import locale
 from thread_ex import ThreadEx
 
-try:
     import IPython
+from IPython.core import iplib, ipapp
     from IPython.utils import genutils
-    from IPython.core import iplib
-except Exception,e:
-    print "Error importing IPython (%s)" % str(e)
-    raise Exception, e
 
 ##############################################################################
 class _Helper(object):
@@ -156,11 +152,16 b' class NonBlockingIPShell(object):'
         #Hack to save sys.displayhook, because ipython seems to overwrite it...
         self.sys_displayhook_ori = sys.displayhook
         
-        self._IP = IPython.shell.make_IPython(
-                                    argv,user_ns=user_ns,
-                                    user_global_ns=user_global_ns,
-                                    embedded=True,
-                                    shell_class=IPython.shell.InteractiveShell)
+        ipython0 = ipapp.IPythonApp(argv,user_ns=user_ns,
+                                    user_global_ns=user_global_ns)
+        ipython0.initialize()
+        self._IP = ipython0.shell
+        
+        ## self._IP = IPython.shell.make_IPython(
+        ##                             argv,user_ns=user_ns,
+        ##                             user_global_ns=user_global_ns,
+        ##                             embedded=True,
+        ##                             shell_class=IPython.shell.InteractiveShell)
 
         #we save ipython0 displayhook and we restore sys.displayhook
         self.displayhook = sys.displayhook
@@ -273,7 +274,7 b' class NonBlockingIPShell(object):'
         @return: The banner string.
         @rtype: string
         """
-        return self._IP.BANNER
+        return self._IP.banner
     
     def get_prompt_count(self):
         """
@@ -470,7 +471,7 b' class NonBlockingIPShell(object):'
         '''
 
         orig_stdout = sys.stdout
-        sys.stdout = IPython.shell.Term.cout
+        sys.stdout = genutils.Term.cout
         #self.sys_displayhook_ori = sys.displayhook
         #sys.displayhook = self.displayhook
         

IPython/gui/wx/wxIPython.py

@@ -109,7 +109,7 b' class MyFrame(wx.Frame):'
 
     def optionSave(self, name, value):
         ip = get()
-        path = ip.config.IPYTHONDIR
+        path = ip.ipython_dir
         opt = open(path + '/options.conf','w')
 
         try:
@@ -126,7 +126,7 b' class MyFrame(wx.Frame):'
     def optionLoad(self):
         try:
             ip = get()
-            path = ip.config.IPYTHONDIR
+            path = ip.ipython_dir
             opt = open(path + '/options.conf','r')
             lines = opt.readlines()
             opt.close()

IPython/kernel/asyncclient.py

@@ -1,3 +1,4 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """Asynchronous clients for the IPython controller.
@@ -9,32 +10,32 b' deferreds to the result.'
 
 The main methods are are `get_*_client` and `get_client`.
 """
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from IPython.kernel import codeutil
-from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.clientconnector import (
+    AsyncClientConnector,
+    AsyncCluster
+)
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask
 from IPython.kernel.error import CompositeError
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Code
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-_client_tub = ClientConnector()
+_client_tub = AsyncClientConnector()
 get_multiengine_client = _client_tub.get_multiengine_client
 get_task_client = _client_tub.get_task_client
 get_client = _client_tub.get_client

IPython/kernel/client.py

@@ -1,3 +1,4 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """This module contains blocking clients for the controller interfaces.
@@ -15,33 +16,36 b' The main classes in this module are:'
     * CompositeError
 """
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
+from cStringIO import StringIO
 import sys
+import warnings
 
 # from IPython.utils import growl
 # growl.start("IPython1 Client")
 
 
 from twisted.internet import reactor
-from IPython.kernel.clientconnector import ClientConnector
+from twisted.internet.error import PotentialZombieWarning
+from twisted.python import log
+
+from IPython.kernel.clientconnector import ClientConnector, Cluster
 from IPython.kernel.twistedutil import ReactorInThread
 from IPython.kernel.twistedutil import blockingCallFromThread
 
 # These enable various things 
 from IPython.kernel import codeutil
-import IPython.kernel.magic
+# import IPython.kernel.magic
 
 # Other things that the user will need
 from IPython.kernel.task import MapTask, StringTask
@@ -51,46 +55,34 b' from IPython.kernel.error import CompositeError'
 # Code
 #-------------------------------------------------------------------------------
 
-_client_tub = ClientConnector()
-
-
-def get_multiengine_client(furl_or_file=''):
-    """Get the blocking MultiEngine client.
-    
-    :Parameters:
-        furl_or_file : str
-            A furl or a filename containing a furl.  If empty, the
-            default furl_file will be used
-            
-    :Returns:
-        The connected MultiEngineClient instance
-    """
-    client = blockingCallFromThread(_client_tub.get_multiengine_client, 
-        furl_or_file)
-    return client.adapt_to_blocking_client()
-
-def get_task_client(furl_or_file=''):
-    """Get the blocking Task client.
-    
-    :Parameters:
-        furl_or_file : str
-            A furl or a filename containing a furl.  If empty, the
-            default furl_file will be used
-            
-    :Returns:
-        The connected TaskClient instance
-    """
-    client = blockingCallFromThread(_client_tub.get_task_client, 
-        furl_or_file)
-    return client.adapt_to_blocking_client()
+warnings.simplefilter('ignore', PotentialZombieWarning)
 
+_client_tub = ClientConnector()
 
+get_multiengine_client = _client_tub.get_multiengine_client
+get_task_client = _client_tub.get_task_client
 MultiEngineClient = get_multiengine_client
 TaskClient = get_task_client
 
-
+# This isn't great.  I should probably set this up in the ReactorInThread
+# class below.  But, it does work for now.
+log.startLogging(sys.stdout, setStdout=0)
 
 # Now we start the reactor in a thread
 rit = ReactorInThread()
 rit.setDaemon(True)
 rit.start()
+
+
+
+
+__all__ = [
+    'MapTask',
+    'StringTask',
+    'MultiEngineClient',
+    'TaskClient',
+    'CompositeError',
+    'get_task_client',
+    'get_multiengine_client',
+    'Cluster'
+]

IPython/kernel/clientconnector.py

@@ -1,41 +1,61 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
-"""A class for handling client connections to the controller."""
+"""Facilities for handling client connections to the controller."""
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
-
-from twisted.internet import defer
+#-----------------------------------------------------------------------------
 
-from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from __future__ import with_statement
+import os
 
-from IPython.kernel.config import config_manager as kernel_config_manager
+from IPython.kernel.fcutil import (
+    Tub,
+    find_furl,
+    is_valid_furl_or_file,
+    validate_furl_or_file,
+    FURLError
+)
+from IPython.kernel.clusterdir import ClusterDir, ClusterDirError
+from IPython.kernel.launcher import IPClusterLauncher
+from IPython.kernel.twistedutil import (
+    gatherBoth, 
+    make_deferred,
+    blockingCallFromThread, 
+    sleep_deferred
+)
 from IPython.utils.importstring import import_item
-from IPython.kernel.fcutil import find_furl
+from IPython.utils.genutils import get_ipython_dir
 
-co = kernel_config_manager.get_config_obj()
-client_co = co['client']
+from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.python import failure, log
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The ClientConnector class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-class ClientConnector(object):
-    """
-    This class gets remote references from furls and returns the wrapped clients.
+DELAY = 0.2
+MAX_TRIES = 9
+
+
+class ClientConnectorError(Exception):
+    pass
     
-    This class is also used in `client.py` and `asyncclient.py` to create 
-    a single per client-process Tub.
+
+class AsyncClientConnector(object):
+    """A class for getting remote references and clients from furls.
+
+    This start a single :class:`Tub` for all remote reference and caches
+    references.
     """
     
     def __init__(self):
@@ -43,100 +63,206 b' class ClientConnector(object):'
         self.tub = Tub()
         self.tub.startService()
     
-    def get_reference(self, furl_or_file):
+    def _find_furl(self, profile='default', cluster_dir=None, 
+                   furl_or_file=None, furl_file_name=None,
+                   ipython_dir=None):
+        """Find a FURL file by profile+ipython_dir or cluster dir.
+
+        This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception 
+        if a FURL file can't be found.
         """
-        Get a remote reference using a furl or a file containing a furl.
+        # Try by furl_or_file
+        if furl_or_file is not None:
+            validate_furl_or_file(furl_or_file)
+            return furl_or_file
+
+        if furl_file_name is None:
+            raise FURLError('A furl_file_name must be provided')
+
+        # Try by cluster_dir
+        if cluster_dir is not None:
+            cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+            sdir = cluster_dir_obj.security_dir
+            furl_file = os.path.join(sdir, furl_file_name)
+            validate_furl_or_file(furl_file)
+            return furl_file
+
+        # Try by profile
+        if ipython_dir is None:
+            ipython_dir = get_ipython_dir()
+        if profile is not None:
+            cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                ipython_dir, profile)
+            sdir = cluster_dir_obj.security_dir
+            furl_file = os.path.join(sdir, furl_file_name)
+            validate_furl_or_file(furl_file)
+            return furl_file
+
+        raise FURLError('Could not find a valid FURL file.')
+
+    def get_reference(self, furl_or_file):
+        """Get a remote reference using a furl or a file containing a furl.
         
         Remote references are cached locally so once a remote reference
         has been retrieved for a given furl, the cached version is 
         returned.
         
-        :Parameters:
+        Parameters
+        ----------
             furl_or_file : str
-                A furl or a filename containing a furl
+            A furl or a filename containing a furl. This should already be
+            validated, but might not yet exist.
         
-        :Returns:
+        Returns
+        -------
             A deferred to a remote reference
         """
-        furl = find_furl(furl_or_file)
+        furl = furl_or_file
         if furl in self._remote_refs:
             d = defer.succeed(self._remote_refs[furl])
         else:
             d = self.tub.getReference(furl)
-            d.addCallback(self.save_ref, furl)
+            d.addCallback(self._save_ref, furl)
         return d
         
-    def save_ref(self, ref, furl):
-        """
-        Cache a remote reference by its furl.
-        """
+    def _save_ref(self, ref, furl):
+        """Cache a remote reference by its furl."""
         self._remote_refs[furl] = ref
         return ref
         
-    def get_task_client(self, furl_or_file=''):
-        """
-        Get the task controller client.
+    def get_task_client(self, profile='default', cluster_dir=None,
+                        furl_or_file=None, ipython_dir=None,
+                        delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task controller client.
         
-        This method is a simple wrapper around `get_client` that allow
-        `furl_or_file` to be empty, in which case, the furls is taken
-        from the default furl file given in the configuration.
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Parameters:
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
             furl_or_file : str
-                A furl or a filename containing a furl.  If empty, the
-                default furl_file will be used
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
                 
-        :Returns:
-            A deferred to the actual client class
+        Returns
+        -------
+        A deferred to the actual client class.
         """
-        task_co = client_co['client_interfaces']['task']
-        if furl_or_file:
-            ff = furl_or_file
-        else:
-            ff = task_co['furl_file']
-        return self.get_client(ff)
+        return self.get_client(
+            profile, cluster_dir, furl_or_file, 
+            'ipcontroller-tc.furl', ipython_dir,
+            delay, max_tries
+        )
 
-    def get_multiengine_client(self, furl_or_file=''):
-        """
-        Get the multiengine controller client.
+    def get_multiengine_client(self, profile='default', cluster_dir=None,
+                               furl_or_file=None, ipython_dir=None,
+                               delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine controller client.
         
-        This method is a simple wrapper around `get_client` that allow
-        `furl_or_file` to be empty, in which case, the furls is taken
-        from the default furl file given in the configuration.
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Parameters:
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
             furl_or_file : str
-                A furl or a filename containing a furl.  If empty, the
-                default furl_file will be used
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
                 
-        :Returns:
-            A deferred to the actual client class
+        Returns
+        -------
+        A deferred to the actual client class.
         """
-        task_co = client_co['client_interfaces']['multiengine']
-        if furl_or_file:
-            ff = furl_or_file
-        else:
-            ff = task_co['furl_file']
-        return self.get_client(ff)
+        return self.get_client(
+            profile, cluster_dir, furl_or_file, 
+            'ipcontroller-mec.furl', ipython_dir,
+            delay, max_tries
+        )
     
-    def get_client(self, furl_or_file):
-        """
-        Get a remote reference and wrap it in a client by furl.
+    def get_client(self, profile='default', cluster_dir=None,
+                   furl_or_file=None, furl_file_name=None, ipython_dir=None,
+                   delay=DELAY, max_tries=MAX_TRIES):
+        """Get a remote reference and wrap it in a client by furl.
         
-        This method first gets a remote reference and then calls its 
-        `get_client_name` method to find the apprpriate client class
-        that should be used to wrap the remote reference.
+        This method is a simple wrapper around `get_client` that passes in
+        the default name of the task client FURL file.  Usually only
+        the ``profile`` option will be needed.  If a FURL file can't be
+        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
         
-        :Parameters:
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
             furl_or_file : str
-                A furl or a filename containing a furl
+            A furl or a filename containing a FURL. This is useful if you 
+            simply know the location of the FURL file.
+        furl_file_name : str
+            The filename (not the full path) of the FURL. This must be
+            provided if ``furl_or_file`` is not.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
         
-        :Returns:
-            A deferred to the actual client class
+        Returns
+        -------
+        A deferred to the actual client class. Or a failure to a 
+        :exc:`FURLError`.
         """
-        furl = find_furl(furl_or_file)
-        d = self.get_reference(furl)
-        def wrap_remote_reference(rr):
+        try:
+            furl_file = self._find_furl(
+                profile, cluster_dir, furl_or_file,
+                furl_file_name, ipython_dir
+            )
+        except FURLError:
+            return defer.fail(failure.Failure())
+
+        def _wrap_remote_reference(rr):
             d = rr.callRemote('get_client_name')
             d.addCallback(lambda name: import_item(name))
             def adapt(client_interface):
@@ -146,5 +272,502 b' class ClientConnector(object):'
             d.addCallback(adapt)
 
             return d
-        d.addCallback(wrap_remote_reference)
+
+        d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
+        d.addCallback(_wrap_remote_reference)
+        d.addErrback(self._handle_error, furl_file)
+        return d
+
+    def _handle_error(self, f, furl_file):
+        raise ClientConnectorError('Could not connect to the controller '
+            'using the FURL file. This usually means that i) the controller '
+            'was not started or ii) a firewall was blocking the client from '
+            'connecting to the controller: %s' % furl_file)
+
+    @inlineCallbacks
+    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
+        """Try to connect to the controller with retry logic."""
+        if attempt < max_tries:
+            log.msg("Connecting [%r]" % attempt)
+            try:
+                self.furl = find_furl(furl_or_file)
+                # Uncomment this to see the FURL being tried.
+                # log.msg("FURL: %s" % self.furl)
+                rr = yield self.get_reference(self.furl)
+                log.msg("Connected: %s" % furl_or_file)
+            except:
+                if attempt==max_tries-1:
+                    # This will propagate the exception all the way to the top
+                    # where it can be handled.
+                    raise
+                else:
+                    yield sleep_deferred(delay)
+                    rr = yield self._try_to_connect(
+                        furl_or_file, 1.5*delay, max_tries, attempt+1
+                    )
+                    returnValue(rr)
+            else:
+                returnValue(rr)
+        else:
+            raise ClientConnectorError(
+                'Could not connect to controller, max_tries (%r) exceeded. '
+                'This usually means that i) the controller was not started, '
+                'or ii) a firewall was blocking the client from connecting '
+                'to the controller.' % max_tries
+            )
+
+
+class ClientConnector(object):
+    """A blocking version of a client connector.
+
+    This class creates a single :class:`Tub` instance and allows remote
+    references and client to be retrieved by their FURLs.  Remote references
+    are cached locally and FURL files can be found using profiles and cluster
+    directories.
+    """
+
+    def __init__(self):
+        self.async_cc = AsyncClientConnector()
+
+    def get_task_client(self, profile='default', cluster_dir=None,
+                        furl_or_file=None, ipython_dir=None,
+                        delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client.
+        
+        Usually only the ``profile`` option will be needed. If a FURL file
+        can't be found by its profile, use ``cluster_dir`` or
+        ``furl_or_file``.
+        
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        The task client instance.
+        """
+        client = blockingCallFromThread(
+            self.async_cc.get_task_client, profile, cluster_dir,
+            furl_or_file, ipython_dir, delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+    def get_multiengine_client(self, profile='default', cluster_dir=None,
+                               furl_or_file=None, ipython_dir=None,
+                               delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client.
+        
+        Usually only the ``profile`` option will be needed. If a FURL file
+        can't be found by its profile, use ``cluster_dir`` or
+        ``furl_or_file``.
+        
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        furl_or_file : str
+            A furl or a filename containing a FURLK. This is useful if you 
+            simply know the location of the FURL file.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+
+        Returns
+        -------
+        The multiengine client instance.
+        """
+        client = blockingCallFromThread(
+            self.async_cc.get_multiengine_client, profile, cluster_dir,
+            furl_or_file, ipython_dir, delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+    def get_client(self, profile='default', cluster_dir=None,
+                   furl_or_file=None, ipython_dir=None,
+                   delay=DELAY, max_tries=MAX_TRIES):
+        client = blockingCallFromThread(
+            self.async_cc.get_client, profile, cluster_dir,
+            furl_or_file, ipython_dir,
+            delay, max_tries
+        )
+        return client.adapt_to_blocking_client()
+
+
+class ClusterStateError(Exception):
+    pass
+
+
+class AsyncCluster(object):
+    """An class that wraps the :command:`ipcluster` script."""
+
+    def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
+                 auto_create=False, auto_stop=True):
+        """Create a class to manage an IPython cluster.
+
+        This class calls the :command:`ipcluster` command with the right
+        options to start an IPython cluster.  Typically a cluster directory
+        must be created (:command:`ipcluster create`) and configured before 
+        using this class. Configuration is done by editing the 
+        configuration files in the top level of the cluster directory.
+
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        auto_create : bool
+            Automatically create the cluster directory it is dones't exist.
+            This will usually only make sense if using a local cluster 
+            (default=False).
+        auto_stop : bool
+            Automatically stop the cluster when this instance is garbage 
+            collected (default=True).  This is useful if you want the cluster
+            to live beyond your current process. There is also an instance
+            attribute ``auto_stop`` to change this behavior.
+        """
+        self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create)
+        self.state = 'before'
+        self.launcher = None
+        self.client_connector = None
+        self.auto_stop = auto_stop
+
+    def __del__(self):
+        if self.auto_stop and self.state=='running':
+            print "Auto stopping the cluster..."
+            self.stop()
+
+    @property
+    def location(self):
+        if hasattr(self, 'cluster_dir_obj'):
+            return self.cluster_dir_obj.location
+        else:
+            return ''
+
+    @property
+    def running(self):
+        if self.state=='running':
+            return True
+        else:
+            return False
+
+    def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create):
+        if ipython_dir is None:
+            ipython_dir = get_ipython_dir()
+        if cluster_dir is not None:
+            try:
+                self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
+            except ClusterDirError:
+                pass
+        if profile is not None:
+            try:
+                self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
+                    ipython_dir, profile)
+            except ClusterDirError:
+                pass
+        if auto_create or profile=='default':
+            # This should call 'ipcluster create --profile default
+            self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
+                ipython_dir, profile)
+        else:
+            raise ClusterDirError('Cluster dir not found.')
+
+    @make_deferred
+    def start(self, n=2):
+        """Start the IPython cluster with n engines.
+
+        Parameters
+        ----------
+        n : int
+            The number of engine to start.
+        """
+        # We might want to add logic to test if the cluster has started
+        # by another process....
+        if not self.state=='running':
+            self.launcher = IPClusterLauncher(os.getcwd())
+            self.launcher.ipcluster_n = n
+            self.launcher.ipcluster_subcommand = 'start'
+            d = self.launcher.start()
+            d.addCallback(self._handle_start)
+            return d
+        else:
+            raise ClusterStateError('Cluster is already running')
+
+    @make_deferred
+    def stop(self):
+        """Stop the IPython cluster if it is running."""
+        if self.state=='running':
+            d1 = self.launcher.observe_stop()
+            d1.addCallback(self._handle_stop)
+            d2 = self.launcher.stop()
+            return gatherBoth([d1, d2], consumeErrors=True)
+        else:
+            raise ClusterStateError("Cluster not running")
+
+    def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client for the running cluster.
+
+        If this fails, it means that the cluster has not finished starting.
+        Usually waiting a few seconds are re-trying will solve this.    
+        """
+        if self.client_connector is None:
+            self.client_connector = AsyncClientConnector()
+        return self.client_connector.get_multiengine_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client for the running cluster.
+
+        If this fails, it means that the cluster has not finished starting.
+        Usually waiting a few seconds are re-trying will solve this.    
+        """
+        if self.client_connector is None:
+            self.client_connector = AsyncClientConnector()
+        return self.client_connector.get_task_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_ipengine_logs(self):
+        return self.get_logs_by_name('ipengine')
+
+    def get_ipcontroller_logs(self):
+        return self.get_logs_by_name('ipcontroller')
+
+    def get_ipcluster_logs(self):
+        return self.get_logs_by_name('ipcluster')
+
+    def get_logs_by_name(self, name='ipcluster'):
+        log_dir = self.cluster_dir_obj.log_dir
+        logs = {}
+        for log in os.listdir(log_dir):
+            if log.startswith(name + '-') and log.endswith('.log'):
+                with open(os.path.join(log_dir, log), 'r') as f:
+                    logs[log] = f.read()
+        return logs
+
+    def get_logs(self):
+        d = self.get_ipcluster_logs()
+        d.update(self.get_ipengine_logs())
+        d.update(self.get_ipcontroller_logs())
         return d
+
+    def _handle_start(self, r):
+        self.state = 'running'
+
+    def _handle_stop(self, r):
+        self.state = 'after'
+
+
+class Cluster(object):
+
+
+    def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
+                 auto_create=False, auto_stop=True):
+        """Create a class to manage an IPython cluster.
+
+        This class calls the :command:`ipcluster` command with the right
+        options to start an IPython cluster.  Typically a cluster directory
+        must be created (:command:`ipcluster create`) and configured before 
+        using this class. Configuration is done by editing the 
+        configuration files in the top level of the cluster directory.
+
+        Parameters
+        ----------
+        profile : str
+            The name of a cluster directory profile (default="default"). The
+            cluster directory "cluster_<profile>" will be searched for
+            in ``os.getcwd()``, the ipython_dir and then in the directories
+            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
+        cluster_dir : str
+            The full path to a cluster directory.  This is useful if profiles
+            are not being used.
+        ipython_dir : str
+            The location of the ipython_dir if different from the default.
+            This is used if the cluster directory is being found by profile.
+        auto_create : bool
+            Automatically create the cluster directory it is dones't exist.
+            This will usually only make sense if using a local cluster 
+            (default=False).
+        auto_stop : bool
+            Automatically stop the cluster when this instance is garbage 
+            collected (default=True).  This is useful if you want the cluster
+            to live beyond your current process. There is also an instance
+            attribute ``auto_stop`` to change this behavior.
+        """
+        self.async_cluster = AsyncCluster(
+            profile, cluster_dir, ipython_dir, auto_create, auto_stop
+        )
+        self.cluster_dir_obj = self.async_cluster.cluster_dir_obj
+        self.client_connector = None
+
+    def _set_auto_stop(self, value):
+        self.async_cluster.auto_stop = value
+
+    def _get_auto_stop(self):
+        return self.async_cluster.auto_stop
+
+    auto_stop = property(_get_auto_stop, _set_auto_stop)
+
+    @property
+    def location(self):
+        return self.async_cluster.location
+
+    @property
+    def running(self):
+        return self.async_cluster.running
+
+    def start(self, n=2):
+        """Start the IPython cluster with n engines.
+
+        Parameters
+        ----------
+        n : int
+            The number of engine to start.
+        """
+        return blockingCallFromThread(self.async_cluster.start, n)
+
+    def stop(self):
+        """Stop the IPython cluster if it is running."""
+        return blockingCallFromThread(self.async_cluster.stop)
+
+    def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the multiengine client for the running cluster.
+
+        This will try to attempt to the controller multiple times. If this 
+        fails altogether, try looking at the following:
+        * Make sure the controller is starting properly by looking at its
+          log files.
+        * Make sure the controller is writing its FURL file in the location
+          expected by the client.
+        * Make sure a firewall on the controller's host is not blocking the
+          client from connecting.
+
+        Parameters
+        ----------
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+        """
+        if self.client_connector is None:
+            self.client_connector = ClientConnector()
+        return self.client_connector.get_multiengine_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
+        """Get the task client for the running cluster.
+
+        This will try to attempt to the controller multiple times. If this 
+        fails altogether, try looking at the following:
+        * Make sure the controller is starting properly by looking at its
+          log files.
+        * Make sure the controller is writing its FURL file in the location
+          expected by the client.
+        * Make sure a firewall on the controller's host is not blocking the
+          client from connecting.
+
+        Parameters
+        ----------
+        delay : float
+            The initial delay between re-connection attempts. Susequent delays
+            get longer according to ``delay[i] = 1.5*delay[i-1]``.
+        max_tries : int
+            The max number of re-connection attempts.
+        """
+        if self.client_connector is None:
+            self.client_connector = ClientConnector()
+        return self.client_connector.get_task_client(
+            cluster_dir=self.cluster_dir_obj.location,
+            delay=delay, max_tries=max_tries
+        )
+
+    def __repr__(self):
+        s = "<Cluster(running=%r, location=%s)" % (self.running, self.location)
+        return s
+
+    def get_logs_by_name(self, name='ipcluter'):
+        """Get a dict of logs by process name (ipcluster, ipengine, etc.)"""
+        return self.async_cluster.get_logs_by_name(name)
+
+    def get_ipengine_logs(self):
+        """Get a dict of logs for all engines in this cluster."""
+        return self.async_cluster.get_ipengine_logs()
+
+    def get_ipcontroller_logs(self):
+        """Get a dict of logs for the controller in this cluster."""
+        return self.async_cluster.get_ipcontroller_logs()
+
+    def get_ipcluster_logs(self):
+        """Get a dict of the ipcluster logs for this cluster."""
+        return self.async_cluster.get_ipcluster_logs()
+
+    def get_logs(self):
+        """Get a dict of all logs for this cluster."""
+        return self.async_cluster.get_logs()
+
+    def _print_logs(self, logs):
+        for k, v in logs.iteritems():
+            print "==================================="
+            print "Logfile: %s" % k
+            print "==================================="
+            print v
+            print
+
+    def print_ipengine_logs(self):
+        """Print the ipengine logs for this cluster to stdout."""
+        self._print_logs(self.get_ipengine_logs())
+
+    def print_ipcontroller_logs(self):
+        """Print the ipcontroller logs for this cluster to stdout."""
+        self._print_logs(self.get_ipcontroller_logs())
+
+    def print_ipcluster_logs(self):
+        """Print the ipcluster logs for this cluster to stdout."""
+        self._print_logs(self.get_ipcluster_logs())
+
+    def print_logs(self):
+        """Print all the logs for this cluster to stdout."""
+        self._print_logs(self.get_logs())
+

IPython/kernel/core/interpreter.py

@@ -211,7 +211,7 b' class Interpreter(object):'
 
     #### Public 'Interpreter' interface ########################################
 
-    def formatTraceback(self, et, ev, tb, message=''):
+    def format_traceback(self, et, ev, tb, message=''):
         """Put a formatted version of the traceback into value and reraise.
         
         When exceptions have to be sent over the network, the traceback 
@@ -375,7 +375,6 b' class Interpreter(object):'
             exec code in self.user_ns
             outflag = 0
         except SystemExit:
-            self.resetbuffer()
             self.traceback_trap.args = sys.exc_info()
         except:
             self.traceback_trap.args = sys.exc_info()
@@ -395,7 +394,7 b' class Interpreter(object):'
             python = self.translator(python)
         self.execute_python(python)
 
-    def getCommand(self, i=None):
+    def get_command(self, i=None):
         """Gets the ith message in the message_cache.
         
         This is implemented here for compatibility with the old ipython1 shell
@@ -492,7 +491,7 b' class Interpreter(object):'
             # somehow.  In the meantime, we'll just stop if there are two lines
             # of pure whitespace at the end.
             last_two = source.rsplit('\n',2)[-2:]
-            print 'last two:',last_two  # dbg
+            #print 'last two:',last_two  # dbg
             if len(last_two)==2 and all(s.isspace() for s in last_two):
                 return COMPLETE_INPUT,False
             else:

IPython/kernel/core/tests/test_redirectors.py

@@ -18,19 +18,19 b' __test__ = {}'
 
 from cStringIO import StringIO
 import os
+import sys
 
 from twisted.trial import unittest
 
-from IPython.testing import decorators_trial as dec
-
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class TestRedirector(unittest.TestCase):
 
-    @dec.skip_win32
+    if sys.platform == 'win32':
+        skip = True
+    
     def test_redirector(self):
         """Checks that the redirector can be used to do synchronous capture.
         """
@@ -51,7 +51,6 b' class TestRedirector(unittest.TestCase):'
         result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
         self.assertEquals(result1, result2)
 
-    @dec.skip_win32
     def test_redirector_output_trap(self):
         """Check the greedy trapping behavior of the traps.
         
@@ -59,7 +58,8 b' class TestRedirector(unittest.TestCase):'
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
         """
-        from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
+        from IPython.kernel.core.redirector_output_trap import \
+            RedirectorOutputTrap
         out = StringIO()
         trap = RedirectorOutputTrap(out.write, out.write)
         try:

IPython/kernel/engineconnector.py

@@ -1,32 +1,38 b''
+#!/usr/bin/env python
 # encoding: utf-8
 
 """A class that manages the engines connection to the controller."""
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 import os
 import cPickle as pickle
 
 from twisted.python import log, failure
 from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, returnValue
 
-from IPython.kernel.fcutil import find_furl
+from IPython.kernel.fcutil import find_furl, validate_furl_or_file
 from IPython.kernel.enginefc import IFCEngine
+from IPython.kernel.twistedutil import sleep_deferred, make_deferred
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # The ClientConnector class
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+
+class EngineConnectorError(Exception):
+    pass
+
 
 class EngineConnector(object):
     """Manage an engines connection to a controller.
@@ -39,7 +45,9 b' class EngineConnector(object):'
     def __init__(self, tub):
         self.tub = tub
         
-    def connect_to_controller(self, engine_service, furl_or_file):
+    @make_deferred
+    def connect_to_controller(self, engine_service, furl_or_file,
+                              delay=0.1, max_tries=10):
         """
         Make a connection to a controller specified by a furl.
         
@@ -53,29 +61,68 b' class EngineConnector(object):'
         `register_engine` method of the controller to actually register the 
         engine.
         
-        :Parameters:
+        This method will try to connect to the controller multiple times with
+        a delay in between.  Each time the FURL file is read anew.
+
+        Parameters
+        __________
             engine_service : IEngineBase
                 An instance of an `IEngineBase` implementer
             furl_or_file : str
                 A furl or a filename containing a furl
+        delay : float
+            The intial time to wait between connection attempts.  Subsequent
+            attempts have increasing delays.
+        max_tries : int
+            The maximum number of connection attempts.
+
+        Returns
+        -------
+        A deferred to the registered client or a failure to an error
+        like :exc:`FURLError`.
         """
         if not self.tub.running:
             self.tub.startService()
         self.engine_service = engine_service
         self.engine_reference = IFCEngine(self.engine_service)
+
+        validate_furl_or_file(furl_or_file)
+        d = self._try_to_connect(furl_or_file, delay, max_tries, attempt=0)
+        d.addCallback(self._register)
+        return d
+
+    @inlineCallbacks
+    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
+        """Try to connect to the controller with retry logic."""
+        if attempt < max_tries:
+            log.msg("Attempting to connect to controller [%r]: %s" % \
+                (attempt, furl_or_file))
         try:
             self.furl = find_furl(furl_or_file)
-        except ValueError:
-            return defer.fail(failure.Failure())
+                # Uncomment this to see the FURL being tried.
+                # log.msg("FURL: %s" % self.furl)
+                rr = yield self.tub.getReference(self.furl)
+            except:
+                if attempt==max_tries-1:
+                    # This will propagate the exception all the way to the top
+                    # where it can be handled.
+                    raise
         else:
-            d = self.tub.getReference(self.furl)
-            d.addCallbacks(self._register, self._log_failure)
-            return d
-    
-    def _log_failure(self, reason):
-        log.err('EngineConnector: engine registration failed:')
-        log.err(reason)
-        return reason
+                    yield sleep_deferred(delay)
+                    rr = yield self._try_to_connect(
+                        furl_or_file, 1.5*delay, max_tries, attempt+1
+                    )
+                    # rr becomes an int when there is a connection!!!
+                    returnValue(rr)
+            else:
+                returnValue(rr)
+        else:
+            raise EngineConnectorError(
+                'Could not connect to controller, max_tries (%r) exceeded. '
+                'This usually means that i) the controller was not started, '
+                'or ii) a firewall was blocking the engine from connecting '
+                'to the controller.' % max_tries
+            )
 
     def _register(self, rr):
         self.remote_ref = rr
@@ -83,7 +130,7 b' class EngineConnector(object):'
         desired_id = self.engine_service.id
         d = self.remote_ref.callRemote('register_engine', self.engine_reference, 
             desired_id, os.getpid(), pickle.dumps(self.engine_service.properties,2))
-        return d.addCallbacks(self._reference_sent, self._log_failure)
+        return d.addCallback(self._reference_sent)
 
     def _reference_sent(self, registration_dict):
         self.engine_service.id = registration_dict['id']

IPython/kernel/engineservice.py

@@ -387,7 +387,7 b' class EngineService(object, service.Service):'
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
-            et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
+            et,ev,tb = self.shell.format_traceback(et,ev,tb,msg)
             # Add another attribute
             ev._ipython_engine_info = msg
             f = failure.Failure(ev,et,None)
@@ -444,7 +444,7 b' class EngineService(object, service.Service):'
         msg = {'engineid':self.id,
                'method':'get_result',
                'args':[repr(i)]}
-        d = self.executeAndRaise(msg, self.shell.getCommand, i)
+        d = self.executeAndRaise(msg, self.shell.get_command, i)
         d.addCallback(self.addIDToResult)
         return d
     
@@ -877,7 +877,7 b' class ThreadedEngineService(EngineService):'
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
-            et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
+            et,ev,tb = self.shell.format_traceback(et,ev,tb,msg)
             # Add another attribute
             
             # Create a new exception with the new attributes

IPython/kernel/error.py

@@ -127,7 +127,9 b' class TaskRejectError(KernelError):'
 class CompositeError(KernelError):
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))
-        self.message = message
+        # Don't use pack_exception because it will conflict with the .message
+        # attribute that is being deprecated in 2.6 and beyond.
+        self.msg = message
         self.elist = elist
   
     def _get_engine_str(self, ev):
@@ -147,7 +149,7 b' class CompositeError(KernelError):'
             return tb
   
     def __str__(self):
-        s = str(self.message)
+        s = str(self.msg)
         for et, ev, etb in self.elist:
             engine_str = self._get_engine_str(ev)
             s = s + '\n' + engine_str + str(et.__name__) + ': ' + str(ev)

IPython/kernel/fcutil.py

@@ -1,27 +1,62 b''
+#!/usr/bin/env python
 # encoding: utf-8
+"""
+Foolscap related utilities.
+"""
 
-"""Foolscap related utilities."""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
+from __future__ import with_statement
 
 import os
+import tempfile
+
+from twisted.internet import reactor, defer
+from twisted.python import log
 
 from foolscap import Tub, UnauthenticatedTub
 
+from IPython.config.loader import Config
+
+from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
+
+from IPython.kernel.error import SecurityError
+
+from IPython.utils.traitlets import Int, Str, Bool, Instance
+from IPython.utils.importstring import import_item
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+# We do this so if a user doesn't have OpenSSL installed, it will try to use
+# an UnauthenticatedTub.  But, they will still run into problems if they
+# try to use encrypted furls.
+try:
+    import OpenSSL
+except:
+    Tub = UnauthenticatedTub
+    have_crypto = False
+else:
+    have_crypto = True
+
+
+class FURLError(Exception):
+    pass
+
+
 def check_furl_file_security(furl_file, secure):
     """Remove the old furl_file if changing security modes."""
-    
     if os.path.isfile(furl_file):
         f = open(furl_file, 'r')
         oldfurl = f.read().strip()
@@ -29,41 +64,210 b' def check_furl_file_security(furl_file, secure):'
         if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
             os.remove(furl_file)
 
+
 def is_secure(furl):
+    """Is the given FURL secure or not."""
     if is_valid(furl):
         if furl.startswith("pb://"):
             return True
         elif furl.startswith("pbu://"):
             return False
     else:
-        raise ValueError("invalid furl: %s" % furl)
+        raise FURLError("invalid FURL: %s" % furl)
+
 
 def is_valid(furl):
+    """Is the str a valid FURL or not."""
     if isinstance(furl, str):
         if furl.startswith("pb://") or furl.startswith("pbu://"):
             return True
     else:
         return False
 
+
 def find_furl(furl_or_file):
+    """Find, validate and return a FURL in a string or file."""
     if isinstance(furl_or_file, str):
         if is_valid(furl_or_file):
             return furl_or_file
     if os.path.isfile(furl_or_file):
-        furl = open(furl_or_file, 'r').read().strip()
+        with open(furl_or_file, 'r') as f:
+            furl = f.read().strip()
         if is_valid(furl):
             return furl
-    raise ValueError("not a furl or a file containing a furl: %s" % furl_or_file)
+    raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
 
-# We do this so if a user doesn't have OpenSSL installed, it will try to use
-# an UnauthenticatedTub.  But, they will still run into problems if they
-# try to use encrypted furls.
-try:
-    import OpenSSL
-except:
-    Tub = UnauthenticatedTub
-    have_crypto = False
+
+def is_valid_furl_or_file(furl_or_file):
+    """Validate a FURL or a FURL file.
+
+    If ``furl_or_file`` looks like a file, we simply make sure its directory
+    exists and that it has a ``.furl`` file extension.  We don't try to see
+    if the FURL file exists or to read its contents. This is useful for
+    cases where auto re-connection is being used.
+    """
+    if isinstance(furl_or_file, str):
+        if is_valid(furl_or_file):
+            return True
+    if isinstance(furl_or_file, (str, unicode)):
+        path, furl_filename = os.path.split(furl_or_file)
+        if os.path.isdir(path) and furl_filename.endswith('.furl'):
+            return True
+    return False
+
+
+def validate_furl_or_file(furl_or_file):
+    if not is_valid_furl_or_file(furl_or_file):
+        raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
+
+
+def get_temp_furlfile(filename):
+    """Return a temporary FURL file."""
+    return tempfile.mktemp(dir=os.path.dirname(filename),
+                           prefix=os.path.basename(filename))
+
+
+def make_tub(ip, port, secure, cert_file):
+    """Create a listening tub given an ip, port, and cert_file location.
+    
+    Parameters
+    ----------
+    ip : str
+        The ip address or hostname that the tub should listen on.  
+        Empty means all interfaces.
+    port : int
+        The port that the tub should listen on.  A value of 0 means
+        pick a random port
+    secure: bool
+        Will the connection be secure (in the Foolscap sense).
+    cert_file: str
+        A filename of a file to be used for theSSL certificate.
+
+    Returns
+    -------
+    A tub, listener tuple.
+    """
+    if secure:
+        if have_crypto:
+            tub = Tub(certFile=cert_file)
 else:
-    have_crypto = True
+            raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
+                                "can't run in secure mode.  Try running without "
+                                "security using 'ipcontroller -xy'.")
+    else:
+        tub = UnauthenticatedTub()
+    
+    # Set the strport based on the ip and port and start listening
+    if ip == '':
+        strport = "tcp:%i" % port
+    else:
+        strport = "tcp:%i:interface=%s" % (port, ip)
+    log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
+    listener = tub.listenOn(strport)
+    
+    return tub, listener
+
+
+class FCServiceFactory(AdaptedConfiguredObjectFactory):
+    """This class creates a tub with various services running in it.
+
+    The basic idea is that :meth:`create` returns a running :class:`Tub`
+    instance that has a number of Foolscap references registered in it.
+    This class is a subclass of :class:`IPython.core.component.Component`
+    so the IPython configuration and component system are used.
+
+    Attributes
+    ----------
+    interfaces : Config
+        A Config instance whose values are sub-Config objects having two
+        keys: furl_file and interface_chain.
+
+    The other attributes are the standard ones for Foolscap.
+    """
+
+    ip = Str('', config=True)
+    port = Int(0, config=True)
+    secure = Bool(True, config=True)
+    cert_file = Str('', config=True)
+    location = Str('', config=True)
+    reuse_furls = Bool(False, config=True)
+    interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
+
+    def __init__(self, config, adaptee):
+        super(FCServiceFactory, self).__init__(config, adaptee)
+        self._check_reuse_furls()
+
+    def _ip_changed(self, name, old, new):
+        if new == 'localhost' or new == '127.0.0.1':
+            self.location = '127.0.0.1'
+
+    def _check_reuse_furls(self):
+        furl_files = [i.furl_file for i in self.interfaces.values()]
+        for ff in furl_files:
+            fullfile = self._get_security_file(ff)
+            if self.reuse_furls:
+                if self.port==0:
+                    raise FURLError("You are trying to reuse the FURL file "
+                        "for this connection, but the port for this connection "
+                        "is set to 0 (autoselect). To reuse the FURL file "
+                        "you need to specify specific port to listen on."
+                    )
+                else:
+                    log.msg("Reusing FURL file: %s" % fullfile)
+            else:
+                if os.path.isfile(fullfile):
+                    log.msg("Removing old FURL file: %s" % fullfile)
+                    os.remove(fullfile)
+
+    def _get_security_file(self, filename):
+        return os.path.join(self.config.Global.security_dir, filename)
+
+    def create(self):
+        """Create and return the Foolscap tub with everything running."""
+
+        self.tub, self.listener = make_tub(
+            self.ip, self.port, self.secure, 
+            self._get_security_file(self.cert_file)
+        )
+        # log.msg("Interfaces to register [%r]: %r" % \
+        #     (self.__class__, self.interfaces))
+        if not self.secure:
+            log.msg("WARNING: running with no security: %s" % \
+                self.__class__.__name__)
+        reactor.callWhenRunning(self.set_location_and_register)
+        return self.tub
+
+    def set_location_and_register(self):
+        """Set the location for the tub and return a deferred."""
+
+        if self.location == '':
+            d = self.tub.setLocationAutomatically()
+        else:
+            d = defer.maybeDeferred(self.tub.setLocation,
+                "%s:%i" % (self.location, self.listener.getPortnum()))
+        self.adapt_to_interfaces(d)
+
+    def adapt_to_interfaces(self, d):
+        """Run through the interfaces, adapt and register."""
+
+        for ifname, ifconfig in self.interfaces.iteritems():
+            ff = self._get_security_file(ifconfig.furl_file)
+            log.msg("Adapting [%s] to interface: %s" % \
+                (self.adaptee.__class__.__name__, ifname))
+            log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
+            check_furl_file_security(ff, self.secure)
+            adaptee = self.adaptee
+            for i in ifconfig.interface_chain:
+                adaptee = import_item(i)(adaptee)
+            d.addCallback(self.register, adaptee, furl_file=ff)
+
+    def register(self, empty, ref, furl_file):
+        """Register the reference with the FURL file.
 
+        The FURL file is created and then moved to make sure that when the
+        file appears, the buffer has been flushed and the file closed.
+        """
+        temp_furl_file = get_temp_furlfile(furl_file)
+        self.tub.registerReference(ref, furlFile=temp_furl_file)
+        os.rename(temp_furl_file, furl_file)
 

IPython/kernel/multiengine.py

@@ -262,9 +262,8 b' class MultiEngine(ControllerAdapterBase):'
         elif targets == 'all':
             eList = self.engines.values()
             if len(eList) == 0:
-                msg = """There are no engines registered.
-                     Check the logs in ~/.ipython/log if you think there should have been."""
-                raise error.NoEnginesRegistered(msg)
+                raise error.NoEnginesRegistered("There are no engines registered. "
+                    "Check the logs if you think there should have been.")
             else:
                 return eList
         else:

IPython/kernel/multiengineclient.py

@@ -263,9 +263,17 b' class InteractiveMultiEngineClient(object):'
         """
         
         try:
-            __IPYTHON__.activeController = self
+            # This is injected into __builtins__.
+            ip = get_ipython()
         except NameError:
-            print "The IPython Controller magics only work within IPython."
+            print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
+        else:
+            pmagic = ip.get_component('parallel_magic')
+            if pmagic is not None:
+                pmagic.active_multiengine_client = self
+            else:
+                print "You must first load the parallelmagic extension " \
+                      "by doing '%load_ext parallelmagic'"
                     
     def __setitem__(self, key, value):
         """Add a dictionary interface for pushing/pulling.

IPython/kernel/scripts/ipcluster

@@ -1,22 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-"""ipcluster script"""
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipcluster
-	ipcluster.main()
+from IPython.kernel.ipclusterapp import launch_new_instance
 
+launch_new_instance()

IPython/kernel/scripts/ipcontroller

@@ -1,20 +1,18 b''
 #!/usr/bin/env python
 # encoding: utf-8
 
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
+
 
-if __name__ == '__main__':
-	from IPython.kernel.scripts import ipcontroller
-	ipcontroller.main()
+from IPython.kernel.ipcontrollerapp import launch_new_instance
 
+launch_new_instance()