Merging upstream

IPython/frontend/tests/test_linefrontend.py

@@ -0,0 +1,37 b''
+# encoding: utf-8
+"""
+Test the LineFrontEnd 
+"""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+from IPython.frontend.linefrontendbase import LineFrontEndBase
+from copy import deepcopy
+import nose.tools as nt
+
+class ConcreteLineFrontEnd(LineFrontEndBase):
+    """ A concrete class to test the LineFrontEndBase.
+    """
+    def capture_output(self):
+        pass
+
+    def release_output(self):
+        pass
+
+
+def test_is_complete():
+    """ Tests line completion heuristic.
+    """
+    frontend = ConcreteLineFrontEnd()
+    yield nt.assert_true, not frontend.is_complete('for x in \\')
+    yield nt.assert_true, not frontend.is_complete('for x in (1, ):')
+    yield nt.assert_true, frontend.is_complete('for x in (1, ):\n  pass')
+
+

IPython/kernel/tests/test_twistedutil.py

@@ -0,0 +1,51 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Tell nose to skip this module
+__test__ = {}
+
+import tempfile
+import os, sys
+
+from twisted.internet import reactor
+from twisted.trial import unittest
+
+from IPython.kernel.error import FileTimeoutError
+from IPython.kernel.twistedutil import wait_for_file
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+class TestWaitForFile(unittest.TestCase):
+
+    def test_delay(self):
+        filename = tempfile.mktemp()
+        def _create_file():
+            open(filename,'w').write('####')
+        dcall = reactor.callLater(0.5, _create_file)
+        d = wait_for_file(filename,delay=0.1)
+        d.addCallback(lambda r: self.assert_(r))
+        def _cancel_dcall(r):
+            if dcall.active():
+                dcall.cancel()
+        d.addCallback(_cancel_dcall)
+        return d
+    
+    def test_timeout(self):
+        filename = tempfile.mktemp()
+        d = wait_for_file(filename,delay=0.1,max_tries=1)
+        d.addErrback(lambda f: self.assertRaises(FileTimeoutError,f.raiseException))
+        return d
+        
  No newline at end of file

IPython/testing/decorators_trial.py

@@ -0,0 +1,132 b''
+# encoding: utf-8
+"""
+Testing related decorators for use with twisted.trial.
+
+The decorators in this files are designed to follow the same API as those
+in the decorators module (in this same directory).  But they can be used
+with twisted.trial
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+from IPython.testing.decorators import make_label_dec
+
+#-----------------------------------------------------------------------------
+# Testing decorators
+#-----------------------------------------------------------------------------
+
+
+def skipif(skip_condition, msg=None):
+    """Create a decorator that marks a test function for skipping.
+
+    The is a decorator factory that returns a decorator that will 
+    conditionally skip a test based on the value of skip_condition.  The
+    skip_condition argument can either be a boolean or a callable that returns
+    a boolean.
+
+    Parameters
+    ----------
+    skip_condition : boolean or callable
+        If this evaluates to True, the test is skipped.
+    msg : str
+        The message to print if the test is skipped.
+
+    Returns
+    -------
+    decorator : function
+        The decorator function that can be applied to the test function.
+    """
+
+    def skip_decorator(f):
+
+        # Allow for both boolean or callable skip conditions.
+        if callable(skip_condition):
+            skip_val = lambda : skip_condition()
+        else:
+            skip_val = lambda : skip_condition
+
+        if msg is None:
+            out = 'Test skipped due to test condition.'
+        else: 
+            out = msg
+        final_msg = "Skipping test: %s. %s" % (f.__name__,out)
+
+        if skip_val():
+            f.skip = final_msg
+
+        return f
+    return skip_decorator
+
+
+def skip(msg=None):
+    """Create a decorator that marks a test function for skipping.
+
+    This is a decorator factory that returns a decorator that will cause
+    tests to be skipped.
+
+    Parameters
+    ----------
+    msg : str
+        Optional message to be added.
+
+    Returns
+    -------
+    decorator : function
+        Decorator, which, when applied to a function, sets the skip
+        attribute of the function causing `twisted.trial` to skip it.
+    """
+
+    return skipif(True,msg)
+
+
+def numpy_not_available():
+    """Can numpy be imported?  Returns true if numpy does NOT import.
+
+    This is used to make a decorator to skip tests that require numpy to be
+    available, but delay the 'import numpy' to test execution time.
+    """
+    try:
+        import numpy
+        np_not_avail = False
+    except ImportError:
+        np_not_avail = True
+
+    return np_not_avail
+
+#-----------------------------------------------------------------------------
+# Decorators for public use
+#-----------------------------------------------------------------------------
+
+# Decorators to skip certain tests on specific platforms.
+skip_win32 = skipif(sys.platform == 'win32',
+                    "This test does not run under Windows")
+skip_linux = skipif(sys.platform == 'linux2',
+                    "This test does not run under Linux")
+skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
+
+# Decorators to skip tests if not on specific platforms.
+skip_if_not_win32 = skipif(sys.platform != 'win32',
+                           "This test only runs under Windows")
+skip_if_not_linux = skipif(sys.platform != 'linux2',
+                           "This test only runs under Linux")
+skip_if_not_osx = skipif(sys.platform != 'darwin',
+                         "This test only runs under OSX")
+
+# Other skip decorators
+skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
+
+skipknownfailure = skip('This test is known to fail')
+
+

IPython/testing/plugin/test_ipdoctest.py

@@ -0,0 +1,94 b''
+"""Tests for the ipdoctest machinery itself.
+
+Note: in a file named test_X, functions whose only test is their docstring (as
+a doctest) and which have no test functionality of their own, should be called
+'doctest_foo' instead of 'test_foo', otherwise they get double-counted (the
+empty function call is counted as a test, which just inflates tests numbers
+artificially).
+"""
+
+def doctest_simple():
+    """ipdoctest must handle simple inputs
+    
+    In [1]: 1
+    Out[1]: 1
+
+    In [2]: print 1
+    1
+    """
+
+
+def doctest_run_builtins():
+    """Check that %run doesn't damage __builtins__ via a doctest.
+
+    This is similar to the test_run_builtins, but I want *both* forms of the
+    test to catch any possible glitches in our testing machinery, since that
+    modifies %run somewhat.  So for this, we have both a normal test (below)
+    and a doctest (this one).
+
+    In [1]: import tempfile
+
+    In [3]: f = tempfile.NamedTemporaryFile()
+
+    In [4]: f.write('pass\\n')
+
+    In [5]: f.flush()
+
+    In [7]: %run $f.name
+    """
+
+def doctest_multiline1():
+    """The ipdoctest machinery must handle multiline examples gracefully.
+
+    In [2]: for i in range(10):
+       ...:     print i,
+       ...:      
+    0 1 2 3 4 5 6 7 8 9
+    """
+
+    
+def doctest_multiline2():
+    """Multiline examples that define functions and print output.
+
+    In [7]: def f(x):
+       ...:     return x+1
+       ...: 
+
+    In [8]: f(1)
+    Out[8]: 2
+
+    In [9]: def g(x):
+       ...:     print 'x is:',x
+       ...:      
+
+    In [10]: g(1)
+    x is: 1
+
+    In [11]: g('hello')
+    x is: hello
+    """
+
+
+def doctest_multiline3():
+    """Multiline examples with blank lines.
+
+    In [12]: def h(x):
+       ....:     if x>1:
+       ....:         return x**2
+       ....:     # To leave a blank line in the input, you must mark it
+       ....:     # with a comment character:
+       ....:     #
+       ....:     # otherwise the doctest parser gets confused.
+       ....:     else:
+       ....:         return -1
+       ....:      
+
+    In [13]: h(5)
+    Out[13]: 25
+
+    In [14]: h(1)
+    Out[14]: -1
+
+    In [15]: h(0)
+    Out[15]: -1
+   """

IPython/testing/tests/test_decorators_trial.py

@@ -0,0 +1,52 b''
+# encoding: utf-8
+"""
+Tests for decorators_trial.py
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Tell nose to skip this module
+__test__ = {}
+
+import os
+import sys
+
+from twisted.trial import unittest
+import IPython.testing.decorators_trial as dec
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+class TestDecoratorsTrial(unittest.TestCase):
+    
+    @dec.skip()
+    def test_deliberately_broken(self):
+        """A deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip('Testing the skip decorator')
+    def test_deliberately_broken2(self):
+        """Another deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip_linux
+    def test_linux(self):
+        self.assertNotEquals(sys.platform,'linux2',"This test can't run under linux")
+
+    @dec.skip_win32
+    def test_win32(self):
+        self.assertNotEquals(sys.platform,'win32',"This test can't run under windows")
+
+    @dec.skip_osx
+    def test_osx(self):
+        self.assertNotEquals(sys.platform,'darwin',"This test can't run under osx")
  No newline at end of file

IPython/testing/tests/test_tools.py

@@ -0,0 +1,52 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Tests for testing.tools
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import nose.tools as nt
+
+from IPython.testing import decorators as dec
+from IPython.testing.tools import full_path
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+
[e-mail]_win32
+def test_full_path_posix():
+    spath = '/foo/bar.py'
+    result = full_path(spath,['a.txt','b.txt'])
+    nt.assert_equal(result, ['/foo/a.txt', '/foo/b.txt'])
+    spath = '/foo'
+    result = full_path(spath,['a.txt','b.txt'])
+    nt.assert_equal(result, ['/a.txt', '/b.txt'])
+    result = full_path(spath,'a.txt')
+    nt.assert_equal(result, ['/a.txt'])
+
+
[e-mail]_if_not_win32
+def test_full_path_win32():
+    spath = 'c:\\foo\\bar.py'
+    result = full_path(spath,['a.txt','b.txt'])
+    nt.assert_equal(result, ['c:\\foo\\a.txt', 'c:\\foo\\b.txt'])
+    spath = 'c:\\foo'
+    result = full_path(spath,['a.txt','b.txt'])
+    nt.assert_equal(result, ['c:\\a.txt', 'c:\\b.txt'])
+    result = full_path(spath,'a.txt')
+    nt.assert_equal(result, ['c:\\a.txt'])
  No newline at end of file

IPython/testing/tools.py

@@ -0,0 +1,89 b''
+"""Generic testing tools that do NOT depend on Twisted.
+
+In particular, this module exposes a set of top-level assert* functions that
+can be used in place of nose.tools.assert* in method generators (the ones in
+nose can not, at least as of nose 0.10.4).
+
+Note: our testing package contains testing.util, which does depend on Twisted
+and provides utilities for tests that manage Deferreds.  All testing support
+tools that only depend on nose, IPython or the standard library should go here
+instead.
+
+
+Authors
+-------
+- Fernando Perez <[e-mail]>
+"""
+
+#*****************************************************************************
+#       Copyright (C) 2009 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+#-----------------------------------------------------------------------------
+# Required modules and packages
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import nose.tools as nt
+
+from IPython.tools import utils
+from IPython.testing import decorators as dec
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Make a bunch of nose.tools assert wrappers that can be used in test
+# generators.  This will expose an assert* function for each one in nose.tools.
+
+_tpl = """
+def %(name)s(*a,**kw):
+    return nt.%(name)s(*a,**kw)
+"""
+
+for _x in [a for a in dir(nt) if a.startswith('assert')]:
+    exec _tpl % dict(name=_x)
+
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
+
+
+def full_path(startPath,files):
+    """Make full paths for all the listed files, based on startPath.
+
+    Only the base part of startPath is kept, since this routine is typically
+    used with a script's __file__ variable as startPath.  The base of startPath
+    is then prepended to all the listed files, forming the output list.
+
+    Parameters
+    ----------
+      startPath : string
+        Initial path to use as the base for the results.  This path is split
+      using os.path.split() and only its first component is kept.
+
+      files : string or list
+        One or more files.
+
+    Examples
+    --------
+
+    >>> full_path('/foo/bar.py',['a.txt','b.txt'])
+    ['/foo/a.txt', '/foo/b.txt']
+
+    >>> full_path('/foo',['a.txt','b.txt'])
+    ['/a.txt', '/b.txt']
+
+    If a single file is given, the output is still a list:
+    >>> full_path('/foo','a.txt')
+    ['/a.txt']
+    """
+
+    files = utils.list_strings(files)
+    base = os.path.split(startPath)[0]
+    return [ os.path.join(base,f) for f in files ]

IPython/tests/refbug.py

@@ -0,0 +1,41 b''
+"""Minimal script to reproduce our nasty reference counting bug.
+
+The problem is related to https://bugs.launchpad.net/ipython/+bug/269966
+
+The original fix for that appeared to work, but John D. Hunter found a
+matplotlib example which, when run twice in a row, would break.  The problem
+were references held by open figures to internals of Tkinter.
+
+This code reproduces the problem that John saw, without matplotlib.
+
+This script is meant to be called by other parts of the test suite that call it
+via %run as if it were executed interactively by the user.  As of 2009-04-13,
+test_magic.py calls it.
+"""
+
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
+import sys
+
+from IPython import ipapi
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+ip = ipapi.get()
+
+if not '_refbug_cache' in ip.user_ns:
+    ip.user_ns['_refbug_cache'] = []
+
+
+aglobal = 'Hello'
+def f():
+    return aglobal
+
+cache = ip.user_ns['_refbug_cache']
+cache.append(f)
+
+def call_f():
+    for func in cache:
+        print 'lowercased:',func().lower()

IPython/tests/test_fakemodule.py

@@ -0,0 +1,17 b''
+"""Tests for the FakeModule objects.
+"""
+
+import nose.tools as nt
+
+from IPython.FakeModule import FakeModule, init_fakemod_dict
+
+# Make a fakemod and check a few properties
+def test_mk_fakemod():
+    fm = FakeModule()
+    yield nt.assert_true,fm
+    yield nt.assert_true,lambda : hasattr(fm,'__file__')
+
+def test_mk_fakemod_fromdict():
+    """Test making a FakeModule object with initial data"""
+    fm = FakeModule(dict(hello=True))
+    nt.assert_true(fm.hello)

IPython/tests/test_platutils.py

@@ -0,0 +1,59 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Tests for platutils.py
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+
+import nose.tools as nt
+
+from IPython.platutils import find_cmd, FindCmdError, get_long_path_name
+from IPython.testing import decorators as dec
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+def test_find_cmd_python():
+    """Make sure we find sys.exectable for python."""
+    nt.assert_equals(find_cmd('python'), sys.executable)
+
[e-mail]_win32
+def test_find_cmd():
+    """Make sure we can find the full path to ls."""
+    path = find_cmd('ls')
+    nt.assert_true(path.endswith('ls'))
+
[e-mail]_if_not_win32
+def test_find_cmd():
+    """Try to find pythonw on Windows."""
+    path = find_cmd('pythonw')
+    nt.assert_true(path.endswith('pythonw.exe'))
+
+def test_find_cmd_fail():
+    """Make sure that FindCmdError is raised if we can't find the cmd."""
+    nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
+
[e-mail]_if_not_win32
+def test_get_long_path_name_win32():
+    p = get_long_path_name('c:\\docume~1')
+    nt.assert_equals(p,u'c:\\Documents and Settings') 
+   
[e-mail]_win32
+def test_get_long_path_name():
+    p = get_long_path_name('/usr/local')
+    nt.assert_equals(p,'/usr/local')
+

docs/examples/kernel/taskreject.py

@@ -0,0 +1,53 b''
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""
+A new example showing how to use `TaskRejectError` to handle dependencies
+in the IPython task system.
+
+To run this example, do::
+
+    $ ipcluster local -n 4
+
+Then, in another terminal start up IPython and do::
+
+    In [0]: %run taskreject.py
+
+    In [1]: mec.execute('run=True', targets=[0,1])
+
+After the first command, the scheduler will keep rescheduling the tasks, as
+they will fail with `TaskRejectError`.  But after the second command, there
+are two engines that the tasks can run on.  The tasks are quickly funneled
+to these engines.
+
+If you want to see how the controller is scheduling and retrying the tasks
+do a `tail -f` on the controller's log file in ~/.ipython/log.
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from IPython.kernel import client
+from IPython.kernel import TaskRejectError
+
+mec = client.MultiEngineClient()
+tc = client.TaskClient()
+
+mec.execute('from IPython.kernel import TaskRejectError')
+mec.execute('run = False')
+
+def map_task():
+    if not run:
+        raise TaskRejectError('task dependency not met')
+    return 3.0e8
+
+task_ids = []
+
+for i in range(10):
+    task = client.MapTask(map_task, retries=20)
+    task_ids.append(tc.run(task, block=False))
+

IPython/Extensions/ipipe.py

@@ -114,6 +114,11 b' except NameError:'
             items.reverse()
         return items
 
+try: # Python 2.4 compatibility
+    GeneratorExit
+except NameError:
+    GeneratorExit = SystemExit
+
 try:
     import pwd
 except ImportError:
@@ -666,7 +671,7 b' def xrepr(item, mode="default"):'
         try:
             for x in func(mode):
                 yield x
-        except (KeyboardInterrupt, SystemExit):
+        except (KeyboardInterrupt, SystemExit, GeneratorExit):
             raise
         except Exception:
             yield (astyle.style_default, repr(item))
@@ -840,20 +845,20 b' def upgradexattr(attr):'
     """
     Convert an attribute descriptor string to a real descriptor object.
 
-    If attr already is a descriptor object return if unmodified. A
+    If attr already is a descriptor object return it unmodified. A
     ``SelfDescriptor`` will be returned if ``attr`` is ``None``. ``"foo"``
     returns an ``AttributeDescriptor`` for the attribute named ``"foo"``.
     ``"foo()"`` returns a ``MethodDescriptor`` for the method named ``"foo"``.
     ``"-foo"`` will return an ``IterAttributeDescriptor`` for the attribute
     named ``"foo"`` and ``"-foo()"`` will return an ``IterMethodDescriptor``
-    for the method named ``"foo"``. Furthermore integer will return the appropriate
+    for the method named ``"foo"``. Furthermore integers will return the appropriate
     ``IndexDescriptor`` and callables will return a ``FunctionDescriptor``.
     """
     if attr is None:
         return selfdescriptor
     elif isinstance(attr, Descriptor):
         return attr
-    elif isinstance(attr, str):
+    elif isinstance(attr, basestring):
         if attr.endswith("()"):
             if attr.startswith("-"):
                 return IterMethodDescriptor(attr[1:-2])

IPython/Extensions/ipy_greedycompleter.py

@@ -66,7 +66,7 b' def attr_matches(self, text):'
     return res
 
 def main():
-    import readline
+    import IPython.rlineimpl as readline
     readline.set_completer_delims(" \n\t")
     # monkeypatch - the code will be folded to normal completer later on
     import IPython.completer

IPython/FakeModule.py

@@ -15,6 +15,37 b' sessions.'
 
 import types
 
+def init_fakemod_dict(fm,adict=None):
+    """Initialize a FakeModule instance __dict__.
+
+    Kept as a standalone function and not a method so the FakeModule API can
+    remain basically empty.
+
+    This should be considered for private IPython use, used in managing
+    namespaces for %run.
+
+    Parameters
+    ----------
+
+    fm : FakeModule instance
+
+    adict : dict, optional
+    """
+
+    dct = {}
+    # It seems pydoc (and perhaps others) needs any module instance to
+    # implement a __nonzero__ method, so we add it if missing:
+    dct.setdefault('__nonzero__',lambda : True)
+    dct.setdefault('__file__',__file__)
+
+    if adict is not None:
+        dct.update(adict)
+
+    # Hard assignment of the object's __dict__.  This is nasty but deliberate.
+    fm.__dict__.clear()
+    fm.__dict__.update(dct)
+
+
 class FakeModule(types.ModuleType):
     """Simple class with attribute access to fake a module.
 
@@ -29,14 +60,7 b' class FakeModule(types.ModuleType):'
         
         # tmp to force __dict__ instance creation, else self.__dict__ fails
         self.__iptmp = None
-        
-        # It seems pydoc (and perhaps others) needs any module instance to
-        # implement a __nonzero__ method, so we add it if missing:
-        self.__dict__.setdefault('__nonzero__',lambda : True)
-        self.__dict__.setdefault('__file__',__file__)
-
         # cleanup our temp trick
         del self.__iptmp
-
-        if adict is not None:
-            self.__dict__.update(adict)
+        # Now, initialize the actual data in the instance dict.
+        init_fakemod_dict(self,adict)

IPython/Magic.py

@@ -1584,23 +1584,17 b' Currently the magic system has the following functions:\\n"""'
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
-            main_mod = FakeModule(prog_ns)
+            main_mod = self.shell.new_main_mod(prog_ns)
         else:
             # Run in a fresh, empty namespace
             if opts.has_key('n'):
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
-            main_mod = FakeModule()
+
+            main_mod = self.shell.new_main_mod()
             prog_ns = main_mod.__dict__
             prog_ns['__name__'] = name
-            
-            # The shell MUST hold a reference to main_mod so after %run exits,
-            # the python deletion mechanism doesn't zero it out (leaving
-            # dangling references).  However, we should drop old versions of
-            # main_mod.  There is now a proper API to manage this caching in
-            # the main shell object, we use that.            
-            self.shell.cache_main_mod(main_mod)
 
         # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
         # set the __file__ global in the script's namespace
@@ -1681,7 +1675,7 b' Currently the magic system has the following functions:\\n"""'
                                    exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
-                            t_sys = t1[1]-t1[1]
+                            t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "  User  : %10s s." % t_usr
                             print "  System: %10s s." % t_sys
@@ -1693,7 +1687,7 b' Currently the magic system has the following functions:\\n"""'
                                        exit_ignore=exit_ignore)
                             t1 = clock2()
                             t_usr = t1[0]-t0[0]
-                            t_sys = t1[1]-t1[1]
+                            t_sys = t1[1]-t0[1]
                             print "\nIPython CPU timings (estimated):"
                             print "Total runs performed:",nruns
                             print "  Times : %10s    %10s" % ('Total','Per run')
@@ -1703,13 +1697,28 b' Currently the magic system has the following functions:\\n"""'
                     else:
                         # regular execution
                         runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
+
                 if opts.has_key('i'):
                     self.shell.user_ns['__name__'] = __name__save
                 else:
+                    # The shell MUST hold a reference to prog_ns so after %run
+                    # exits, the python deletion mechanism doesn't zero it out
+                    # (leaving dangling references).
+                    self.shell.cache_main_mod(prog_ns,filename)
                     # update IPython interactive namespace
                     del prog_ns['__name__']
                     self.shell.user_ns.update(prog_ns)
         finally:
+            # It's a bit of a mystery why, but __builtins__ can change from
+            # being a module to becoming a dict missing some key data after
+            # %run.  As best I can see, this is NOT something IPython is doing
+            # at all, and similar problems have been reported before:
+            # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
+            # Since this seems to be done by the interpreter itself, the best
+            # we can do is to at least restore __builtins__ for the user on
+            # exit.
+            self.shell.user_ns['__builtins__'] = __builtin__
+            
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
@@ -1719,6 +1728,7 b' Currently the magic system has the following functions:\\n"""'
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
+
             self.shell.reloadhist()
                 
         return stats
@@ -1800,7 +1810,28 b' Currently the magic system has the following functions:\\n"""'
         import timeit
         import math
 
-        units = [u"s", u"ms", u"\xb5s", u"ns"]
+        # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
+        # certain terminals.  Until we figure out a robust way of
+        # auto-detecting if the terminal can deal with it, use plain 'us' for
+        # microseconds.  I am really NOT happy about disabling the proper
+        # 'micro' prefix, but crashing is worse... If anyone knows what the
+        # right solution for this is, I'm all ears...
+        #
+        # Note: using
+        #
+        # s = u'\xb5'
+        # s.encode(sys.getdefaultencoding())
+        #
+        # is not sufficient, as I've seen terminals where that fails but
+        # print s
+        #
+        # succeeds
+        #
+        # See bug: https://bugs.launchpad.net/ipython/+bug/348466
+        
+        #units = [u"s", u"ms",u'\xb5',"ns"]
+        units = [u"s", u"ms",u'us',"ns"]
+        
         scaling = [1, 1e3, 1e6, 1e9]
 
         opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
@@ -1839,9 +1870,9 b' Currently the magic system has the following functions:\\n"""'
             # determine number so that 0.2 <= total time < 2.0
             number = 1
             for i in range(1, 10):
-                number *= 10
                 if timer.timeit(number) >= 0.2:
                     break
+                number *= 10
         
         best = min(timer.repeat(repeat, number)) / number
 

IPython/Prompts.py

@@ -126,9 +126,13 b' prompt_specials_color = {'
     # Just the prompt counter number, WITHOUT any coloring wrappers, so users
     # can get numbers displayed in whatever color they want.
     r'\N': '${self.cache.prompt_count}',
+
     # Prompt/history count, with the actual digits replaced by dots.  Used
     # mainly in continuation prompts (prompt_in2)
-    r'\D': '${"."*len(str(self.cache.prompt_count))}',
+    #r'\D': '${"."*len(str(self.cache.prompt_count))}',
+    # More robust form of the above expression, that uses __builtins__
+    r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}',
+
     # Current working directory
     r'\w': '${os.getcwd()}',
     # Current time

IPython/Release.py

@@ -20,10 +20,10 b" name = 'ipython'"
 # because bdist_rpm does not accept dashes (an RPM) convention, and
 # bdist_deb does not accept underscores (a Debian convention).
 
-development = False    # change this to False to do a release
-version_base = '0.9.1'
+development = True    # change this to False to do a release
+version_base = '0.10'
 branch = 'ipython'
-revision = '1143'
+revision = '1163'
 
 if development:
     if branch == 'ipython':

IPython/Shell.py

@@ -292,17 +292,19 b' if HAS_CTYPES:'
         """raises the exception, performs cleanup if needed"""
         if not inspect.isclass(exctype):
             raise TypeError("Only types can be raised (not instances)")
-        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid,
+        # Explicit cast to c_long is necessary for 64-bit support:
+        # See https://bugs.launchpad.net/ipython/+bug/237073
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid),
                                                          ctypes.py_object(exctype))
         if res == 0:
             raise ValueError("invalid thread id")
         elif res != 1:
-            # """if it returns a number greater than one, you're in trouble, 
-            # and you should call it again with exc=NULL to revert the effect"""
+            # If it returns a number greater than one, you're in trouble, 
+            # and you should call it again with exc=NULL to revert the effect
             ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0)
             raise SystemError("PyThreadState_SetAsyncExc failed")
 
-    def sigint_handler (signum,stack_frame):
+    def sigint_handler(signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_
@@ -321,7 +323,7 b' if HAS_CTYPES:'
             Term.cout.flush()
 
 else:
-    def sigint_handler (signum,stack_frame):
+    def sigint_handler(signum,stack_frame):
         """Sigint handler for threaded apps.
 
         This is a horrible hack to pass information about SIGINT _without_

IPython/UserConfig/ipy_user_conf.py

@@ -99,16 +99,14 b' def main():'
     #import readline
     #readline.parse_and_bind('set completion-query-items 1000')
     #readline.parse_and_bind('set page-completions no')
-    
-    
-    
-    
+
+
 # some config helper functions you can use 
 def import_all(modules):
     """ Usage: import_all("os sys") """ 
     for m in modules.split():
         ip.ex("from %s import *" % m)
-        
+
 def execf(fname):
     """ Execute a file in user namespace """
     ip.ex('execfile("%s")' % os.path.expanduser(fname))

IPython/background_jobs.py

@@ -248,7 +248,7 b' class BackgroundJobManager:'
         self._update_status()
         new_comp = self._group_report(self._comp_report,'Completed')
         new_dead = self._group_report(self._dead_report,
-                                      'Dead, call job.traceback() for details')
+                                      'Dead, call jobs.traceback() for details')
         self._comp_report[:] = []
         self._dead_report[:] = []
         return new_comp or new_dead
@@ -340,7 +340,7 b' class BackgroundJobBase(threading.Thread):'
     stat_created   = 'Created'; stat_created_c = 0
     stat_running   = 'Running'; stat_running_c = 1
     stat_completed = 'Completed'; stat_completed_c = 2
-    stat_dead      = 'Dead (Exception), call job.traceback() for details'
+    stat_dead      = 'Dead (Exception), call jobs.traceback() for details'
     stat_dead_c = -1
 
     def __init__(self):
@@ -391,7 +391,7 b' class BackgroundJobBase(threading.Thread):'
             self.status    = BackgroundJobBase.stat_dead
             self.stat_code = BackgroundJobBase.stat_dead_c
             self.finished  = None
-            self.result    = ('<BackgroundJob died, call job.traceback() for details>')
+            self.result    = ('<BackgroundJob died, call jobs.traceback() for details>')
             self._tb       = self._make_tb()
         else:
             self.status    = BackgroundJobBase.stat_completed

IPython/frontend/asyncfrontendbase.py

@@ -14,12 +14,13 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-from IPython.external import guid
 
+from IPython.external import guid
 
 from zope.interface import Interface, Attribute, implements, classProvides
 from twisted.python.failure import Failure
-from IPython.frontend.frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
+from IPython.frontend.frontendbase import (
+    FrontEndBase, IFrontEnd, IFrontEndFactory)
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.engineservice import IEngineCore
 

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -15,30 +15,38 b' __docformat__ = "restructuredtext en"'
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
+from twisted.internet.defer import succeed
+
+from IPython.kernel.core.interpreter import Interpreter
+import IPython.kernel.engineservice as es
+
 try:
-    from IPython.kernel.core.interpreter import Interpreter
-    import IPython.kernel.engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from twisted.internet.defer import succeed
-    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController                                                    
+    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
     from Foundation import NSMakeRect
-    from AppKit import NSTextView, NSScrollView                                                
+    from AppKit import NSTextView, NSScrollView
 except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted, Foolscap and PyObjC")
+    # This tells twisted.trial to skip this module if PyObjC is not found
+    skip = True
 
-class TestIPythonCocoaControler(DeferredTestCase):
+#---------------------------------------------------------------------------
+# Tests
+#---------------------------------------------------------------------------
+class TestIPythonCocoaControler(unittest.TestCase):
     """Tests for IPythonCocoaController"""
-   
+
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
-   
+
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
-   
+
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
@@ -47,48 +55,46 b' class TestIPythonCocoaControler(DeferredTestCase):'
             del result['number']
             del result['id']
             return result
-        self.assertDeferredEquals(
-            self.controller.execute(code).addCallback(removeNumberAndID), 
-            expected)
-   
+        d = self.controller.execute(code)
+        d.addCallback(removeNumberAndID)
+        d.addCallback(lambda r: self.assertEquals(r, expected))
+
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
-       
         self.controller.execute(code).addCallback(testControllerUserNS)
-   
-   
+
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
-   
+
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
-       
+
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
-       
+
         self.controller.execute(code).addCallback(testCompletes)
-   
-   
+
+
     def testCurrentIndent(self):
         """test that current_indent_string returns current indent or None.
         Uses _indent_for_block for direct unit testing.
         """
-       
+        
         self.controller.tabUsesSpaces = True
         self.assert_(self.controller._indent_for_block("""a=3""") == None)
         self.assert_(self.controller._indent_for_block("") == None)
         block = """def test():\n    a=3"""
         self.assert_(self.controller._indent_for_block(block) == \
                     ' ' * self.controller.tabSpaces)
-       
+        
         block = """if(True):\n%sif(False):\n%spass""" % \
                     (' '*self.controller.tabSpaces,
                      2*' '*self.controller.tabSpaces)
         self.assert_(self.controller._indent_for_block(block) == \
                     2*(' '*self.controller.tabSpaces))
-   
+

IPython/frontend/linefrontendbase.py

@@ -18,10 +18,8 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 import re
 
-import IPython
 import sys
 import codeop
-import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
@@ -58,6 +56,9 b' class LineFrontEndBase(FrontEndBase):'
     # programatic control of the frontend. 
     last_result = dict(number=0)
 
+    # The last prompt displayed. Useful for continuation prompts.
+    last_prompt = ''
+
     # The input buffer being edited
     input_buffer = ''
 
@@ -151,8 +152,12 b' class LineFrontEndBase(FrontEndBase):'
             self.capture_output()
             try:
                 # Add line returns here, to make sure that the statement is
-                # complete.
-                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                # complete (except if '\' was used).
+                # This should probably be done in a different place (like
+                # maybe 'prefilter_input' method? For now, this works.
+                clean_string = string.rstrip('\n')
+                if not clean_string.endswith('\\'): clean_string +='\n\n' 
+                is_complete = codeop.compile_command(clean_string,
                             "<string>", "exec")
                 self.release_output()
             except Exception, e:
@@ -183,16 +188,6 b' class LineFrontEndBase(FrontEndBase):'
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
 
-        ## try:
-        ##     self.history.input_cache[-1] = raw_string.rstrip()
-        ##     result = self.shell.execute(python_string)
-        ##     self.last_result = result
-        ##     self.render_result(result)
-        ## except:
-        ##     self.show_traceback()
-        ## finally:
-        ##     self.after_execute()
-
         try:
             try:
                 self.history.input_cache[-1] = raw_string.rstrip()
@@ -272,15 +267,15 b' class LineFrontEndBase(FrontEndBase):'
         symbols_per_line = max(1, chars_per_line/max_len)
 
         pos = 1
-        buf = []
+        completion_string = []
         for symbol in possibilities:
             if pos < symbols_per_line:
-                buf.append(symbol.ljust(max_len))
+                completion_string.append(symbol.ljust(max_len))
                 pos += 1
             else:
-                buf.append(symbol.rstrip() + '\n')
+                completion_string.append(symbol.rstrip() + '\n')
                 pos = 1
-        self.write(''.join(buf))
+        self.write(''.join(completion_string))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
         self.input_buffer = new_line
@@ -297,26 +292,70 b' class LineFrontEndBase(FrontEndBase):'
         self.write(prompt)
 
 
+    def continuation_prompt(self):
+        """Returns the current continuation prompt.
+        """
+        return ("."*(len(self.last_prompt)-2) + ': ')
+ 
+
+    def execute_command(self, command, hidden=False):
+        """ Execute a command, not only in the model, but also in the
+            view, if any.
+        """
+        return self.shell.execute(command)
+
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
  
-    def _on_enter(self):
+    def _on_enter(self, new_line_pos=0):
         """ Called when the return key is pressed in a line editing
             buffer.
+
+            Parameters
+            ----------
+            new_line_pos : integer, optional
+                Position of the new line to add, starting from the
+                end (0 adds a new line after the last line, -1 before
+                the last line...)
+
+            Returns
+            -------
+            True if execution is triggered
         """
         current_buffer = self.input_buffer
-        cleaned_buffer = self.prefilter_input(current_buffer)
+        # XXX: This string replace is ugly, but there should be no way it
+        # fails.
+        prompt_less_buffer = re.sub('^' + self.continuation_prompt(),
+                '', current_buffer).replace('\n' + self.continuation_prompt(),
+                                            '\n')
+        cleaned_buffer = self.prefilter_input(prompt_less_buffer)
         if self.is_complete(cleaned_buffer):
             self.execute(cleaned_buffer, raw_string=current_buffer)
+            return True
         else:
-            self.input_buffer += self._get_indent_string(
-                                                current_buffer[:-1])
-            if len(current_buffer.split('\n')) == 2:
-                self.input_buffer += '\t\t'
-            if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
-                self.input_buffer += '\t'
-
+            # Start a new line.
+            new_line_pos = -new_line_pos
+            lines = current_buffer.split('\n')[:-1]
+            prompt_less_lines = prompt_less_buffer.split('\n')
+            # Create the new line, with the continuation prompt, and the 
+            # same amount of indent than the line above it.
+            new_line = self.continuation_prompt() + \
+                  self._get_indent_string('\n'.join(
+                                    prompt_less_lines[:new_line_pos-1]))
+            if len(lines) == 1:
+                # We are starting a first continuation line. Indent it.
+                new_line += '\t'
+            elif current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
+                # The last line ends with ":", autoindent the new line.
+                new_line += '\t'
+
+            if new_line_pos == 0:
+                lines.append(new_line)
+            else:
+                lines.insert(new_line_pos, new_line)
+            self.input_buffer = '\n'.join(lines)
+            
 
     def _get_indent_string(self, string):
         """ Return the string of whitespace that prefixes a line. Used to

IPython/frontend/prefilterfrontend.py

@@ -22,9 +22,10 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 import sys
-
-from linefrontendbase import LineFrontEndBase, common_prefix
-from frontendbase import FrontEndBase
+import pydoc
+import os
+import re
+import __builtin__
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
@@ -33,9 +34,8 b' from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap'
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
 
 from IPython.genutils import Term
-import pydoc
-import os
-import sys
+
+from linefrontendbase import LineFrontEndBase, common_prefix
 
 
 def mk_system_call(system_call_function, command):
@@ -45,6 +45,8 b' def mk_system_call(system_call_function, command):'
     """
     def my_system_call(args):
         system_call_function("%s %s" % (command, args))
+
+    my_system_call.__doc__ = "Calls %s" % command
     return my_system_call
 
 #-------------------------------------------------------------------------------
@@ -62,13 +64,25 b' class PrefilterFrontEnd(LineFrontEndBase):'
 
     debug = False
     
-    def __init__(self, ipython0=None, *args, **kwargs):
+    def __init__(self, ipython0=None, argv=None, *args, **kwargs):
         """ Parameters:
             -----------
 
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
+
+            argv : list, optional
+              Used as the instance's argv value.  If not given, [] is used.
         """
+        if argv is None:
+            argv = []
+        # This is a hack to avoid the IPython exception hook to trigger
+        # on exceptions (https://bugs.launchpad.net/bugs/337105)
+        # XXX: This is horrible: module-leve monkey patching -> side
+        # effects.
+        from IPython import iplib
+        iplib.InteractiveShell.isthreaded = True
+
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
                             out_callback=self.write,
@@ -83,10 +97,16 b' class PrefilterFrontEnd(LineFrontEndBase):'
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
+            # Suppress all key input, to avoid waiting
+            def my_rawinput(x=None):
+                return '\n'
+            old_rawinput = __builtin__.raw_input
+            __builtin__.raw_input = my_rawinput
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[], 
+            ipython0 = make_IPython(argv=argv, 
                                     user_ns=self.shell.user_ns,
                                     user_global_ns=self.shell.user_global_ns)
+            __builtin__.raw_input = old_rawinput
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
@@ -98,16 +118,17 b' class PrefilterFrontEnd(LineFrontEndBase):'
         self._ip.system = self.system_call
         # XXX: Muck around with magics so that they work better
         # in our environment
-        self.ipython0.magic_ls = mk_system_call(self.system_call, 
-                                                            'ls -CF')
+        if not sys.platform.startswith('win'):
+            self.ipython0.magic_ls = mk_system_call(self.system_call, 
+                                                                'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
 
 
         if not 'banner' in kwargs and self.banner is None:
-            self.banner = self.ipython0.BANNER + """
-This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
+            self.banner = self.ipython0.BANNER
 
+        # FIXME: __init__ and start should be two different steps
         self.start()
 
     #--------------------------------------------------------------------------
@@ -117,7 +138,10 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
     def show_traceback(self):
         """ Use ipython0 to capture the last traceback and display it.
         """
-        self.capture_output()
+        # Don't do the capture; the except_hook has already done some
+        # modifications to the IO streams, if we store them, we'll be
+        # storing the wrong ones.
+        #self.capture_output()
         self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
@@ -171,7 +195,7 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
     def complete(self, line):
         # FIXME: This should be factored out in the linefrontendbase
         # method.
-        word = line.split('\n')[-1].split(' ')[-1]
+        word = self._get_completion_text(line)
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
         key = lambda x: x.replace('_', '')
@@ -244,3 +268,18 b' This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""'
         """
         self.ipython0.atexit_operations()
 
+
+    def _get_completion_text(self, line):
+        """ Returns the text to be completed by breaking the line at specified
+        delimiters.
+        """
+        # Break at: spaces, '=', all parentheses (except if balanced).
+        # FIXME2: In the future, we need to make the implementation similar to
+        # that in the 'pyreadline' module (modes/basemode.py) where we break at
+        # each delimiter and try to complete the residual line, until we get a
+        # successful list of completions.
+        expression = '\s|=|,|:|\((?!.*\))|\[(?!.*\])|\{(?!.*\})' 
+        complete_sep = re.compile(expression)
+        text = complete_sep.split(line)[-1]
+        return text
+

IPython/frontend/process/killableprocess.py

@@ -151,7 +151,12 b' else:'
             self._thread = ht
             self.pid = pid
 
-            winprocess.AssignProcessToJobObject(self._job, hp)
+            # XXX: A try/except to fix UAC-related problems under
+            # Windows Vista, when reparenting jobs.
+            try:
+                winprocess.AssignProcessToJobObject(self._job, hp)
+            except WindowsError:
+                pass
             winprocess.ResumeThread(ht)
 
             if p2cread is not None:

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -1,6 +1,6 b''
 # encoding: utf-8
 
-"""This file contains unittests for the frontendbase module."""
+"""This file contains unittests for the asyncfrontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
@@ -15,17 +15,15 b' __docformat__ = "restructuredtext en"'
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
-import unittest
+# Tell nose to skip this module
+__test__ = {}
 
-try:
-    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-    from IPython.frontend import frontendbase 
-    from IPython.kernel.engineservice import EngineService
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+from twisted.trial import unittest
+from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+from IPython.frontend import frontendbase 
+from IPython.kernel.engineservice import EngineService
+from IPython.testing.parametric import Parametric, parametric
 
-from IPython.testing.decorators import skip
 
 class FrontEndCallbackChecker(AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
@@ -44,14 +42,11 b' class FrontEndCallbackChecker(AsyncFrontEndBase):'
         self.renderResultCalled = True
         return result
     
-    
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
-    
 
 
-    
 class TestAsyncFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
@@ -59,97 +54,56 b' class TestAsyncFrontendBase(unittest.TestCase):'
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
     def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(
+        self.assert_(frontendbase.IFrontEnd.implementedBy(
                                     AsyncFrontEndBase))
     
     def test_is_complete_returns_False_for_incomplete_block(self):
-        """"""
-        
         block = """def test(a):"""
-        
-        assert(self.fb.is_complete(block) == False)
+        self.assert_(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
-        """"""
-        
         block = """def test(a): pass"""
-            
-        assert(self.fb.is_complete(block))
-        
+        self.assert_(self.fb.is_complete(block))
         block = """a=3"""
-        
-        assert(self.fb.is_complete(block))
+        self.assert_(self.fb.is_complete(block))
     
     def test_blockID_added_to_result(self):
         block = """3+3"""
-        
         d = self.fb.execute(block, blockID='TEST_ID')
-        
-        d.addCallback(self.checkBlockID, expected='TEST_ID')
+        d.addCallback(lambda r: self.assert_(r['blockID']=='TEST_ID'))
+        return d
     
     def test_blockID_added_to_failure(self):
         block = "raise Exception()"
-        
         d = self.fb.execute(block,blockID='TEST_ID')
-        d.addErrback(self.checkFailureID, expected='TEST_ID')
-    
-    def checkBlockID(self, result, expected=""):
-        assert(result['blockID'] == expected)
-    
-    
-    def checkFailureID(self, failure, expected=""):
-        assert(failure.blockID == expected)
-    
+        d.addErrback(lambda f: self.assert_(f.blockID=='TEST_ID'))
+        return d
     
     def test_callbacks_added_to_execute(self):
-        """test that
-        update_cell_prompt
-        render_result
-        
-        are added to execute request
-        """
-        
         d = self.fb.execute("10+10")
-        d.addCallback(self.checkCallbacks)
+        d.addCallback(lambda r: self.assert_(self.fb.updateCalled and self.fb.renderResultCalled))
+        return d
     
-    def checkCallbacks(self, result):
-        assert(self.fb.updateCalled)
-        assert(self.fb.renderResultCalled)
-    
-    @skip("This test fails and lead to an unhandled error in a Deferred.")
     def test_error_callback_added_to_execute(self):
-        """test that render_error called on execution error"""
+        """Test that render_error called on execution error."""
         
         d = self.fb.execute("raise Exception()")
-        d.addCallback(self.checkRenderError)
-    
-    def checkRenderError(self, result):
-        assert(self.fb.renderErrorCalled)
+        d.addErrback(lambda f: self.assert_(self.fb.renderErrorCalled))
+        return d
     
     def test_history_returns_expected_block(self):
-        """Make sure history browsing doesn't fail"""
+        """Make sure history browsing doesn't fail."""
         
         blocks = ["a=1","a=2","a=3"]
-        for b in blocks:
-            d = self.fb.execute(b)
-        
-        # d is now the deferred for the last executed block
-        d.addCallback(self.historyTests, blocks)
-        
-    
-    def historyTests(self, result, blocks):
-        """historyTests"""
-        
-        assert(len(blocks) >= 3)
-        assert(self.fb.get_history_previous("") == blocks[-2])
-        assert(self.fb.get_history_previous("") == blocks[-3])
-        assert(self.fb.get_history_next() == blocks[-2])
-    
-    
-    def test_history_returns_none_at_startup(self):
-        """test_history_returns_none_at_startup"""
-        
-        assert(self.fb.get_history_previous("")==None)
-        assert(self.fb.get_history_next()==None)
-    
-    
+        d = self.fb.execute(blocks[0])
+        d.addCallback(lambda _: self.fb.execute(blocks[1]))
+        d.addCallback(lambda _: self.fb.execute(blocks[2]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-2]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-3]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_next()==blocks[-2]))
+        return d
+    
+    def test_history_returns_none_at_startup(self):        
+        self.assert_(self.fb.get_history_previous("")==None)
+        self.assert_(self.fb.get_history_next()==None)
+

IPython/frontend/tests/test_prefilterfrontend.py

@@ -12,12 +12,32 b' __docformat__ = "restructuredtext en"'
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
+from copy import copy, deepcopy
 from cStringIO import StringIO
 import string
 
-from IPython.ipapi import get as get_ipython0
+from nose.tools import assert_equal
+
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
-from copy import deepcopy
+from IPython.ipapi import get as get_ipython0
+from IPython.testing.plugin.ipdoctest import default_argv
+
+
+def safe_deepcopy(d):
+    """ Deep copy every key of the given dict, when possible. Elsewhere
+        do a copy.
+    """
+    copied_d = dict()
+    for key, value in d.iteritems():
+        try:
+            copied_d[key] = deepcopy(value)
+        except:
+            try:
+                copied_d[key] = copy(value)
+            except:
+                copied_d[key] = value
+    return copied_d
+
 
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
@@ -26,16 +46,8 b' class TestPrefilterFrontEnd(PrefilterFrontEnd):'
     banner = ''
 
     def __init__(self):
-        ipython0 = get_ipython0().IP
         self.out = StringIO()
-        PrefilterFrontEnd.__init__(self, ipython0=ipython0)
-        # Clean up the namespace for isolation between tests
-        user_ns = self.ipython0.user_ns
-        # We need to keep references to things so that they don't
-        # get garbage collected (this stinks).
-        self.shadow_ns = dict()
-        for i in self.ipython0.magic_who_ls():
-            self.shadow_ns[i] = user_ns.pop(i)
+        PrefilterFrontEnd.__init__(self,argv=default_argv())
         # Some more code for isolation (yeah, crazy)
         self._on_enter()
         self.out.flush()
@@ -52,17 +64,31 b' class TestPrefilterFrontEnd(PrefilterFrontEnd):'
 
 def isolate_ipython0(func):
     """ Decorator to isolate execution that involves an iptyhon0.
+
+        Notes
+        -----
+
+        Apply only to functions with no arguments. Nose skips functions
+        with arguments.
     """
-    def my_func(*args, **kwargs):
-        ipython0 = get_ipython0().IP
-        user_ns = deepcopy(ipython0.user_ns)
-        global_ns = deepcopy(ipython0.global_ns)
+    def my_func():
+        iplib = get_ipython0()
+        if iplib is None:
+            return func()
+        ipython0 = iplib.IP
+        global_ns = safe_deepcopy(ipython0.user_global_ns)
+        user_ns = safe_deepcopy(ipython0.user_ns)
         try:
-            func(*args, **kwargs)
+            out = func()
         finally:
             ipython0.user_ns = user_ns
-            ipython0.global_ns = global_ns
+            ipython0.user_global_ns = global_ns
+        # Undo the hack at creation of PrefilterFrontEnd
+        from IPython import iplib
+        iplib.InteractiveShell.isthreaded = False
+        return out
 
+    my_func.__name__ = func.__name__
     return my_func
 
 
@@ -74,7 +100,7 b' def test_execution():'
     f.input_buffer = 'print 1'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value  == '1\n'
+    assert_equal(out_value, '1\n')
 
 
 @isolate_ipython0
@@ -87,20 +113,20 b' def test_multiline():'
     f.input_buffer += 'print 1'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == ''
+    yield assert_equal, out_value, ''
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1\n'
+    yield assert_equal, out_value, '1\n'
     f = TestPrefilterFrontEnd()
     f.input_buffer='(1 +'
     f._on_enter()
     f.input_buffer += '0)'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == ''
+    yield assert_equal, out_value, ''
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1\n'
+    yield assert_equal, out_value, '1\n'
 
 
 @isolate_ipython0
@@ -113,13 +139,13 b' def test_capture():'
             'import os; out=os.fdopen(1, "w"); out.write("1") ; out.flush()'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1'
+    yield assert_equal, out_value, '1'
     f = TestPrefilterFrontEnd()
     f.input_buffer = \
             'import os; out=os.fdopen(2, "w"); out.write("1") ; out.flush()'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1'
+    yield assert_equal, out_value, '1'
 
      
 @isolate_ipython0
@@ -129,10 +155,16 b' def test_magic():'
         This test is fairly fragile and will break when magics change.
     """
     f = TestPrefilterFrontEnd()
+    # Before checking the interactive namespace, make sure it's clear (it can
+    # otherwise pick up things stored in the user's local db)
+    f.input_buffer += '%reset -f'
+    f._on_enter()
+    f.complete_current_input()
+    # Now, run the %who magic and check output
     f.input_buffer += '%who'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == 'Interactive namespace is empty.\n'
+    assert_equal(out_value, 'Interactive namespace is empty.\n')
 
 
 @isolate_ipython0
@@ -156,8 +188,8 b' def test_help():'
 
 
 @isolate_ipython0
-def test_completion():
-    """ Test command-line completion.
+def test_completion_simple():
+    """ Test command-line completion on trivial examples.
     """
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'zzza = 1'
@@ -167,8 +199,47 b' def test_completion():'
     f.input_buffer = 'zz'
     f.complete_current_input()
     out_value = f.out.getvalue()
-    assert out_value == '\nzzza zzzb '
-    assert f.input_buffer == 'zzz'
+    yield assert_equal, out_value, '\nzzza zzzb '
+    yield assert_equal, f.input_buffer, 'zzz'
+
+
+@isolate_ipython0
+def test_completion_parenthesis():
+    """ Test command-line completion when a parenthesis is open.
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'zzza = 1'
+    f._on_enter()
+    f.input_buffer = 'zzzb = 2'
+    f._on_enter()
+    f.input_buffer = 'map(zz'
+    f.complete_current_input()
+    out_value = f.out.getvalue()
+    yield assert_equal, out_value, '\nzzza zzzb '
+    yield assert_equal, f.input_buffer, 'map(zzz'
+
+
+@isolate_ipython0
+def test_completion_indexing():
+    """ Test command-line completion when indexing on objects.
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'a = [0]'
+    f._on_enter()
+    f.input_buffer = 'a[0].'
+    f.complete_current_input()
+    assert_equal(f.input_buffer, 'a[0].__')
+
+
+@isolate_ipython0
+def test_completion_equal():
+    """ Test command-line completion when the delimiter is "=", not " ".
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'a=1.'
+    f.complete_current_input()
+    assert_equal(f.input_buffer, 'a=1.__')
+
 
 
 if __name__ == '__main__':
@@ -177,4 +248,5 b" if __name__ == '__main__':"
     test_execution()
     test_multiline()
     test_capture()
-    test_completion()
+    test_completion_simple()
+    test_completion_complex()

IPython/frontend/tests/test_process.py

@@ -5,18 +5,18 b' Test process execution and IO redirection.'
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from cStringIO import StringIO
 from time import sleep
 import sys
 
-from IPython.frontend._process import PipedProcess
+from IPython.frontend.process import PipedProcess
 from IPython.testing import decorators as testdec
 
 

IPython/frontend/wx/console_widget.py

@@ -25,6 +25,8 b' import wx.stc as stc'
 from wx.py import editwindow
 import time
 import sys
+import string
+
 LINESEP = '\n'
 if sys.platform == 'win32':
     LINESEP = '\n\r'
@@ -33,20 +35,26 b' import re'
 
 # FIXME: Need to provide an API for non user-generated display on the
 # screen: this should not be editable by the user.
+#-------------------------------------------------------------------------------
+# Constants 
+#-------------------------------------------------------------------------------
+_COMPLETE_BUFFER_MARKER = 31
+_ERROR_MARKER = 30
+_INPUT_MARKER = 29
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
     _DEFAULT_SIZE = 12
 
 _DEFAULT_STYLE = {
-    'stdout'      : 'fore:#0000FF',
-    'stderr'      : 'fore:#007f00',
-    'trace'       : 'fore:#FF0000',
-
+    #background definition
     'default'     : 'size:%d' % _DEFAULT_SIZE,
     'bracegood'   : 'fore:#00AA00,back:#000000,bold',
     'bracebad'    : 'fore:#FF0000,back:#000000,bold',
 
+    # Edge column: a number of None
+    'edge_column' : -1,
+
     # properties for the various Python lexer styles
     'comment'       : 'fore:#007F00',
     'number'        : 'fore:#007F7F',
@@ -57,7 +65,24 b' _DEFAULT_STYLE = {'
     'tripledouble'  : 'fore:#7F0000',
     'class'         : 'fore:#0000FF,bold,underline',
     'def'           : 'fore:#007F7F,bold',
-    'operator'      : 'bold'
+    'operator'      : 'bold',
+
+    # Default colors
+    'trace'         : '#FAFAF1', # Nice green
+    'stdout'        : '#FDFFD3', # Nice yellow
+    'stderr'        : '#FFF1F1', # Nice red
+
+    # Default scintilla settings
+    'antialiasing'  : True,
+    'carret_color'  : 'BLACK',
+    'background_color' :'WHITE', 
+
+    #prompt definition
+    'prompt_in1'    : \
+        '\n\x01\x1b[0;34m\x02In [\x01\x1b[1;34m\x02$number\x01\x1b[0;34m\x02]: \x01\x1b[0m\x02',
+
+    'prompt_out': \
+        '\x01\x1b[0;31m\x02Out[\x01\x1b[1;31m\x02$number\x01\x1b[0;31m\x02]: \x01\x1b[0m\x02',
     }
 
 # new style numbers
@@ -69,6 +94,47 b' _TRACE_STYLE = 17'
 # system colors
 #SYS_COLOUR_BACKGROUND = wx.SystemSettings.GetColour(wx.SYS_COLOUR_BACKGROUND)
 
+# Translation table from ANSI escape sequences to color. 
+ANSI_STYLES = {'0;30': [0, 'BLACK'],            '0;31': [1, 'RED'],
+               '0;32': [2, 'GREEN'],           '0;33': [3, 'BROWN'],
+               '0;34': [4, 'BLUE'],            '0;35': [5, 'PURPLE'],
+               '0;36': [6, 'CYAN'],            '0;37': [7, 'LIGHT GREY'],
+               '1;30': [8, 'DARK GREY'],       '1;31': [9, 'RED'],
+               '1;32': [10, 'SEA GREEN'],      '1;33': [11, 'YELLOW'],
+               '1;34': [12, 'LIGHT BLUE'],     '1;35': 
+                                                 [13, 'MEDIUM VIOLET RED'],
+               '1;36': [14, 'LIGHT STEEL BLUE'], '1;37': [15, 'YELLOW']}
+
+# XXX: Maybe one day we should factor this code with ColorANSI. Right now
+# ColorANSI is hard to reuse and makes our code more complex.
+
+#we define platform specific fonts
+if wx.Platform == '__WXMSW__':
+    FACES = { 'times': 'Times New Roman',
+                'mono' : 'Courier New',
+                'helv' : 'Arial',
+                'other': 'Comic Sans MS',
+                'size' : 10,
+                'size2': 8,
+                }
+elif wx.Platform == '__WXMAC__':
+    FACES = { 'times': 'Times New Roman',
+                'mono' : 'Monaco',
+                'helv' : 'Arial',
+                'other': 'Comic Sans MS',
+                'size' : 10,
+                'size2': 8,
+                }
+else:
+    FACES = { 'times': 'Times',
+                'mono' : 'Courier',
+                'helv' : 'Helvetica',
+                'other': 'new century schoolbook',
+                'size' : 10,
+                'size2': 8,
+                }
+ 
+
 #-------------------------------------------------------------------------------
 # The console widget class
 #-------------------------------------------------------------------------------
@@ -83,6 +149,9 b' class ConsoleWidget(editwindow.EditWindow):'
     # stored.
     title = 'Console'
 
+    # Last prompt printed
+    last_prompt = ''
+
     # The buffer being edited.
     def _set_input_buffer(self, string):
         self.SetSelection(self.current_prompt_pos, self.GetLength())
@@ -103,19 +172,11 b' class ConsoleWidget(editwindow.EditWindow):'
 
     # Translation table from ANSI escape sequences to color. Override
     # this to specify your colors.
-    ANSI_STYLES = {'0;30': [0, 'BLACK'],            '0;31': [1, 'RED'],
-                   '0;32': [2, 'GREEN'],            '0;33': [3, 'BROWN'],
-                   '0;34': [4, 'BLUE'],             '0;35': [5, 'PURPLE'],
-                   '0;36': [6, 'CYAN'],             '0;37': [7, 'LIGHT GREY'],
-                   '1;30': [8, 'DARK GREY'],        '1;31': [9, 'RED'],
-                   '1;32': [10, 'SEA GREEN'],       '1;33': [11, 'YELLOW'],
-                   '1;34': [12, 'LIGHT BLUE'],      '1;35':
-                                                     [13, 'MEDIUM VIOLET RED'],
-                   '1;36': [14, 'LIGHT STEEL BLUE'], '1;37': [15, 'YELLOW']}
-
-    # The color of the carret (call _apply_style() after setting)
-    carret_color = 'BLACK'
+    ANSI_STYLES = ANSI_STYLES.copy()
     
+    # Font faces
+    faces = FACES.copy()
+                 
     # Store the last time a refresh was done
     _last_refresh_time = 0
 
@@ -126,7 +187,11 b' class ConsoleWidget(editwindow.EditWindow):'
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
                         size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
-        self._configure_scintilla()
+        self.configure_scintilla()
+        # Track if 'enter' key as ever been processed
+        # This variable will only be reallowed until key goes up
+        self.enter_catched = False 
+        self.current_prompt_pos = 0
 
         self.Bind(wx.EVT_KEY_DOWN, self._on_key_down)
         self.Bind(wx.EVT_KEY_UP, self._on_key_up)
@@ -193,8 +258,19 b' class ConsoleWidget(editwindow.EditWindow):'
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
         self.EnsureCaretVisible()
+        self.last_prompt = prompt
 
 
+    def continuation_prompt(self):
+        """ Returns the current continuation prompt.
+            We need to implement this method here to deal with the
+            ascii escape sequences cleaning up.
+        """
+        # ASCII-less prompt
+        ascii_less = ''.join(self.color_pat.split(self.last_prompt)[2::2])
+        return "."*(len(ascii_less)-2) + ': '
+ 
+
     def scroll_to_bottom(self):
         maxrange = self.GetScrollRange(wx.VERTICAL)
         self.ScrollLines(maxrange)
@@ -216,37 +292,24 b' class ConsoleWidget(editwindow.EditWindow):'
         """
         return self.GetSize()[0]/self.GetCharWidth()
 
-    #--------------------------------------------------------------------------
-    # EditWindow API
-    #--------------------------------------------------------------------------
 
-    def OnUpdateUI(self, event):
-        """ Override the OnUpdateUI of the EditWindow class, to prevent 
-            syntax highlighting both for faster redraw, and for more
-            consistent look and feel.
+    def configure_scintilla(self):
+        """ Set up all the styling option of the embedded scintilla
+            widget.
         """
+        p = self.style.copy()
+        
+        # Marker for complete buffer.
+        self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['trace'])
 
-    #--------------------------------------------------------------------------
-    # Private API
-    #--------------------------------------------------------------------------
-    
-    def _apply_style(self):
-        """ Applies the colors for the different text elements and the
-            carret.
-        """
-        self.SetCaretForeground(self.carret_color)
-
-        #self.StyleClearAll()
-        self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT,  
-                          "fore:#FF0000,back:#0000FF,bold")
-        self.StyleSetSpec(stc.STC_STYLE_BRACEBAD,
-                          "fore:#000000,back:#FF0000,bold")
-
-        for style in self.ANSI_STYLES.values():
-            self.StyleSetSpec(style[0], "bold,fore:%s" % style[1])
+        # Marker for current input buffer.
+        self.MarkerDefine(_INPUT_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['stdout'])
+        # Marker for tracebacks.
+        self.MarkerDefine(_ERROR_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['stderr'])
 
-   
-    def _configure_scintilla(self):
         self.SetEOLMode(stc.STC_EOL_LF)
 
         # Ctrl"+" or Ctrl "-" can be used to zoomin/zoomout the text inside 
@@ -268,7 +331,9 b' class ConsoleWidget(editwindow.EditWindow):'
         self.SetWrapMode(stc.STC_WRAP_CHAR)
         self.SetWrapMode(stc.STC_WRAP_WORD)
         self.SetBufferedDraw(True)
-        self.SetUseAntiAliasing(True)
+
+        self.SetUseAntiAliasing(p['antialiasing'])
+
         self.SetLayoutCache(stc.STC_CACHE_PAGE)
         self.SetUndoCollection(False)
         self.SetUseTabs(True)
@@ -289,23 +354,48 b' class ConsoleWidget(editwindow.EditWindow):'
         self.SetMarginWidth(1, 0)
         self.SetMarginWidth(2, 0)
 
-        self._apply_style()
-
         # Xterm escape sequences
         self.color_pat = re.compile('\x01?\x1b\[(.*?)m\x02?')
         self.title_pat = re.compile('\x1b]0;(.*?)\x07')
 
-        #self.SetEdgeMode(stc.STC_EDGE_LINE)
-        #self.SetEdgeColumn(80)
-
         # styles
-        p = self.style
-        self.StyleSetSpec(stc.STC_STYLE_DEFAULT, p['default'])
+        
+        self.SetCaretForeground(p['carret_color'])
+        
+        background_color = p['background_color']
+            
+        if 'default' in p:
+            if 'back' not in p['default']:
+                p['default'] += ',back:%s' % background_color
+            if 'size' not in p['default']:
+                p['default'] += ',size:%s' % self.faces['size']
+            if 'face' not in p['default']:
+                p['default'] += ',face:%s' % self.faces['mono']
+            
+            self.StyleSetSpec(stc.STC_STYLE_DEFAULT, p['default'])
+        else:
+            self.StyleSetSpec(stc.STC_STYLE_DEFAULT, 
+                            "fore:%s,back:%s,size:%d,face:%s" 
+                            % (self.ANSI_STYLES['0;30'][1], 
+                               background_color,
+                               self.faces['size'], self.faces['mono']))
+        
         self.StyleClearAll()
+        
+        # XXX: two lines below are usefull if not using the lexer        
+        #for style in self.ANSI_STYLES.values():
+        #    self.StyleSetSpec(style[0], "bold,fore:%s" % style[1])        
+
+        # prompt definition
+        self.prompt_in1 = p['prompt_in1']
+        self.prompt_out = p['prompt_out']
+
+        self.output_prompt_template = string.Template(self.prompt_out)
+        self.input_prompt_template = string.Template(self.prompt_in1)
+
         self.StyleSetSpec(_STDOUT_STYLE, p['stdout'])
         self.StyleSetSpec(_STDERR_STYLE, p['stderr'])
         self.StyleSetSpec(_TRACE_STYLE, p['trace'])
-
         self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT, p['bracegood'])
         self.StyleSetSpec(stc.STC_STYLE_BRACEBAD, p['bracebad'])
         self.StyleSetSpec(stc.STC_P_COMMENTLINE, p['comment'])
@@ -321,6 +411,28 b' class ConsoleWidget(editwindow.EditWindow):'
         self.StyleSetSpec(stc.STC_P_OPERATOR, p['operator'])
         self.StyleSetSpec(stc.STC_P_COMMENTBLOCK, p['comment'])
 
+        edge_column = p['edge_column']
+        if edge_column is not None and edge_column > 0:
+            #we add a vertical line to console widget
+            self.SetEdgeMode(stc.STC_EDGE_LINE)
+            self.SetEdgeColumn(edge_column)
+ 
+ 
+    #--------------------------------------------------------------------------
+    # EditWindow API
+    #--------------------------------------------------------------------------
+
+    def OnUpdateUI(self, event):
+        """ Override the OnUpdateUI of the EditWindow class, to prevent 
+            syntax highlighting both for faster redraw, and for more
+            consistent look and feel.
+        """
+
+       
+    #--------------------------------------------------------------------------
+    # Private API
+    #--------------------------------------------------------------------------
+   
     def _on_key_down(self, event, skip=True):
         """ Key press callback used for correcting behavior for 
             console-like interfaces: the cursor is constraint to be after
@@ -329,6 +441,11 b' class ConsoleWidget(editwindow.EditWindow):'
             Return True if event as been catched.
         """
         catched = True
+        # XXX: Would the right way to do this be to have a
+        #  dictionary at the instance level associating keys with
+        #  callbacks? How would we deal with inheritance? And Do the
+        #  different callbacks share local variables?
+
         # Intercept some specific keys.
         if event.KeyCode == ord('L') and event.ControlDown() :
             self.scroll_to_bottom()
@@ -346,6 +463,10 b' class ConsoleWidget(editwindow.EditWindow):'
             self.ScrollPages(-1)
         elif event.KeyCode == wx.WXK_PAGEDOWN:
             self.ScrollPages(1)
+        elif event.KeyCode == wx.WXK_HOME:
+            self.GotoPos(self.GetLength())
+        elif event.KeyCode == wx.WXK_END:
+            self.GotoPos(self.GetLength())
         elif event.KeyCode == wx.WXK_UP and event.ShiftDown():
             self.ScrollLines(-1)
         elif event.KeyCode == wx.WXK_DOWN and event.ShiftDown():
@@ -357,16 +478,20 b' class ConsoleWidget(editwindow.EditWindow):'
             event.Skip()
         else:
             if event.KeyCode in (13, wx.WXK_NUMPAD_ENTER) and \
-                        event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
+                                event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN,
+                                                    wx.MOD_SHIFT):
                 catched = True
-                self.CallTipCancel()
-                self.write('\n', refresh=False)
-                # Under windows scintilla seems to be doing funny stuff to the 
-                # line returns here, but the getter for input_buffer filters 
-                # this out.
-                if sys.platform == 'win32':
-                    self.input_buffer = self.input_buffer
-                self._on_enter()
+                if not self.enter_catched:
+                    self.CallTipCancel()
+                    if event.Modifiers == wx.MOD_SHIFT:
+                        # Try to force execution
+                        self.GotoPos(self.GetLength())
+                        self.write('\n' + self.continuation_prompt(), 
+                                        refresh=False)
+                        self._on_enter()
+                    else:
+                        self._on_enter()
+                    self.enter_catched = True
 
             elif event.KeyCode == wx.WXK_HOME:
                 if event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
@@ -391,16 +516,28 b' class ConsoleWidget(editwindow.EditWindow):'
                 catched = True
 
             elif event.KeyCode in (wx.WXK_LEFT, wx.WXK_BACK):
-                if self.GetCurrentPos() > self.current_prompt_pos:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() - 1):
+                    event.Skip()
+                catched = True
+
+            elif event.KeyCode == wx.WXK_RIGHT:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() + 1):
+                    event.Skip()
+                catched = True
+
+
+            elif event.KeyCode == wx.WXK_DELETE:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() - 1):
                     event.Skip()
                 catched = True
 
             if skip and not catched:
                 # Put the cursor back in the edit region
-                if self.GetCurrentPos() < self.current_prompt_pos:
-                    self.GotoPos(self.current_prompt_pos)
-                else:
-                    event.Skip()
+                if not self._keep_cursor_in_buffer():
+                    if not (self.GetCurrentPos() == self.GetLength()
+                                and event.KeyCode == wx.WXK_DELETE):
+                        event.Skip()
+                    catched = True
 
         return catched
 
@@ -408,17 +545,69 b' class ConsoleWidget(editwindow.EditWindow):'
     def _on_key_up(self, event, skip=True):
         """ If cursor is outside the editing region, put it back.
         """
-        event.Skip()
-        if self.GetCurrentPos() < self.current_prompt_pos:
-            self.GotoPos(self.current_prompt_pos)
+        if skip:
+            event.Skip()
+        self._keep_cursor_in_buffer()
+
+
+    # XXX:  I need to avoid the problem of having an empty glass;
+    def _keep_cursor_in_buffer(self, pos=None):
+        """ Checks if the cursor is where it is allowed to be. If not,
+            put it back.
 
+            Returns
+            -------
+            cursor_moved: Boolean
+                whether or not the cursor was moved by this routine.
+
+            Notes
+            ------
+                WARNING: This does proper checks only for horizontal
+                movements.
+        """
+        if pos is None:
+            current_pos = self.GetCurrentPos()
+        else:
+            current_pos = pos
+        if  current_pos < self.current_prompt_pos:
+            self.GotoPos(self.current_prompt_pos)
+            return True
+        line_num = self.LineFromPosition(current_pos)
+        if not current_pos > self.GetLength():
+            line_pos = self.GetColumn(current_pos)
+        else:
+            line_pos = self.GetColumn(self.GetLength())
+        line = self.GetLine(line_num)
+        # Jump the continuation prompt
+        continuation_prompt = self.continuation_prompt()
+        if ( line.startswith(continuation_prompt)
+                     and line_pos < len(continuation_prompt)):
+            if line_pos < 2:
+                # We are at the beginning of the line, trying to move
+                # forward: jump forward.
+                self.GotoPos(current_pos + 1 +
+                                    len(continuation_prompt) - line_pos)
+            else:
+                # Jump back up
+                self.GotoPos(self.GetLineEndPosition(line_num-1))
+            return True
+        elif ( current_pos > self.GetLineEndPosition(line_num) 
+                        and not current_pos == self.GetLength()): 
+            # Jump to next line
+            self.GotoPos(current_pos + 1 +
+                                    len(continuation_prompt))
+            return True
+
+        # We re-allow enter event processing
+        self.enter_catched = False 
+        return False
 
 
 if __name__ == '__main__':
     # Some simple code to test the console widget.
     class MainWindow(wx.Frame):
         def __init__(self, parent, id, title):
-            wx.Frame.__init__(self, parent, id, title, size=(300,250))
+            wx.Frame.__init__(self, parent, id, title, size=(300, 250))
             self._sizer = wx.BoxSizer(wx.VERTICAL)
             self.console_widget = ConsoleWidget(self)
             self._sizer.Add(self.console_widget, 1, wx.EXPAND)

IPython/frontend/wx/ipythonx.py

@@ -80,6 +80,15 b' class IPythonX(wx.Frame):'
         self.SetSizer(self._sizer)
         self.SetAutoLayout(1)
         self.Show(True)
+        wx.EVT_CLOSE(self, self.on_close)
+
+        
+    def on_close(self, event):
+        """ Called on closing the windows. 
+            
+            Stops the event loop, to close all the child windows.
+        """
+        wx.CallAfter(wx.Exit)
 
 
 def main():

IPython/frontend/wx/wx_frontend.py

@@ -25,38 +25,19 b' __docformat__ = "restructuredtext en"'
 # Major library imports
 import re
 import __builtin__
-from time import sleep
 import sys
 from threading import Lock
-import string
 
 import wx
 from wx import stc
 
 # Ipython-specific imports.
-from IPython.frontend._process import PipedProcess
-from console_widget import ConsoleWidget
+from IPython.frontend.process import PipedProcess
+from console_widget import ConsoleWidget, _COMPLETE_BUFFER_MARKER, \
+    _ERROR_MARKER, _INPUT_MARKER
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 
 #-------------------------------------------------------------------------------
-# Constants 
-#-------------------------------------------------------------------------------
-
-_COMPLETE_BUFFER_BG = '#FAFAF1' # Nice green
-_INPUT_BUFFER_BG = '#FDFFD3' # Nice yellow
-_ERROR_BG = '#FFF1F1' # Nice red
-
-_COMPLETE_BUFFER_MARKER = 31
-_ERROR_MARKER = 30
-_INPUT_MARKER = 29
-
-prompt_in1 = \
-        '\n\x01\x1b[0;34m\x02In [\x01\x1b[1;34m\x02$number\x01\x1b[0;34m\x02]: \x01\x1b[0m\x02'
-
-prompt_out = \
-    '\x01\x1b[0;31m\x02Out[\x01\x1b[1;31m\x02$number\x01\x1b[0;31m\x02]: \x01\x1b[0m\x02'
-
-#-------------------------------------------------------------------------------
 # Classes to implement the Wx frontend
 #-------------------------------------------------------------------------------
 class WxController(ConsoleWidget, PrefilterFrontEnd):
@@ -66,11 +47,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     This class inherits from ConsoleWidget, that provides a console-like
     widget to provide a text-rendering widget suitable for a terminal.
     """
-
-    output_prompt_template = string.Template(prompt_out)
-
-    input_prompt_template = string.Template(prompt_in1)
-
+    
     # Print debug info on what is happening to the console.
     debug = False
 
@@ -138,25 +115,24 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
                  size=wx.DefaultSize,
                  style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
+                 styledef=None,
                  *args, **kwds):
         """ Create Shell instance.
+
+            Parameters
+            -----------
+            styledef : dict, optional
+                styledef is the dictionary of options used to define the
+                style.
         """
+        if styledef is not None:
+            self.style = styledef
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
         
         # Stick in our own raw_input:
         self.ipython0.raw_input = self.raw_input
 
-        # Marker for complete buffer.
-        self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_COMPLETE_BUFFER_BG)
-        # Marker for current input buffer.
-        self.MarkerDefine(_INPUT_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_INPUT_BUFFER_BG)
-        # Marker for tracebacks.
-        self.MarkerDefine(_ERROR_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_ERROR_BG)
-
         # A time for flushing the write buffer
         BUFFER_FLUSH_TIMER_ID = 100
         self._buffer_flush_timer = wx.Timer(self, BUFFER_FLUSH_TIMER_ID)
@@ -171,8 +147,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             self.shell.user_ns['self'] = self
         # Inject our own raw_input in namespace
         self.shell.user_ns['raw_input'] = self.raw_input
-
-
+        
     def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
@@ -251,11 +226,8 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
-            offset=0
             if completions:
-                complete_sep =  re.compile('[\s\{\}\[\]\(\)\= ,:]')
-                residual = complete_sep.split(line)[-1]
-                offset = len(residual)
+                offset = len(self._get_completion_text(line))
                 self.pop_completion(completions, offset=offset)
                 if self.debug:
                     print >>sys.__stdout__, completions 
@@ -276,6 +248,14 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                                         milliseconds=100, oneShot=True)
 
 
+    def clear_screen(self):
+        """ Empty completely the widget.
+        """
+        self.ClearAll()
+        self.new_prompt(self.input_prompt_template.substitute(
+                                number=(self.last_result['number'] + 1)))
+
+
     #--------------------------------------------------------------------------
     # LineFrontEnd interface 
     #--------------------------------------------------------------------------
@@ -299,6 +279,41 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                                             raw_string=raw_string)
         wx.CallAfter(callback)
 
+
+    def execute_command(self, command, hidden=False):
+        """ Execute a command, not only in the model, but also in the
+            view.
+        """
+        # XXX: This method needs to be integrated in the base fronted
+        # interface
+        if hidden:
+            return self.shell.execute(command)
+        else:
+            # XXX: we are not storing the input buffer previous to the
+            # execution, as this forces us to run the execution
+            # input_buffer a yield, which is not good.
+            ##current_buffer = self.shell.control.input_buffer
+            command = command.rstrip()
+            if len(command.split('\n')) > 1:
+                # The input command is several lines long, we need to
+                # force the execution to happen
+                command += '\n'
+            cleaned_command = self.prefilter_input(command)
+            self.input_buffer = command
+            # Do not use wx.Yield() (aka GUI.process_events()) to avoid
+            # recursive yields.
+            self.ProcessEvent(wx.PaintEvent())
+            self.write('\n')
+            if not self.is_complete(cleaned_command + '\n'):
+                self._colorize_input_buffer()
+                self.render_error('Incomplete or invalid input')
+                self.new_prompt(self.input_prompt_template.substitute(
+                                number=(self.last_result['number'] + 1)))
+                return False
+            self._on_enter()
+            return True
+
+
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
@@ -356,10 +371,16 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         self._markers[i] = self.MarkerAdd(i, _INPUT_MARKER)
 
 
+    def continuation_prompt(self, *args, **kwargs):
+        # Avoid multiple inheritence, be explicit about which
+        # parent method class gets called
+        return ConsoleWidget.continuation_prompt(self, *args, **kwargs)
+
+
     def write(self, *args, **kwargs):
         # Avoid multiple inheritence, be explicit about which
         # parent method class gets called
-        ConsoleWidget.write(self, *args, **kwargs)
+        return ConsoleWidget.write(self, *args, **kwargs)
 
 
     def _on_key_down(self, event, skip=True):
@@ -367,7 +388,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             widget handle them, and put our logic afterward.
         """
         # FIXME: This method needs to be broken down in smaller ones.
-        current_line_number = self.GetCurrentLine()
+        current_line_num = self.GetCurrentLine()
         if event.KeyCode in (ord('c'), ord('C')) and event.ControlDown():
             # Capture Control-C
             if self._input_state == 'subprocess':
@@ -413,7 +434,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
         else:
             # Up history
             if event.KeyCode == wx.WXK_UP and (
-                    ( current_line_number == self.current_prompt_line and
+                    ( current_line_num == self.current_prompt_line and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_previous(
@@ -425,7 +446,7 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                         self.GotoPos(self.current_prompt_pos)
             # Down history
             elif event.KeyCode == wx.WXK_DOWN and (
-                    ( current_line_number == self.LineCount -1 and
+                    ( current_line_num == self.LineCount -1 and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_next()
@@ -433,15 +454,43 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                current_line, current_line_number = self.CurLine
+                current_line, current_line_num = self.CurLine
                 if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
                 else:
                     event.Skip()
+            elif event.KeyCode == wx.WXK_BACK:
+                # If characters where erased, check if we have to
+                # remove a line.
+                # XXX: What about DEL?
+                # FIXME: This logics should be in ConsoleWidget, as it is
+                # independant of IPython
+                current_line, _ = self.CurLine
+                current_pos = self.GetCurrentPos()
+                current_line_num = self.LineFromPosition(current_pos)
+                current_col = self.GetColumn(current_pos)
+                len_prompt = len(self.continuation_prompt())
+                if ( current_line.startswith(self.continuation_prompt())
+                                            and current_col == len_prompt):
+                    new_lines = []
+                    for line_num, line in enumerate(
+                                    self.input_buffer.split('\n')):
+                        if (line_num + self.current_prompt_line ==
+                                            current_line_num):
+                            new_lines.append(line[len_prompt:])
+                        else:
+                            new_lines.append('\n'+line)
+                    # The first character is '\n', due to the above
+                    # code:
+                    self.input_buffer = ''.join(new_lines)[1:]
+                    self.GotoPos(current_pos - 1 - len_prompt)
+                else:
+                    ConsoleWidget._on_key_down(self, event, skip=skip)
             else:
                 ConsoleWidget._on_key_down(self, event, skip=skip)
+                        
 
 
     def _on_key_up(self, event, skip=True):
@@ -453,14 +502,40 b' class WxController(ConsoleWidget, PrefilterFrontEnd):'
             wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
+        # Make sure the continuation_prompts are always followed by a 
+        # whitespace
+        new_lines = []
+        if self._input_state == 'readline':
+            position = self.GetCurrentPos()
+            continuation_prompt = self.continuation_prompt()[:-1]
+            for line in self.input_buffer.split('\n'):
+                if not line == continuation_prompt:
+                    new_lines.append(line)
+            self.input_buffer = '\n'.join(new_lines)
+            self.GotoPos(position)
 
 
     def _on_enter(self):
         """ Called on return key down, in readline input_state.
         """
+        last_line_num = self.LineFromPosition(self.GetLength())
+        current_line_num = self.LineFromPosition(self.GetCurrentPos())
+        new_line_pos = (last_line_num - current_line_num)
         if self.debug:
             print >>sys.__stdout__, repr(self.input_buffer)
-        PrefilterFrontEnd._on_enter(self)
+        self.write('\n', refresh=False)
+        # Under windows scintilla seems to be doing funny
+        # stuff to the line returns here, but the getter for
+        # input_buffer filters this out.
+        if sys.platform == 'win32':
+            self.input_buffer = self.input_buffer
+        old_prompt_num = self.current_prompt_pos
+        has_executed = PrefilterFrontEnd._on_enter(self, 
+                                            new_line_pos=new_line_pos)
+        if old_prompt_num == self.current_prompt_pos:
+            # No execution has happened 
+            self.GotoPos(self.GetLineEndPosition(current_line_num + 1))
+        return has_executed
 
 
     #--------------------------------------------------------------------------

IPython/genutils.py

@@ -1007,7 +1007,17 b' def get_security_dir():'
     else:
         os.chmod(security_dir, 0700)
     return security_dir
-        
+
+def get_log_dir():
+    """Get the IPython log directory.
+    
+    If the log directory does not exist, it is created.
+    """
+    log_dir = os.path.join(get_ipython_dir(), 'log')
+    if not os.path.isdir(log_dir):
+        os.mkdir(log_dir, 0777)
+    return log_dir
+
 #****************************************************************************
 # strings and text
 

IPython/iplib.py

@@ -54,7 +54,7 b' from pprint import pprint, pformat'
 from IPython import Debugger,OInspect,PyColorize,ultraTB
 from IPython.ColorANSI import ColorScheme,ColorSchemeTable  # too long names
 from IPython.Extensions import pickleshare
-from IPython.FakeModule import FakeModule
+from IPython.FakeModule import FakeModule, init_fakemod_dict
 from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns
 from IPython.Logger import Logger
 from IPython.Magic import Magic
@@ -108,6 +108,201 b' def softspace(file, newvalue):'
     return oldvalue
 
 
+def user_setup(ipythondir,rc_suffix,mode='install',interactive=True):
+    """Install or upgrade the user configuration directory.
+
+    Can be called when running for the first time or to upgrade the user's
+    .ipython/ directory.
+
+    Parameters
+    ----------
+      ipythondir : path
+        The directory to be used for installation/upgrade.  In 'install' mode,
+        if this path already exists, the function exits immediately.
+
+      rc_suffix : str
+        Extension for the config files.  On *nix platforms it is typically the
+        empty string, while Windows normally uses '.ini'.
+
+      mode : str, optional
+        Valid modes are 'install' and 'upgrade'.
+
+      interactive : bool, optional
+        If False, do not wait for user input on any errors.  Normally after
+        printing its status information, this function waits for the user to
+        hit Return before proceeding.  This is because the default use case is
+        when first installing the IPython configuration, so we want the user to
+        acknowledge the initial message, which contains some useful
+        information.
+        """
+
+    # For automatic use, deactivate all i/o
+    if interactive:
+        def wait():
+            try:
+                raw_input("Please press <RETURN> to start IPython.")
+            except EOFError:
+                print >> Term.cout
+            print '*'*70
+
+        def printf(s):
+            print s
+    else:
+        wait = lambda : None
+        printf = lambda s : None
+
+    # Install mode should be re-entrant: if the install dir already exists,
+    # bail out cleanly.
+    # XXX.  This is too hasty to return.  We need to check to make sure that
+    # all the expected config files and directories are actually there. We
+    # currently have a failure mode if someone deletes a needed config file
+    # but still has the ipythondir.
+    if mode == 'install' and os.path.isdir(ipythondir):
+        return
+
+    cwd = os.getcwd()  # remember where we started
+    glb = glob.glob
+
+    printf('*'*70)
+    if mode == 'install':
+        printf(
+"""Welcome to IPython. I will try to create a personal configuration directory
+where you can customize many aspects of IPython's functionality in:\n""")
+    else:
+        printf('I am going to upgrade your configuration in:')
+
+    printf(ipythondir)
+
+    rcdirend = os.path.join('IPython','UserConfig')
+    cfg = lambda d: os.path.join(d,rcdirend)
+    try:
+        rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
+        printf("Initializing from configuration: %s" % rcdir)
+    except IndexError:
+        warning = """
+Installation error. IPython's directory was not found.
+
+Check the following:
+
+The ipython/IPython directory should be in a directory belonging to your
+PYTHONPATH environment variable (that is, it should be in a directory
+belonging to sys.path). You can copy it explicitly there or just link to it.
+
+IPython will create a minimal default configuration for you.
+
+"""
+        warn(warning)
+        wait()
+
+        if sys.platform =='win32':
+            inif = 'ipythonrc.ini'
+        else:
+            inif = 'ipythonrc'
+        minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults',
+                         inif : '# intentionally left blank' }
+        os.makedirs(ipythondir, mode = 0777)
+        for f, cont in minimal_setup.items():
+            # In 2.5, this can be more cleanly done using 'with'
+            fobj = file(ipythondir + '/' + f,'w')
+            fobj.write(cont)
+            fobj.close()
+
+        return
+
+    if mode == 'install':
+        try:
+            shutil.copytree(rcdir,ipythondir)
+            os.chdir(ipythondir)
+            rc_files = glb("ipythonrc*")
+            for rc_file in rc_files:
+                os.rename(rc_file,rc_file+rc_suffix)
+        except:
+            warning = """
+
+There was a problem with the installation:
+%s
+Try to correct it or contact the developers if you think it's a bug.
+IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
+            warn(warning)
+            wait()
+            return
+
+    elif mode == 'upgrade':
+        try:
+            os.chdir(ipythondir)
+        except:
+            printf("""
+Can not upgrade: changing to directory %s failed. Details:
+%s
+""" % (ipythondir,sys.exc_info()[1]) )
+            wait()
+            return
+        else:
+            sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
+            for new_full_path in sources:
+                new_filename = os.path.basename(new_full_path)
+                if new_filename.startswith('ipythonrc'):
+                    new_filename = new_filename + rc_suffix
+                # The config directory should only contain files, skip any
+                # directories which may be there (like CVS)
+                if os.path.isdir(new_full_path):
+                    continue
+                if os.path.exists(new_filename):
+                    old_file = new_filename+'.old'
+                    if os.path.exists(old_file):
+                        os.remove(old_file)
+                    os.rename(new_filename,old_file)
+                shutil.copy(new_full_path,new_filename)
+    else:
+        raise ValueError('unrecognized mode for install: %r' % mode)
+
+    # Fix line-endings to those native to each platform in the config
+    # directory.
+    try:
+        os.chdir(ipythondir)
+    except:
+        printf("""
+Problem: changing to directory %s failed.
+Details:
+%s
+
+Some configuration files may have incorrect line endings.  This should not
+cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1]) )
+        wait()
+    else:
+        for fname in glb('ipythonrc*'):
+            try:
+                native_line_ends(fname,backup=0)
+            except IOError:
+                pass
+
+    if mode == 'install':
+        printf("""
+Successful installation!
+
+Please read the sections 'Initial Configuration' and 'Quick Tips' in the
+IPython manual (there are both HTML and PDF versions supplied with the
+distribution) to make sure that your system environment is properly configured
+to take advantage of IPython's features.
+
+Important note: the configuration system has changed! The old system is
+still in place, but its setting may be partly overridden by the settings in 
+"~/.ipython/ipy_user_conf.py" config file. Please take a look at the file 
+if some of the new settings bother you. 
+
+""")
+    else:
+        printf("""
+Successful upgrade!
+
+All files in your directory:
+%(ipythondir)s
+which would have been overwritten by the upgrade were backed up with a .old
+extension.  If you had made particular customizations in those files you may
+want to merge them back into the new files.""" % locals() )
+    wait()
+    os.chdir(cwd)
+
 #****************************************************************************
 # Local use exceptions
 class SpaceInInput(exceptions.Exception): pass
@@ -308,13 +503,24 b' class InteractiveShell(object,Magic):'
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
-        # these modules from deletion by keeping a cache.  To avoid keeping
-        # stale modules around (we only need the one from the last run), we use
-        # a dict keyed with the full path to the script, so only the last
-        # version of the module is held in the cache.  The %reset command will
-        # flush this cache.  See the cache_main_mod() and clear_main_mod_cache()
-        # methods for details on use.
-        self._user_main_modules = {}
+        # these modules from deletion by keeping a cache.
+        # 
+        # To avoid keeping stale modules around (we only need the one from the
+        # last run), we use a dict keyed with the full path to the script, so
+        # only the last version of the module is held in the cache.  Note,
+        # however, that we must cache the module *namespace contents* (their
+        # __dict__).  Because if we try to cache the actual modules, old ones
+        # (uncached) could be destroyed while still holding references (such as
+        # those held by GUI objects that tend to be long-lived)>
+        # 
+        # The %reset command will flush this cache.  See the cache_main_mod()
+        # and clear_main_mod_cache() methods for details on use.
+
+        # This is the cache used for 'main' namespaces
+        self._main_ns_cache = {}
+        # And this is the single instance of FakeModule whose __dict__ we keep
+        # copying and clearing for reuse on each %run
+        self._user_main_module = FakeModule()
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
@@ -330,7 +536,7 b' class InteractiveShell(object,Magic):'
         # a simple list.
         self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
                                self.alias_table, self.internal_ns,
-                               self._user_main_modules ]
+                               self._main_ns_cache ]
         
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
@@ -1114,156 +1320,11 b' class InteractiveShell(object,Magic):'
     def user_setup(self,ipythondir,rc_suffix,mode='install'):
         """Install the user configuration directory.
 
-        Can be called when running for the first time or to upgrade the user's
-        .ipython/ directory with the mode parameter. Valid modes are 'install'
-        and 'upgrade'."""
-
-        def wait():
-            try:
-                raw_input("Please press <RETURN> to start IPython.")
-            except EOFError:
-                print >> Term.cout
-            print '*'*70
-
-        cwd = os.getcwd()  # remember where we started
-        glb = glob.glob
-        print '*'*70
-        if mode == 'install':
-            print \
-"""Welcome to IPython. I will try to create a personal configuration directory
-where you can customize many aspects of IPython's functionality in:\n"""
-        else:
-            print 'I am going to upgrade your configuration in:'
-
-        print ipythondir
-
-        rcdirend = os.path.join('IPython','UserConfig')
-        cfg = lambda d: os.path.join(d,rcdirend)
-        try:
-            rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
-            print "Initializing from configuration",rcdir
-        except IndexError:
-            warning = """
-Installation error. IPython's directory was not found.
-
-Check the following:
-
-The ipython/IPython directory should be in a directory belonging to your
-PYTHONPATH environment variable (that is, it should be in a directory
-belonging to sys.path). You can copy it explicitly there or just link to it.
-
-IPython will create a minimal default configuration for you.
-
-"""
-            warn(warning)
-            wait()
-            
-            if sys.platform =='win32':
-                inif = 'ipythonrc.ini'
-            else:
-                inif = 'ipythonrc'
-            minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults',
-                             inif : '# intentionally left blank' }
-            os.makedirs(ipythondir, mode = 0777)
-            for f, cont in minimal_setup.items():
-                open(ipythondir + '/' + f,'w').write(cont)
-                             
-            return
-
-        if mode == 'install':
-            try:
-                shutil.copytree(rcdir,ipythondir)
-                os.chdir(ipythondir)
-                rc_files = glb("ipythonrc*")
-                for rc_file in rc_files:
-                    os.rename(rc_file,rc_file+rc_suffix)
-            except:
-                warning = """
-
-There was a problem with the installation:
-%s
-Try to correct it or contact the developers if you think it's a bug.
-IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
-                warn(warning)
-                wait()
-                return
-
-        elif mode == 'upgrade':
-            try:
-                os.chdir(ipythondir)
-            except:
-                print """
-Can not upgrade: changing to directory %s failed. Details:
-%s
-""" % (ipythondir,sys.exc_info()[1])
-                wait()
-                return
-            else:
-                sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
-                for new_full_path in sources:
-                    new_filename = os.path.basename(new_full_path)
-                    if new_filename.startswith('ipythonrc'):
-                        new_filename = new_filename + rc_suffix
-                    # The config directory should only contain files, skip any
-                    # directories which may be there (like CVS)
-                    if os.path.isdir(new_full_path):
-                        continue
-                    if os.path.exists(new_filename):
-                        old_file = new_filename+'.old'
-                        if os.path.exists(old_file):
-                            os.remove(old_file)
-                        os.rename(new_filename,old_file)
-                    shutil.copy(new_full_path,new_filename)
-        else:
-            raise ValueError,'unrecognized mode for install:',`mode`
-
-        # Fix line-endings to those native to each platform in the config
-        # directory.
-        try:
-            os.chdir(ipythondir)
-        except:
-            print """
-Problem: changing to directory %s failed.
-Details:
-%s
-
-Some configuration files may have incorrect line endings.  This should not
-cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1])
-            wait()
-        else:
-            for fname in glb('ipythonrc*'):
-                try:
-                    native_line_ends(fname,backup=0)
-                except IOError:
-                    pass
-
-        if mode == 'install':
-            print """
-Successful installation!
-
-Please read the sections 'Initial Configuration' and 'Quick Tips' in the
-IPython manual (there are both HTML and PDF versions supplied with the
-distribution) to make sure that your system environment is properly configured
-to take advantage of IPython's features.
-
-Important note: the configuration system has changed! The old system is
-still in place, but its setting may be partly overridden by the settings in 
-"~/.ipython/ipy_user_conf.py" config file. Please take a look at the file 
-if some of the new settings bother you. 
-
-"""
-        else:
-            print """
-Successful upgrade!
-
-All files in your directory:
-%(ipythondir)s
-which would have been overwritten by the upgrade were backed up with a .old
-extension.  If you had made particular customizations in those files you may
-want to merge them back into the new files.""" % locals()
-        wait()
-        os.chdir(cwd)
-        # end user_setup()
+        Note
+        ----
+        DEPRECATED: use the top-level user_setup() function instead.
+        """
+        return user_setup(ipythondir,rc_suffix,mode)
 
     def atexit_operations(self):
         """This will be executed at the time of exit.
@@ -1417,8 +1478,9 b' want to merge them back into the new files.""" % locals()'
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
-            # remove some chars from the delimiters list
-            delims = readline.get_completer_delims()
+            # Remove some chars from the delimiters list.  If we encounter
+            # unicode chars, discard them.
+            delims = readline.get_completer_delims().encode("ascii", "ignore")
             delims = delims.translate(string._idmap,
                                       self.rc.readline_remove_delims)
             readline.set_completer_delims(delims)
@@ -1441,35 +1503,53 b' want to merge them back into the new files.""" % locals()'
             return True
         return ask_yes_no(prompt,default)
 
-    def cache_main_mod(self,mod):
-        """Cache a main module.
+    def new_main_mod(self,ns=None):
+        """Return a new 'main' module object for user code execution.
+        """
+        main_mod = self._user_main_module
+        init_fakemod_dict(main_mod,ns)
+        return main_mod
+
+    def cache_main_mod(self,ns,fname):
+        """Cache a main module's namespace.
 
-        When scripts are executed via %run, we must keep a reference to their
-        __main__ module (a FakeModule instance) around so that Python doesn't
-        clear it, rendering objects defined therein useless.
+        When scripts are executed via %run, we must keep a reference to the
+        namespace of their __main__ module (a FakeModule instance) around so
+        that Python doesn't clear it, rendering objects defined therein
+        useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the module object (which corresponds to the script
         path).  This way, for multiple executions of the same script we only
-        keep one copy of __main__ (the last one), thus preventing memory leaks
-        from old references while allowing the objects from the last execution
-        to be accessible.
+        keep one copy of the namespace (the last one), thus preventing memory
+        leaks from old references while allowing the objects from the last
+        execution to be accessible.
+
+        Note: we can not allow the actual FakeModule instances to be deleted,
+        because of how Python tears down modules (it hard-sets all their
+        references to None without regard for reference counts).  This method
+        must therefore make a *copy* of the given namespace, to allow the
+        original module's __dict__ to be cleared and reused.
 
+        
         Parameters
         ----------
-          mod : a module object
+          ns : a namespace (a dict, typically)
+
+          fname : str
+            Filename associated with the namespace.
 
         Examples
         --------
 
         In [10]: import IPython
 
-        In [11]: _ip.IP.cache_main_mod(IPython)
+        In [11]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__)
 
-        In [12]: IPython.__file__ in _ip.IP._user_main_modules
+        In [12]: IPython.__file__ in _ip.IP._main_ns_cache
         Out[12]: True
         """
-        self._user_main_modules[os.path.abspath(mod.__file__) ] = mod
+        self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
@@ -1481,17 +1561,17 b' want to merge them back into the new files.""" % locals()'
 
         In [15]: import IPython
 
-        In [16]: _ip.IP.cache_main_mod(IPython)
+        In [16]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__)
 
-        In [17]: len(_ip.IP._user_main_modules) > 0
+        In [17]: len(_ip.IP._main_ns_cache) > 0
         Out[17]: True
 
         In [18]: _ip.IP.clear_main_mod_cache()
 
-        In [19]: len(_ip.IP._user_main_modules) == 0
+        In [19]: len(_ip.IP._main_ns_cache) == 0
         Out[19]: True
         """
-        self._user_main_modules.clear()
+        self._main_ns_cache.clear()
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
@@ -2129,7 +2209,7 b' want to merge them back into the new files.""" % locals()'
 
         try:
             code = self.compile(source,filename,symbol)
-        except (OverflowError, SyntaxError, ValueError, TypeError):
+        except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)
             return None

IPython/ipmaker.py

@@ -55,9 +55,9 b' from IPython.iplib import InteractiveShell'
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.genutils import *
 
-def force_import(modname):
-    if modname in sys.modules:
-        print "reload",modname
+def force_import(modname,force_reload=False):
+    if modname in sys.modules and force_reload:
+        info("reloading: %s" % modname)
         reload(sys.modules[modname])
     else:
         __import__(modname)
@@ -625,25 +625,25 b" object? -> Details about 'object'. ?object also works, ?? prints more."
     except:
         IP.InteractiveTB()
         import_fail_info('ipy_system_conf')
-        
+
     # only import prof module if ipythonrc-PROF was not found
     if opts_all.profile and not profile_handled_by_legacy:
         profmodname = 'ipy_profile_' + opts_all.profile
         try:
-            
             force_import(profmodname)
         except:
             IP.InteractiveTB()
-            print "Error importing",profmodname,"- perhaps you should run %upgrade?"
+            print "Error importing",profmodname,\
+                  "- perhaps you should run %upgrade?"
             import_fail_info(profmodname)
         else:
             opts.profile = opts_all.profile
     else:
         force_import('ipy_profile_none')
+    # XXX - this is wrong: ipy_user_conf should not be loaded unconditionally,
+    # since the user could have specified a config file path by hand.
     try:
-            
         force_import('ipy_user_conf')
-        
     except:
         conf = opts_all.ipythondir + "/ipy_user_conf.py"
         IP.InteractiveTB()

IPython/kernel/__init__.py

@@ -15,10 +15,11 b' if they need blocking clients or in `asyncclient.py` if they want asynchronous,'
 deferred/Twisted using clients.
 """
 __docformat__ = "restructuredtext en"
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-    
  No newline at end of file
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.error import TaskRejectError
  No newline at end of file

IPython/kernel/config/__init__.py

@@ -15,6 +15,7 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+import os, sys
 from os.path import join as pjoin
 
 from IPython.external.configobj import ConfigObj
@@ -23,6 +24,7 b' from IPython.genutils import get_ipython_dir, get_security_dir'
 
 default_kernel_config = ConfigObj()
 
+# This will raise OSError if ipythondir doesn't exist.
 security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------

IPython/kernel/core/interpreter.py

@@ -679,21 +679,22 b' class Interpreter(object):'
         # to exec will fail however.  There seems to be some inconsistency in
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
-
+        
         # The compiler module does not like unicode. We need to convert
         # it encode it:
         if isinstance(python, unicode):
             # Use the utf-8-sig BOM so the compiler detects this a UTF-8
             # encode string.
             python = '\xef\xbb\xbf' + python.encode('utf-8')
-
+        
         # The compiler module will parse the code into an abstract syntax tree.
+        # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
         ast = compiler.parse(python)
-
+        
         # Uncomment to help debug the ast tree
         # for n in ast.node:
         #     print n.lineno,'->',n
-                
+        
         # Each separate command is available by iterating over ast.node. The
         # lineno attribute is the line number (1-indexed) beginning the commands
         # suite.
@@ -703,20 +704,26 b' class Interpreter(object):'
         # We might eventually discover other cases where lineno is None and have
         # to put in a more sophisticated test.
         linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
-
+        
         # When we finally get the slices, we will need to slice all the way to
         # the end even though we don't have a line number for it. Fortunately,
         # None does the job nicely.
         linenos.append(None)
+        
+        # Same problem at the other end: sometimes the ast tree has its
+        # first complete statement not starting on line 0. In this case
+        # we might miss part of it.  This fixes ticket 266993.  Thanks Gael!
+        linenos[0] = 0
+        
         lines = python.splitlines()
-
+        
         # Create a list of atomic commands.
         cmds = []
         for i, j in zip(linenos[:-1], linenos[1:]):
             cmd = lines[i:j]
             if cmd:
                 cmds.append('\n'.join(cmd)+'\n')
-
+        
         return cmds
 
     def error(self, text):

IPython/kernel/core/notification.py

@@ -15,6 +15,8 b' __docformat__ = "restructuredtext en"'
 #  the file COPYING, distributed as part of this software.                    
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
 
 class NotificationCenter(object):
     """Synchronous notification center

IPython/kernel/core/tests/test_interpreter.py

@@ -2,25 +2,61 b''
 
 """This file contains unittests for the interpreter.py module."""
 
-__docformat__ = "restructuredtext en"
-
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                           
-#                                                                             
-#  Distributed under the terms of the BSD License.  The full license is in    
-#  the file COPYING, distributed as part of this software.                    
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
-                                                                              
+
 #-----------------------------------------------------------------------------
-# Imports                                                                     
+# Imports
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
 from IPython.kernel.core.interpreter import Interpreter
 
-def test_unicode():
-    """ Test unicode handling with the interpreter.
-    """
-    i = Interpreter()
-    i.execute_python(u'print "ù"')
-    i.execute_python('print "ù"')
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+class TestInterpreter(unittest.TestCase):
+
+    def test_unicode(self):
+        """ Test unicode handling with the interpreter."""
+        i = Interpreter()
+        i.execute_python(u'print "ù"')
+        i.execute_python('print "ù"')
+
+    def test_ticket266993(self):
+        """ Test for ticket 266993."""
+        i = Interpreter()
+        i.execute('str("""a\nb""")')
+
+    def test_ticket364347(self):
+        """Test for ticket 364347."""
+        i = Interpreter()
+        i.split_commands('str("a\\nb")')
+
+    def test_split_commands(self):
+        """ Test that commands are indeed individually split."""
+        i = Interpreter()
+        test_atoms = [('(1\n + 1)', ),
+                      ('1', '1', ),
+                      ]
+        for atoms in test_atoms:
+            atoms = [atom.rstrip() + '\n' for atom in atoms]
+            self.assertEquals(i.split_commands(''.join(atoms)),atoms)
+
+    def test_long_lines(self):
+        """ Test for spurious syntax error created by the interpreter."""
+        test_strings = [u'( 1 +\n 1\n )\n\n',
+                        u'(1 \n + 1\n )\n\n',
+                       ]
+        i = Interpreter()
+        for s in test_strings:
+            i.execute(s)
 

IPython/kernel/core/tests/test_notification.py

@@ -2,26 +2,26 b''
 
 """This file contains unittests for the notification.py module."""
 
-__docformat__ = "restructuredtext en"
-
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                           
-#                                                                             
-#  Distributed under the terms of the BSD License.  The full license is in    
-#  the file COPYING, distributed as part of this software.                    
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
-                                                                              
+
 #-----------------------------------------------------------------------------
-# Imports                                                                     
+# Imports
 #-----------------------------------------------------------------------------
 
-import unittest
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
 import IPython.kernel.core.notification as notification
-from nose.tools import timed
 
-#
-# Supporting test classes
-#
+#-----------------------------------------------------------------------------
+# Support Classes
+#-----------------------------------------------------------------------------
 
 class Observer(object):
     """docstring for Observer"""
@@ -36,7 +36,6 b' class Observer(object):'
                             self.expectedType, 
                             self.expectedSender)
     
-    
     def callback(self, theType, sender, args={}):
         """callback"""
         
@@ -47,7 +46,6 b' class Observer(object):'
         assert(args == self.expectedKwArgs)
         self.recieved = True
     
-    
     def verify(self):
         """verify"""
         
@@ -57,7 +55,6 b' class Observer(object):'
         """reset"""
         
         self.recieved = False
-    
 
 
 class Notifier(object):
@@ -72,11 +69,10 b' class Notifier(object):'
         
         center.post_notification(self.theType, self,
             **self.kwargs)
-    
 
-#
-# Test Cases
-#
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
 
 class NotificationTests(unittest.TestCase):
     """docstring for NotificationTests"""
@@ -94,7 +90,6 b' class NotificationTests(unittest.TestCase):'
         
         observer.verify()
     
-    
     def test_type_specificity(self):
         """Test that observers are registered by type"""
         
@@ -109,7 +104,6 b' class NotificationTests(unittest.TestCase):'
     
         observer.verify()
     
-    
     def test_sender_specificity(self):
         """Test that observers are registered by sender"""
         
@@ -123,7 +117,6 b' class NotificationTests(unittest.TestCase):'
         
         observer.verify()
     
-    
     def test_remove_all_observers(self):
         """White-box test for remove_all_observers"""
         
@@ -136,8 +129,7 b' class NotificationTests(unittest.TestCase):'
         notification.sharedCenter.remove_all_observers()
         
         self.assert_(len(notification.sharedCenter.observers) == 0, "observers removed")
-    
-    
+
     def test_any_sender(self):
         """test_any_sender"""
         
@@ -153,9 +145,7 b' class NotificationTests(unittest.TestCase):'
         observer.reset()
         sender2.post()
         observer.verify()
-        
-    
-    @timed(.01)
+
     def test_post_performance(self):
         """Test that post_notification, even with many registered irrelevant
         observers is fast"""
@@ -168,4 +158,4 b' class NotificationTests(unittest.TestCase):'
         notification.sharedCenter.post_notification('EXPECTED_TYPE', self)
         
         o.verify()
-    
+

IPython/kernel/core/tests/test_redirectors.py

@@ -2,69 +2,77 b''
 """
 Test the output capture at the OS level, using file descriptors.
 """
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Tell nose to skip this module
+__test__ = {}
 
-# Stdlib imports
-import os
 from cStringIO import StringIO
+import os
+
+from twisted.trial import unittest
 
-# Our own imports
-from IPython.testing import decorators as dec
+from IPython.testing import decorators_trial as dec
 
 #-----------------------------------------------------------------------------
-# Test functions
+# Tests
+#-----------------------------------------------------------------------------
 
[e-mail]_win32
-def test_redirector():
-    """ Checks that the redirector can be used to do synchronous capture.
-    """
-    from IPython.kernel.core.fd_redirector import FDRedirector
-    r = FDRedirector()
-    out = StringIO()
-    try:
-        r.start()
-        for i in range(10):
-            os.system('echo %ic' % i)
-            print >>out, r.getvalue(),
-            print >>out, i
-    except:
-        r.stop()
-        raise
-    r.stop()
-    result1 = out.getvalue()
-    result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
-    assert result1 == result2
 
+class TestRedirector(unittest.TestCase):
 
[e-mail]_win32
-def test_redirector_output_trap():
-    """ This test check not only that the redirector_output_trap does
+    @dec.skip_win32
+    def test_redirector(self):
+        """Checks that the redirector can be used to do synchronous capture.
+        """
+        from IPython.kernel.core.fd_redirector import FDRedirector
+        r = FDRedirector()
+        out = StringIO()
+        try:
+            r.start()
+            for i in range(10):
+                os.system('echo %ic' % i)
+                print >>out, r.getvalue(),
+                print >>out, i
+        except:
+            r.stop()
+            raise
+        r.stop()
+        result1 = out.getvalue()
+        result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
+        self.assertEquals(result1, result2)
+
+    @dec.skip_win32
+    def test_redirector_output_trap(self):
+        """Check the greedy trapping behavior of the traps.
+        
+        This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
-    """
-    from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
-    out = StringIO()
-    trap = RedirectorOutputTrap(out.write, out.write)
-    try:
-        trap.set()
-        for i in range(10):
-            os.system('echo %ic' % i)
-            print "%ip" % i
-            print >>out, i
-    except:
+        """
+        from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
+        out = StringIO()
+        trap = RedirectorOutputTrap(out.write, out.write)
+        try:
+            trap.set()
+            for i in range(10):
+                os.system('echo %ic' % i)
+                print "%ip" % i
+                print >>out, i
+        except:
+            trap.unset()
+            raise
         trap.unset()
-        raise
-    trap.unset()
-    result1 = out.getvalue()
-    result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
-    assert result1 == result2
-    
+        result1 = out.getvalue()
+        result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
+        self.assertEquals(result1, result2)
+

IPython/kernel/core/ultraTB.py

@@ -268,6 +268,8 b' def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):'
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
+            # Again, reference to a global __IPYTHON__ that doesn't exist.
+            # XXX
             scheme = __IPYTHON__.rc.colors
         except:
             scheme = DEFAULT_SCHEME
@@ -487,10 +489,14 b' class ListTB(TBTools):'
             else:
                 list.append('%s\n' % str(stype))
 
-        # vds:>>
-        if have_filedata:
-            __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
-        # vds:<<
+        # This is being commented out for now as the __IPYTHON__ variable
+        # referenced here is not resolved and causes massive test failures
+        # and errors. B. Granger, 04/2009. XXX
+        # See https://bugs.launchpad.net/bugs/362137
+        # # vds:>>
+        # if have_filedata:
+        #     __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        # # vds:<<
 
         return list
 
@@ -804,13 +810,17 b' class VerboseTB(TBTools):'
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
-        # vds: >>
-        if records:
-             filepath, lnum = records[-1][1:3]
-             #print "file:", str(file), "linenb", str(lnum) # dbg
-             filepath = os.path.abspath(filepath)
-             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
-        # vds: <<
+        # This is being commented out for now as the __IPYTHON__ variable
+        # referenced here is not resolved and causes massive test failures
+        # and errors. B. Granger, 04/2009. XXX
+        # See https://bugs.launchpad.net/bugs/362137
+        # # vds: >>
+        # if records:
+        #      filepath, lnum = records[-1][1:3]
+        #      #print "file:", str(file), "linenb", str(lnum) # dbg
+        #      filepath = os.path.abspath(filepath)
+        #      __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
+        # # vds: <<
                 
         # return all our info assembled as a single string
         return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )

IPython/kernel/engineconnector.py

@@ -67,10 +67,10 b' class EngineConnector(object):'
             self.furl = find_furl(furl_or_file)
         except ValueError:
             return defer.fail(failure.Failure())
-            # return defer.fail(failure.Failure(ValueError('not a valid furl or furl file: %r' % furl_or_file)))
-        d = self.tub.getReference(self.furl)
-        d.addCallbacks(self._register, self._log_failure)
-        return d
+        else:
+            d = self.tub.getReference(self.furl)
+            d.addCallbacks(self._register, self._log_failure)
+            return d
     
     def _log_failure(self, reason):
         log.err('EngineConnector: engine registration failed:')

IPython/kernel/engineservice.py

@@ -34,6 +34,9 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
+
 import os, sys, copy
 import cPickle as pickle
 from new import instancemethod
@@ -266,8 +269,8 b' class StrictDict(dict):'
             pickle.dumps(key, 2)
             pickle.dumps(value, 2)
             newvalue = copy.deepcopy(value)
-        except:
-            raise error.InvalidProperty(value)
+        except Exception, e:
+            raise error.InvalidProperty("can't be a value: %r" % value)
         dict.__setitem__(self, key, newvalue)
         self.modified = True
     

IPython/kernel/error.py

@@ -104,6 +104,23 b' class StopLocalExecution(KernelError):'
 class SecurityError(KernelError):
     pass
 
+class FileTimeoutError(KernelError):
+    pass
+
+class TaskRejectError(KernelError):
+    """Exception to raise when a task should be rejected by an engine.
+    
+    This exception can be used to allow a task running on an engine to test
+    if the engine (or the user's namespace on the engine) has the needed
+    task dependencies.  If not, the task should raise this exception.  For
+    the task to be retried on another engine, the task should be created
+    with the `retries` argument > 1.
+    
+    The advantage of this approach over our older properties system is that
+    tasks have full access to the user's namespace on the engines and the
+    properties don't have to be managed or tested by the controller.
+    """
+
 class CompositeError(KernelError):
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))

IPython/kernel/multiengineclient.py

@@ -20,6 +20,7 b' import sys'
 import cPickle as pickle
 from types import FunctionType
 import linecache
+import warnings
 
 from twisted.internet import reactor
 from twisted.python import components, log
@@ -389,6 +390,14 b' def strip_whitespace(source):'
 #-------------------------------------------------------------------------------
 
 
+_prop_warn = """\
+
+We are currently refactoring the task dependency system.  This might
+involve the removal of this method and other methods related to engine
+properties.  Please see the docstrings for IPython.kernel.TaskRejectError 
+for more information."""
+
+
 class IFullBlockingMultiEngineClient(Interface):
     pass
 
@@ -730,22 +739,27 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):'
         return self._blockFromThread(self.smultiengine.queue_status, targets=targets, block=block)
     
     def set_properties(self, properties, targets=None, block=None):
+        warnings.warn(_prop_warn)
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.set_properties, properties, targets=targets, block=block)
     
     def get_properties(self, keys=None, targets=None, block=None):
+        warnings.warn(_prop_warn)
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.get_properties, keys, targets=targets, block=block)
     
     def has_properties(self, keys, targets=None, block=None):
+        warnings.warn(_prop_warn)
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.has_properties, keys, targets=targets, block=block)
     
     def del_properties(self, keys, targets=None, block=None):
+        warnings.warn(_prop_warn)
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.del_properties, keys, targets=targets, block=block)
     
     def clear_properties(self, targets=None, block=None):
+        warnings.warn(_prop_warn)
         targets, block = self._findTargetsAndBlock(targets, block)
         return self._blockFromThread(self.smultiengine.clear_properties, targets=targets, block=block)
     

IPython/kernel/scripts/ipcluster.py

@@ -1,4 +1,4 b''
-#!/usr/bin/env python
+ #!/usr/bin/env python
 # encoding: utf-8
 
 """Start an IPython cluster = (controller + engines)."""
@@ -29,29 +29,35 b' from twisted.python import failure, log'
 
 from IPython.external import argparse
 from IPython.external import Itpl
-from IPython.genutils import get_ipython_dir, num_cpus
+from IPython.genutils import (
+    get_ipython_dir, 
+    get_log_dir, 
+    get_security_dir, 
+    num_cpus
+)
 from IPython.kernel.fcutil import have_crypto
-from IPython.kernel.error import SecurityError
+
+# Create various ipython directories if they don't exist.
+# This must be done before IPython.kernel.config is imported.
+from IPython.iplib import user_setup
+if os.name == 'posix':
+    rc_suffix = ''
+else:
+    rc_suffix = '.ini'
+user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
+get_log_dir()
+get_security_dir()
+
+from IPython.kernel.config import config_manager as kernel_config_manager
+from IPython.kernel.error import SecurityError, FileTimeoutError
 from IPython.kernel.fcutil import have_crypto
-from IPython.kernel.twistedutil import gatherBoth
+from IPython.kernel.twistedutil import gatherBoth, wait_for_file
 from IPython.kernel.util import printer
 
-
 #-----------------------------------------------------------------------------
 # General process handling code
 #-----------------------------------------------------------------------------
 
-def find_exe(cmd):
-    try:
-        import win32api
-    except ImportError:
-        raise ImportError('you need to have pywin32 installed for this to work')
-    else:
-        try:
-            (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe')
-        except:
-            (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat')
-    return path
 
 class ProcessStateError(Exception):
     pass
@@ -184,8 +190,10 b' class ControllerLauncher(ProcessLauncher):'
             from IPython.kernel.scripts import ipcontroller
             script_location = ipcontroller.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
-            # on Win32 to prevent wierd conflict and problems with Twisted
-            args = [find_exe('python'), '-u', script_location]
+            # on Win32 to prevent wierd conflict and problems with Twisted.
+            # Also, use sys.executable to make sure we are picking up the 
+            # right python exe.
+            args = [sys.executable, '-u', script_location]
         else:
             args = ['ipcontroller']
         self.extra_args = extra_args
@@ -204,8 +212,10 b' class EngineLauncher(ProcessLauncher):'
             from IPython.kernel.scripts import ipengine
             script_location = ipengine.__file__.replace('.pyc', '.py')
             # The -u option here turns on unbuffered output, which is required
-            # on Win32 to prevent wierd conflict and problems with Twisted
-            args = [find_exe('python'), '-u', script_location]
+            # on Win32 to prevent wierd conflict and problems with Twisted.
+            # Also, use sys.executable to make sure we are picking up the 
+            # right python exe.
+            args = [sys.executable, '-u', script_location]
         else:
             args = ['ipengine']
         self.extra_args = extra_args
@@ -465,7 +475,9 b' class SSHEngineSet(object):'
 # The main functions should then just parse the command line arguments, create
 # the appropriate class and call a 'start' method.
 
+
 def check_security(args, cont_args):
+    """Check to see if we should run with SSL support."""
     if (not args.x or not args.y) and not have_crypto:
         log.err("""
 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
@@ -478,7 +490,9 b' Try running ipcluster with the -xy flags: ipcluster local -xy -n 4""")'
         cont_args.append('-y')
     return True
 
+
 def check_reuse(args, cont_args):
+    """Check to see if we should try to resuse FURL files."""
     if args.r:
         cont_args.append('-r')
         if args.client_port == 0 or args.engine_port == 0:
@@ -491,6 +505,25 b' the --client-port and --engine-port options.""")'
         cont_args.append('--engine-port=%i' % args.engine_port)
     return True
 
+
+def _err_and_stop(f):
+    """Errback to log a failure and halt the reactor on a fatal error."""
+    log.err(f)
+    reactor.stop()
+
+
+def _delay_start(cont_pid, start_engines, furl_file, reuse):
+    """Wait for controller to create FURL files and the start the engines."""
+    if not reuse:
+        if os.path.isfile(furl_file):
+            os.unlink(furl_file)
+    log.msg('Waiting for controller to finish starting...')
+    d = wait_for_file(furl_file, delay=0.2, max_tries=50)
+    d.addCallback(lambda _: log.msg('Controller started'))
+    d.addCallback(lambda _: start_engines(cont_pid))
+    return d
+
+
 def main_local(args):
     cont_args = []
     cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
@@ -520,13 +553,10 b' def main_local(args):'
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start(args.n)
         return d
-    def delay_start(cont_pid):
-        # This is needed because the controller doesn't start listening
-        # right when it starts and the controller needs to write
-        # furl files for the engine to pick up
-        reactor.callLater(1.0, start_engines, cont_pid)
-    dstart.addCallback(delay_start)
-    dstart.addErrback(lambda f: f.raiseException())
+    config = kernel_config_manager.get_config_obj()
+    furl_file = config['controller']['engine_furl_file']
+    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
+    dstart.addErrback(_err_and_stop)
 
 
 def main_mpi(args):
@@ -562,13 +592,10 b' def main_mpi(args):'
         signal.signal(signal.SIGINT,shutdown)
         d = eset.start()
         return d
-    def delay_start(cont_pid):
-        # This is needed because the controller doesn't start listening
-        # right when it starts and the controller needs to write
-        # furl files for the engine to pick up
-        reactor.callLater(1.0, start_engines, cont_pid)
-    dstart.addCallback(delay_start)
-    dstart.addErrback(lambda f: f.raiseException())
+    config = kernel_config_manager.get_config_obj()
+    furl_file = config['controller']['engine_furl_file']
+    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
+    dstart.addErrback(_err_and_stop)
 
 
 def main_pbs(args):
@@ -595,8 +622,10 b' def main_pbs(args):'
         signal.signal(signal.SIGINT,shutdown)
         d = pbs_set.start(args.n)
         return d
-    dstart.addCallback(start_engines)
-    dstart.addErrback(lambda f: f.raiseException())
+    config = kernel_config_manager.get_config_obj()
+    furl_file = config['controller']['engine_furl_file']
+    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
+    dstart.addErrback(_err_and_stop)
 
 
 def main_ssh(args):
@@ -637,12 +666,10 b' def main_ssh(args):'
         signal.signal(signal.SIGINT,shutdown)
         d = ssh_set.start(clusterfile['send_furl'])
         return d
-    
-    def delay_start(cont_pid):
-        reactor.callLater(1.0, start_engines, cont_pid)
-        
-    dstart.addCallback(delay_start)
-    dstart.addErrback(lambda f: f.raiseException())
+    config = kernel_config_manager.get_config_obj()
+    furl_file = config['controller']['engine_furl_file']
+    dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
+    dstart.addErrback(_err_and_stop)
 
 
 def get_args():
@@ -697,8 +724,11 b' def get_args():'
     
     parser = argparse.ArgumentParser(
         description='IPython cluster startup.  This starts a controller and\
-        engines using various approaches.  THIS IS A TECHNOLOGY PREVIEW AND\
-        THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.'
+        engines using various approaches.  Use the IPYTHONDIR environment\
+        variable to change your IPython directory from the default of\
+        .ipython or _ipython.  The log and security subdirectories of your\
+        IPython directory will be used by this script for log files and\
+        security files.'
     )
     subparsers = parser.add_subparsers(
         help='available cluster types.  For help, do "ipcluster TYPE --help"')

IPython/kernel/scripts/ipcontroller.py

@@ -21,8 +21,10 b' __docformat__ = "restructuredtext en"'
 import sys
 sys.path.insert(0, '')
 
-import sys, time, os
 from optparse import OptionParser
+import os
+import time
+import tempfile
 
 from twisted.application import internet, service
 from twisted.internet import reactor, error, defer
@@ -37,6 +39,18 b' from IPython.kernel.error import SecurityError'
 from IPython.kernel import controllerservice
 from IPython.kernel.fcutil import check_furl_file_security
 
+# Create various ipython directories if they don't exist.
+# This must be done before IPython.kernel.config is imported.
+from IPython.iplib import user_setup
+from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
+if os.name == 'posix':
+    rc_suffix = ''
+else:
+    rc_suffix = '.ini'
+user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
+get_log_dir()
+get_security_dir()
+
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.config.cutils import import_item
 
@@ -45,6 +59,10 b' from IPython.config.cutils import import_item'
 # Code
 #-------------------------------------------------------------------------------
 
+def get_temp_furlfile(filename):
+    return tempfile.mktemp(dir=os.path.dirname(filename),
+                           prefix=os.path.basename(filename))
+
 def make_tub(ip, port, secure, cert_file):
     """
     Create a listening tub given an ip, port, and cert_file location.
@@ -107,13 +125,18 b' def make_client_service(controller_service, config):'
         """Set the location for the tub and return a deferred."""
 
         def register(empty, ref, furl_file):
-            client_tub.registerReference(ref, furlFile=furl_file)
+            # We create and then move to make sure that when the file
+            # appears to other processes, the buffer has the flushed
+            # and the file has been closed
+            temp_furl_file = get_temp_furlfile(furl_file)
+            client_tub.registerReference(ref, furlFile=temp_furl_file)
+            os.rename(temp_furl_file, furl_file)
         
         if location == '':
             d = client_tub.setLocationAutomatically()
         else:
             d = defer.maybeDeferred(client_tub.setLocation, "%s:%i" % (location, client_listener.getPortnum()))
-            
+        
         for ciname, ci in config['controller']['controller_interfaces'].iteritems():
             log.msg("Adapting Controller to interface: %s" % ciname)
             furl_file = ci['furl_file']
@@ -154,7 +177,12 b' def make_engine_service(controller_service, config):'
         """Set the location for the tub and return a deferred."""
 
         def register(empty, ref, furl_file):
-            engine_tub.registerReference(ref, furlFile=furl_file)
+            # We create and then move to make sure that when the file
+            # appears to other processes, the buffer has the flushed
+            # and the file has been closed
+            temp_furl_file = get_temp_furlfile(furl_file)
+            engine_tub.registerReference(ref, furlFile=temp_furl_file)
+            os.rename(temp_furl_file, furl_file)
         
         if location == '':
             d = engine_tub.setLocationAutomatically()
@@ -236,7 +264,14 b' def init_config():'
     Initialize the configuration using default and command line options.
     """
     
-    parser = OptionParser()
+    parser = OptionParser("""ipcontroller [options]
+
+Start an IPython controller.
+
+Use the IPYTHONDIR environment variable to change your IPython directory 
+from the default of .ipython or _ipython.  The log and security 
+subdirectories of your IPython directory will be used by this script 
+for log files and security files.""")
     
     # Client related options
     parser.add_option(
@@ -325,12 +360,6 b' def init_config():'
         help="log file name (default is stdout)"
     )
     parser.add_option(
-        "--ipythondir",
-        type="string",
-        dest="ipythondir",
-        help="look for config files and profiles in this directory"
-    )
-    parser.add_option(
         "-r",
         action="store_true",
         dest="reuse_furls",
@@ -339,7 +368,6 b' def init_config():'
     
     (options, args) = parser.parse_args()
     
-    kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
     config = kernel_config_manager.get_config_obj()
     
     # Update with command line options

IPython/kernel/scripts/ipengine.py

@@ -21,8 +21,8 b' __docformat__ = "restructuredtext en"'
 import sys
 sys.path.insert(0, '')
 
-import sys, os
 from optparse import OptionParser
+import os
 
 from twisted.application import service
 from twisted.internet import reactor
@@ -33,6 +33,19 b' from IPython.kernel.fcutil import Tub, UnauthenticatedTub'
 from IPython.kernel.core.config import config_manager as core_config_manager
 from IPython.config.cutils import import_item
 from IPython.kernel.engineservice import EngineService
+
+# Create various ipython directories if they don't exist.
+# This must be done before IPython.kernel.config is imported.
+from IPython.iplib import user_setup
+from IPython.genutils import get_ipython_dir, get_log_dir, get_security_dir
+if os.name == 'posix':
+    rc_suffix = ''
+else:
+    rc_suffix = '.ini'
+user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
+get_log_dir()
+get_security_dir()
+
 from IPython.kernel.config import config_manager as kernel_config_manager
 from IPython.kernel.engineconnector import EngineConnector
 
@@ -106,13 +119,19 b' def start_engine():'
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
     log.msg("Using furl file: %s" % furl_file)
-    d = engine_connector.connect_to_controller(engine_service, furl_file)
-    def handle_error(f):
-        log.err(f)
-        if reactor.running:
-            reactor.stop()
-    d.addErrback(handle_error)
     
+    def call_connect(engine_service, furl_file):
+        d = engine_connector.connect_to_controller(engine_service, furl_file)
+        def handle_error(f):
+            # If this print statement is replaced by a log.err(f) I get
+            # an unhandled error, which makes no sense.  I shouldn't have
+            # to use a print statement here.  My only thought is that
+            # at the beginning of the process the logging is still starting up
+            print "error connecting to controller:", f.getErrorMessage()
+            reactor.callLater(0.1, reactor.stop)
+        d.addErrback(handle_error)
+    
+    reactor.callWhenRunning(call_connect, engine_service, furl_file)
     reactor.run()
 
 
@@ -121,7 +140,14 b' def init_config():'
     Initialize the configuration using default and command line options.
     """
     
-    parser = OptionParser()
+    parser = OptionParser("""ipengine [options]
+
+Start an IPython engine.
+
+Use the IPYTHONDIR environment variable to change your IPython directory 
+from the default of .ipython or _ipython.  The log and security 
+subdirectories of your IPython directory will be used by this script 
+for log files and security files.""")
     
     parser.add_option(
         "--furl-file",
@@ -142,18 +168,9 b' def init_config():'
         dest="logfile",
         help="log file name (default is stdout)"
     )
-    parser.add_option(
-        "--ipythondir",
-        type="string",
-        dest="ipythondir",
-        help="look for config files and profiles in this directory"
-    )
     
     (options, args) = parser.parse_args()
     
-    kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
-    core_config_manager.update_config_obj_from_default_file(options.ipythondir)
-    
     kernel_config = kernel_config_manager.get_config_obj()
     # Now override with command line options
     if options.furl_file is not None:

IPython/kernel/task.py

@@ -16,6 +16,9 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
+
 import copy, time
 from types import FunctionType
 

IPython/kernel/taskclient.py

@@ -49,7 +49,7 b' class BlockingTaskClient(object):'
     """
     
     implements(
-        IBlockingTaskClient, 
+        IBlockingTaskClient,
         ITaskMapperFactory,
         IMapper,
         ITaskParallelDecorator
@@ -62,7 +62,7 b' class BlockingTaskClient(object):'
     def run(self, task, block=False):
         """Run a task on the `TaskController`.
         
-        See the documentation of the `MapTask` and `StringTask` classes for 
+        See the documentation of the `MapTask` and `StringTask` classes for
         details on how to build a task of different types.
         
         :Parameters:

IPython/kernel/tests/engineservicetest.py

@@ -363,7 +363,8 b' class IEnginePropertiesTestCase(object):'
 p = get_engine(%s).properties"""%self.engine.id
         d = self.engine.execute(s)
         d.addCallback(lambda r: self.engine.execute("p['a'] = lambda _:None"))
-        d = self.assertDeferredRaises(d, error.InvalidProperty)
+        d.addErrback(lambda f: self.assertRaises(error.InvalidProperty,
+            f.raiseException))
         d.addCallback(lambda r: self.engine.execute("p['a'] = range(5)"))
         d.addCallback(lambda r: self.engine.execute("p['a'].append(5)"))
         d.addCallback(lambda r: self.engine.get_properties('a'))

IPython/kernel/tests/test_contexts.py

@@ -1,3 +1,6 b''
+# Tell nose to skip this module
+__test__ = {}
+
 #from __future__ import with_statement
 
 # XXX This file is currently disabled to preserve 2.4 compatibility.

IPython/kernel/tests/test_controllerservice.py

@@ -23,15 +23,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.application.service import IService
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel.tests import multienginetest as met
-    from controllertest import IControllerCoreTestCase
-    from IPython.testing.util import DeferredTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.application.service import IService
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel.tests import multienginetest as met
+from controllertest import IControllerCoreTestCase
+from IPython.testing.util import DeferredTestCase
+
 
 class BasicControllerServiceTest(DeferredTestCase,
     IControllerCoreTestCase):

IPython/kernel/tests/test_enginefc.py

@@ -15,30 +15,29 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.python import components
-    from twisted.internet import reactor, defer
-    from twisted.spread import pb
-    from twisted.internet.base import DelayedCall
-    DelayedCall.debug = True
+# Tell nose to skip this module
+__test__ = {}
 
-    import zope.interface as zi
+from twisted.python import components
+from twisted.internet import reactor, defer
+from twisted.spread import pb
+from twisted.internet.base import DelayedCall
+DelayedCall.debug = True
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
-    from IPython.kernel import engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import IControllerBase
-    from IPython.kernel.enginefc import FCRemoteEngineRefFromService, IEngineBase
-    from IPython.kernel.engineservice import IEngineQueued
-    from IPython.kernel.engineconnector import EngineConnector
-    
-    from IPython.kernel.tests.engineservicetest import \
-        IEngineCoreTestCase, \
-        IEngineSerializedTestCase, \
-        IEngineQueuedTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+import zope.interface as zi
+
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from IPython.kernel import engineservice as es
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import IControllerBase
+from IPython.kernel.enginefc import FCRemoteEngineRefFromService, IEngineBase
+from IPython.kernel.engineservice import IEngineQueued
+from IPython.kernel.engineconnector import EngineConnector
+
+from IPython.kernel.tests.engineservicetest import \
+    IEngineCoreTestCase, \
+    IEngineSerializedTestCase, \
+    IEngineQueuedTestCase
 
 
 class EngineFCTest(DeferredTestCase, 

IPython/kernel/tests/test_engineservice.py

@@ -23,20 +23,19 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer
-    from twisted.application.service import IService
-    
-    from IPython.kernel import engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.tests.engineservicetest import \
-        IEngineCoreTestCase, \
-        IEngineSerializedTestCase, \
-        IEngineQueuedTestCase, \
-        IEnginePropertiesTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")    
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from twisted.application.service import IService
+
+from IPython.kernel import engineservice as es
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.tests.engineservicetest import \
+    IEngineCoreTestCase, \
+    IEngineSerializedTestCase, \
+    IEngineQueuedTestCase, \
+    IEnginePropertiesTestCase
 
 
 class BasicEngineServiceTest(DeferredTestCase,

IPython/kernel/tests/test_multiengine.py

@@ -4,29 +4,28 b''
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
-
-try:
-    from twisted.internet import defer
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel import multiengine as me
-    from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
-        ISynchronousMultiEngineTestCase)
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-
- 
+#-----------------------------------------------------------------------------
+
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel import multiengine as me
+from IPython.kernel.tests.multienginetest import (IMultiEngineTestCase,
+    ISynchronousMultiEngineTestCase)
+
+
 class BasicMultiEngineTestCase(DeferredTestCase, IMultiEngineTestCase):
 
     def setUp(self):

IPython/kernel/tests/test_multienginefc.py

@@ -14,25 +14,25 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer, reactor
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer, reactor
+
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.controllerservice import ControllerService
+from IPython.kernel.multiengine import IMultiEngine
+from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase
+from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
+from IPython.kernel import multiengine as me
+from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.parallelfunction import ParallelFunction
+from IPython.kernel.error import CompositeError
+from IPython.kernel.util import printer
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
 
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.controllerservice import ControllerService
-    from IPython.kernel.multiengine import IMultiEngine
-    from IPython.kernel.tests.multienginetest import IFullSynchronousMultiEngineTestCase
-    from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
-    from IPython.kernel import multiengine as me
-    from IPython.kernel.clientconnector import ClientConnector
-    from IPython.kernel.parallelfunction import ParallelFunction
-    from IPython.kernel.error import CompositeError
-    from IPython.kernel.util import printer
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-    
 def _raise_it(f):
     try:
         f.raiseException()

IPython/kernel/tests/test_newserialized.py

@@ -4,36 +4,36 b''
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 # Imports
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
-try:
-    import zope.interface as zi
-    from twisted.trial import unittest
-    from IPython.testing.util import DeferredTestCase
+# Tell nose to skip this module
+__test__ = {}
 
-    from IPython.kernel.newserialized import \
-        ISerialized, \
-        IUnSerialized, \
-        Serialized, \
-        UnSerialized, \
-        SerializeIt, \
-        UnSerializeIt
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+import zope.interface as zi
+from twisted.trial import unittest
+from IPython.testing.util import DeferredTestCase
 
-#-------------------------------------------------------------------------------
+from IPython.kernel.newserialized import \
+    ISerialized, \
+    IUnSerialized, \
+    Serialized, \
+    UnSerialized, \
+    SerializeIt, \
+    UnSerializeIt
+
+
+#-----------------------------------------------------------------------------
 # Tests
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 class SerializedTestCase(unittest.TestCase):
 

IPython/kernel/tests/test_pendingdeferred.py

@@ -16,18 +16,18 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    from twisted.internet import defer
-    from twisted.python import failure
-
-    from IPython.testing.util import DeferredTestCase
-    import IPython.kernel.pendingdeferred as pd
-    from IPython.kernel import error
-    from IPython.kernel.util import printer
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
-    
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.internet import defer
+from twisted.python import failure
+
+from IPython.testing.util import DeferredTestCase
+import IPython.kernel.pendingdeferred as pd
+from IPython.kernel import error
+from IPython.kernel.util import printer
+
+
 class Foo(object):
 
     def bar(self, bahz):

IPython/kernel/tests/test_task.py

@@ -15,19 +15,19 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    import time
-
-    from twisted.internet import defer
-    from twisted.trial import unittest
-
-    from IPython.kernel import task, controllerservice as cs, engineservice as es
-    from IPython.kernel.multiengine import IMultiEngine
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.tests.tasktest import ITaskControllerTestCase
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+# Tell nose to skip this module
+__test__ = {}
+
+import time
+
+from twisted.internet import defer
+from twisted.trial import unittest
+
+from IPython.kernel import task, controllerservice as cs, engineservice as es
+from IPython.kernel.multiengine import IMultiEngine
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.tests.tasktest import ITaskControllerTestCase
+
 
 #-------------------------------------------------------------------------------
 # Tests

IPython/kernel/tests/test_taskfc.py

@@ -14,27 +14,26 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
-try:
-    import time
+# Tell nose to skip this module
+__test__ = {}
 
-    from twisted.internet import defer, reactor
+import time
 
-    from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+from twisted.internet import defer, reactor
 
-    from IPython.kernel import task as taskmodule
-    from IPython.kernel import controllerservice as cs
-    import IPython.kernel.multiengine as me
-    from IPython.testing.util import DeferredTestCase
-    from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
-    from IPython.kernel.taskfc import IFCTaskController
-    from IPython.kernel.util import printer
-    from IPython.kernel.tests.tasktest import ITaskControllerTestCase
-    from IPython.kernel.clientconnector import ClientConnector
-    from IPython.kernel.error import CompositeError
-    from IPython.kernel.parallelfunction import ParallelFunction
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+from IPython.kernel.fcutil import Tub, UnauthenticatedTub
+
+from IPython.kernel import task as taskmodule
+from IPython.kernel import controllerservice as cs
+import IPython.kernel.multiengine as me
+from IPython.testing.util import DeferredTestCase
+from IPython.kernel.multienginefc import IFCSynchronousMultiEngine
+from IPython.kernel.taskfc import IFCTaskController
+from IPython.kernel.util import printer
+from IPython.kernel.tests.tasktest import ITaskControllerTestCase
+from IPython.kernel.clientconnector import ClientConnector
+from IPython.kernel.error import CompositeError
+from IPython.kernel.parallelfunction import ParallelFunction
 
 
 #-------------------------------------------------------------------------------

IPython/kernel/twistedutil.py

@@ -16,12 +16,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+import os, sys
 import threading, Queue, atexit
-import twisted
 
+import twisted
 from twisted.internet import defer, reactor
 from twisted.python import log, failure
 
+from IPython.kernel.error import FileTimeoutError
+
 #-------------------------------------------------------------------------------
 # Classes related to twisted and threads
 #-------------------------------------------------------------------------------
@@ -204,3 +207,43 b' class DeferredList(defer.Deferred):'
             result = None
 
         return result
+
+
+def wait_for_file(filename, delay=0.1, max_tries=10):
+    """Wait (poll) for a file to be created.
+    
+    This method returns a Deferred that will fire when a file exists. It
+    works by polling os.path.isfile in time intervals specified by the
+    delay argument.  If `max_tries` is reached, it will errback with a 
+    `FileTimeoutError`.
+    
+    Parameters
+    ----------
+    filename : str
+        The name of the file to wait for.
+    delay : float
+        The time to wait between polls.
+    max_tries : int
+        The max number of attempts before raising `FileTimeoutError`
+    
+    Returns
+    -------
+    d : Deferred
+        A Deferred instance that will fire when the file exists.
+    """
+    
+    d = defer.Deferred()
+    
+    def _test_for_file(filename, attempt=0):
+        if attempt >= max_tries:
+            d.errback(FileTimeoutError(
+                'timeout waiting for file to be created: %s' % filename
+            ))
+        else:
+            if os.path.isfile(filename):
+                d.callback(True)
+            else:
+                reactor.callLater(delay, _test_for_file, filename, attempt+1)
+    
+    _test_for_file(filename)
+    return d

IPython/platutils.py

@@ -14,6 +14,7 b' for your operation system, from platutils_PLATFORMNAME module.'
 
 import os
 import sys
+import warnings
 
 # Import the platform-specific implementations
 if os.name == 'posix':
@@ -61,6 +62,41 b' def set_term_title(title):'
     _platutils.set_term_title(title)
 
 
+class FindCmdError(Exception):
+    pass
+
+def find_cmd(cmd):
+    """Find full path to executable cmd in a cross platform manner.
+    
+    This function tries to determine the full path to a command line program
+    using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
+    time it will use the version that is first on the users `PATH`.  If
+    cmd is `python` return `sys.executable`.
+
+    Parameters
+    ----------
+    cmd : str
+        The command line program to look for.
+    """
+    if cmd == 'python':
+        return sys.executable
+    try:
+        path = _platutils.find_cmd(cmd)
+    except:
+        raise FindCmdError('command could not be found: %s' % cmd)
+    # which returns empty if not found
+    if path == '':
+        raise FindCmdError('command could not be found: %s' % cmd)
+    return path
+
+def get_long_path_name(path):
+    """Expand a path into its long form.
+
+    On Windows this expands any ~ in the paths. On other platforms, it is
+    a null operation.
+    """
+    return _platutils.get_long_path_name(path)
+
 #-----------------------------------------------------------------------------
 # Deprecated functions
 #-----------------------------------------------------------------------------

IPython/platutils_dummy.py

@@ -23,3 +23,11 b' ignore_termtitle = True'
 def set_term_title(*args,**kw):
     """Dummy no-op."""
     pass
+
+def find_cmd(cmd):
+    """Find the full path to a command using which."""
+    return os.popen('which %s' % cmd).read().strip()
+
+def get_long_path_name(path):
+    """Dummy no-op."""
+    return path

IPython/platutils_posix.py

@@ -30,3 +30,11 b" if os.environ.get('TERM','') == 'xterm':"
     set_term_title = _set_term_title_xterm
 else:
     set_term_title = _dummy_op
+
+def find_cmd(cmd):
+    """Find the full path to a command using which."""
+    return os.popen('which %s' % cmd).read().strip()
+
+def get_long_path_name(path):
+    """Dummy no-op."""
+    return path

IPython/platutils_win32.py

@@ -41,3 +41,42 b' except ImportError:'
         if ret:
             # non-zero return code signals error, don't try again
             ignore_termtitle = True
+
+def find_cmd(cmd):
+    """Find the full path to a .bat or .exe using the win32api module."""
+    try:
+        import win32api
+    except ImportError:
+        raise ImportError('you need to have pywin32 installed for this to work')
+    else:
+        try:
+            (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe')
+        except:
+            (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat')
+    return path
+
+
+def get_long_path_name(path):
+    """Get a long path name (expand ~) on Windows using ctypes.
+
+    Examples
+    --------
+
+    >>> get_long_path_name('c:\\docume~1')
+    u'c:\\\\Documents and Settings'
+
+    """
+    try:
+        import ctypes
+    except ImportError:
+        raise ImportError('you need to have ctypes installed for this to work')
+    _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
+    _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
+        ctypes.c_uint ]
+
+    buf = ctypes.create_unicode_buffer(260)
+    rv = _GetLongPathName(path, buf, 260)
+    if rv == 0 or rv > 260:
+        return path
+    else:
+        return buf.value

IPython/testing/decorators.py

@@ -123,10 +123,10 b' def skipif(skip_condition, msg=None):'
     Parameters
     ----------
     skip_condition : bool or callable. 
- 	Flag to determine whether to skip test.  If the condition is a 
- 	callable, it is used at runtime to dynamically make the decision.  This 
- 	is useful for tests that may require costly imports, to delay the cost 
- 	until the test suite is actually executed.        
+    Flag to determine whether to skip test.  If the condition is a 
+    callable, it is used at runtime to dynamically make the decision.  This 
+    is useful for tests that may require costly imports, to delay the cost 
+    until the test suite is actually executed.        
     msg : string
         Message to give on raising a SkipTest exception
 

IPython/testing/decorators_numpy.py

@@ -29,13 +29,6 b' def setastest(tf=True):'
     tf : bool
         If True specifies this is a test, not a test otherwise
 
-    e.g
-    >>> from numpy.testing.decorators import setastest
-    >>> @setastest(False)
-    ... def func_with_test_in_name(arg1, arg2): pass
-    ...
-    >>>
-
     This decorator cannot use the nose namespace, because it can be
     called from a non-test module. See also istest and nottest in
     nose.tools

IPython/testing/iptest.py

@@ -1,54 +1,121 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
-This module provides a main entry point to a user script to test IPython itself
-from the command line.  The main() routine can be used in a similar manner to
-the ``nosetests`` script, and it takes similar arguments, but if no arguments
-are given it defaults to testing all of IPython.  This should be preferred to
-using plain ``nosetests`` because a number of nose plugins necessary to test
-IPython correctly are automatically configured by this code.
+This module provides a main entry point to a user script to test IPython
+itself from the command line. There are two ways of running this script:
+
+1. With the syntax `iptest all`.  This runs our entire test suite by
+   calling this script (with different arguments) or trial recursively.  This
+   causes modules and package to be tested in different processes, using nose
+   or trial where appropriate.
+2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
+   the script simply calls nose, but with special command line flags and
+   plugins loaded.
+
+For now, this script requires that both nose and twisted are installed.  This
+will change in the future.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
-# stdlib
+import os
+import os.path as path
 import sys
+import subprocess
+import time
 import warnings
 
-# third-party
 import nose.plugins.builtin
 from nose.core import TestProgram
 
-# Our own imports
+from IPython.platutils import find_cmd
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
+pjoin = path.join
+
 #-----------------------------------------------------------------------------
-# Constants and globals
+# Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
+def test_for(mod):
+    """Test to see if mod is importable."""
+    try:
+        __import__(mod)
+    except ImportError:
+        return False
+    else:
+        return True
+
+have_curses = test_for('_curses')
+have_wx = test_for('wx')
+have_zi = test_for('zope.interface')
+have_twisted = test_for('twisted')
+have_foolscap = test_for('foolscap')
+have_objc = test_for('objc')
+have_pexpect = test_for('pexpect')
+
 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
 # testing problems.  We should strive to minimize the number of skipped
 # modules, since this means untested code.  As the testing machinery
 # solidifies, this list should eventually become empty.
-EXCLUDE = ['IPython/external/',
-           'IPython/platutils_win32',
-           'IPython/frontend/cocoa',
-           'IPython_doctest_plugin',
-           'IPython/Gnuplot',
-           'IPython/Extensions/ipy_',
-           'IPython/Extensions/clearcmd',
-           'IPython/Extensions/PhysicalQIn',
-           'IPython/Extensions/scitedirector',
+EXCLUDE = [pjoin('IPython', 'external'),
+           pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
+           pjoin('IPython_doctest_plugin'),
+           pjoin('IPython', 'Gnuplot'),
+           pjoin('IPython', 'Extensions', 'ipy_'),
+           pjoin('IPython', 'Extensions', 'clearcmd'),
+           pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
+           pjoin('IPython', 'Extensions', 'scitedirector'),
+           pjoin('IPython', 'Extensions', 'numeric_formats'),
+           pjoin('IPython', 'testing', 'attic'),
+           pjoin('IPython', 'testing', 'tutils'),
+           pjoin('IPython', 'testing', 'tools'),
+           pjoin('IPython', 'testing', 'mkdoctests')
            ]
 
+if not have_wx:
+    EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
+    EXCLUDE.append(pjoin('IPython', 'gui'))
+    EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
+
+if not have_objc:
+    EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
+
+if not have_curses:
+    EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
+
+if not sys.platform == 'win32':
+    EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
+
+# These have to be skipped on win32 because the use echo, rm, cd, etc.
+# See ticket https://bugs.launchpad.net/bugs/366982
+if sys.platform == 'win32':
+    EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
+    EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
+
+if not os.name == 'posix':
+    EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
+
+if not have_pexpect:
+    EXCLUDE.append(pjoin('IPython', 'irunner'))
+
+# This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
+if sys.platform == 'win32':
+    EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+
+
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
-def main():
-    """Run the IPython test suite.
+def run_iptest():
+    """Run the IPython test suite using nose.
+    
+    This function is called when this script is **not** called with the form
+    `iptest all`.  It simply calls nose with appropriate command line flags
+    and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
@@ -60,8 +127,8 b' def main():'
                         # test suite back into working shape.  Our nose
                         # plugin needs to be gone through with a fine
                         # toothed comb to find what is causing the problem.
-                        # '--with-ipdoctest',
-                        '--doctest-tests','--doctest-extension=txt',
+                        '--with-ipdoctest',
+                        '--ipdoctest-tests','--ipdoctest-extension=txt',
                         '--detailed-errors',
                        
                         # We add --exe because of setuptools' imbecility (it
@@ -81,11 +148,13 b' def main():'
            (':' in arg and  '.py' in arg):
             has_tests = True
             break
+        
     # If nothing was specifically requested, test full IPython
     if not has_tests:
         argv.append('IPython')
 
-    # Construct list of plugins, omitting the existing doctest plugin.
+    # Construct list of plugins, omitting the existing doctest plugin, which
+    # ours replaces (and extends).
     plugins = [IPythonDoctest(EXCLUDE)]
     for p in nose.plugins.builtin.plugins:
         plug = p()
@@ -96,3 +165,136 b' def main():'
         plugins.append(plug)
 
     TestProgram(argv=argv,plugins=plugins)
+
+
+class IPTester(object):
+    """Call that calls iptest or trial in a subprocess.
+    """
+    def __init__(self,runner='iptest',params=None):
+        """ """
+        if runner == 'iptest':
+            self.runner = ['iptest','-v']
+        else:
+            self.runner = [find_cmd('trial')]
+        if params is None:
+            params = []
+        if isinstance(params,str):
+            params = [params]
+        self.params = params
+
+        # Assemble call
+        self.call_args = self.runner+self.params
+
+    def run(self):
+        """Run the stored commands"""
+        return subprocess.call(self.call_args)
+
+
+def make_runners():
+    """Define the modules and packages that need to be tested.
+    """
+    
+    # This omits additional top-level modules that should not be doctested.
+    # XXX: Shell.py is also ommited because of a bug in the skip_doctest
+    # decorator.  See ticket https://bugs.launchpad.net/bugs/366209
+    top_mod = \
+      ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
+       'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
+       'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
+       'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
+       'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
+       'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
+       'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
+       'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
+       'shellglobals.py', 'strdispatch.py', 'twshell.py',
+       'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
+       # See note above for why this is skipped
+       # 'Shell.py',
+       'winconsole.py']
+
+    if have_pexpect:
+        top_mod.append('irunner.py')
+
+    if sys.platform == 'win32':
+        top_mod.append('platutils_win32.py')
+    elif os.name == 'posix':
+        top_mod.append('platutils_posix.py')
+    else:
+        top_mod.append('platutils_dummy.py')
+
+    # These are tested by nose, so skip IPython.kernel
+    top_pack = ['config','Extensions','frontend',
+                'testing','tests','tools','UserConfig']
+
+    if have_wx:
+        top_pack.append('gui')
+
+    modules  = ['IPython.%s' % m[:-3] for m in top_mod ]
+    packages = ['IPython.%s' % m for m in top_pack ]
+
+    # Make runners
+    runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
+    
+    # Test IPython.kernel using trial if twisted is installed
+    if have_zi and have_twisted and have_foolscap:
+        runners['trial'] = IPTester('trial',['IPython'])
+
+    runners['modules'] = IPTester(params=modules)
+
+    return runners
+
+
+def run_iptestall():
+    """Run the entire IPython test suite by calling nose and trial.
+    
+    This function constructs :class:`IPTester` instances for all IPython
+    modules and package and then runs each of them.  This causes the modules
+    and packages of IPython to be tested each in their own subprocess using
+    nose or twisted.trial appropriately.
+    """
+    runners = make_runners()
+    # Run all test runners, tracking execution time
+    failed = {}
+    t_start = time.time()
+    for name,runner in runners.iteritems():
+        print '*'*77
+        print 'IPython test set:',name
+        res = runner.run()
+        if res:
+            failed[name] = res
+    t_end = time.time()
+    t_tests = t_end - t_start
+    nrunners = len(runners)
+    nfail = len(failed)
+    # summarize results
+    print
+    print '*'*77
+    print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
+    print
+    if not failed:
+        print 'OK'
+    else:
+        # If anything went wrong, point out what command to rerun manually to
+        # see the actual errors and individual summary
+        print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
+        for name in failed:
+            failed_runner = runners[name]
+            print '-'*40
+            print 'Runner failed:',name
+            print 'You may wish to rerun this one individually, with:'
+            print ' '.join(failed_runner.call_args)
+            print
+
+
+def main():
+    if len(sys.argv) == 1:
+        run_iptestall()
+    else:
+        if sys.argv[1] == 'all':
+            run_iptestall()
+        else:
+            run_iptest()
+
+
+if __name__ == '__main__':
+    main()
  No newline at end of file

IPython/testing/plugin/ipdoctest.py

@@ -15,7 +15,6 b' Limitations:'
   won't even have these special _NN variables set at all.
 """
 
-
 #-----------------------------------------------------------------------------
 # Module imports
 
@@ -60,6 +59,19 b' log = logging.getLogger(__name__)'
 # machinery into a fit.  This code should be considered a gross hack, but it
 # gets the job done.
 
+def default_argv():
+    """Return a valid default argv for creating testing instances of ipython"""
+
+    # Get the install directory for the user configuration and tell ipython to
+    # use the default profile from there.
+    from IPython import UserConfig
+    ipcdir = os.path.dirname(UserConfig.__file__)
+    #ipconf = os.path.join(ipcdir,'ipy_user_conf.py')
+    ipconf = os.path.join(ipcdir,'ipythonrc')
+    #print 'conf:',ipconf # dbg
+    
+    return ['--colors=NoColor','--noterm_title','-rcfile=%s' % ipconf]
+
 
 # Hack to modify the %run command so we can sync the user's namespace with the
 # test globals.  Once we move over to a clean magic system, this will be done
@@ -85,9 +97,19 b' def _run_ns_sync(self,arg_s,runner=None):'
     This is strictly needed for running doctests that call %run.
     """
 
-    finder = py_file_finder(_run_ns_sync.test_filename)
+    # When tests call %run directly (not via doctest) these function attributes
+    # are not set
+    try:
+        fname = _run_ns_sync.test_filename
+    except AttributeError:
+        fname = arg_s
+
+    finder = py_file_finder(fname)
     out = _ip.IP.magic_run_ori(arg_s,runner,finder)
-    _run_ns_sync.test_globs.update(_ip.user_ns)
+    
+    # Simliarly, there is no test_globs when a test is NOT a doctest
+    if hasattr(_run_ns_sync,'test_globs'):
+        _run_ns_sync.test_globs.update(_ip.user_ns)
     return out
 
 
@@ -114,15 +136,31 b' class ipnsdict(dict):'
     def update(self,other):
         self._checkpoint()
         dict.update(self,other)
+
         # If '_' is in the namespace, python won't set it when executing code,
         # and we have examples that test it.  So we ensure that the namespace
         # is always 'clean' of it before it's used for test code execution.
         self.pop('_',None)
+
+        # The builtins namespace must *always* be the real __builtin__ module,
+        # else weird stuff happens.  The main ipython code does have provisions
+        # to ensure this after %run, but since in this class we do some
+        # aggressive low-level cleaning of the execution namespace, we need to
+        # correct for that ourselves, to ensure consitency with the 'real'
+        # ipython.
+        self['__builtins__'] = __builtin__
         
 
 def start_ipython():
     """Start a global IPython shell, which we need for IPython-specific syntax.
     """
+
+    # This function should only ever run once!
+    if hasattr(start_ipython,'already_called'):
+        return
+    start_ipython.already_called = True
+
+    # Ok,  first time we're called, go ahead
     import new
 
     import IPython
@@ -142,10 +180,11 b' def start_ipython():'
     _excepthook = sys.excepthook
     _main = sys.modules.get('__main__')
 
+    argv = default_argv()
+    
     # Start IPython instance.  We customize it to start with minimal frills.
     user_ns,global_ns = IPython.ipapi.make_user_namespaces(ipnsdict(),dict())
-    IPython.Shell.IPShell(['--colors=NoColor','--noterm_title'],
-                          user_ns,global_ns)
+    IPython.Shell.IPShell(argv,user_ns,global_ns)
 
     # Deactivate the various python system hooks added by ipython for
     # interactive convenience so we don't confuse the doctest system
@@ -691,6 +730,7 b' class ExtensionDoctest(doctests.Doctest):'
           to exclude any filename which matches them from inclusion in the test
           suite (using pattern.search(), NOT pattern.match() ).
         """
+
         if exclude_patterns is None:
             exclude_patterns = []
         self.exclude_patterns = map(re.compile,exclude_patterns)
@@ -800,11 +840,11 b' class ExtensionDoctest(doctests.Doctest):'
         Modified version that accepts extension modules as valid containers for
         doctests.
         """
-        #print '*** ipdoctest- wantFile:',filename  # dbg
+        # print '*** ipdoctest- wantFile:',filename  # dbg
 
         for pat in self.exclude_patterns:
             if pat.search(filename):
-                #print '###>>> SKIP:',filename  # dbg
+                # print '###>>> SKIP:',filename  # dbg
                 return False
 
         if is_extension_module(filename):
@@ -836,15 +876,33 b' class IPythonDoctest(ExtensionDoctest):'
                                   optionflags=optionflags,
                                   checker=self.checker)
 
-    def configure(self, options, config):
+    def options(self, parser, env=os.environ):
+        Plugin.options(self, parser, env)
+        parser.add_option('--ipdoctest-tests', action='store_true',
+                          dest='ipdoctest_tests',
+                          default=env.get('NOSE_IPDOCTEST_TESTS',True),
+                          help="Also look for doctests in test modules. "
+                          "Note that classes, methods and functions should "
+                          "have either doctests or non-doctest tests, "
+                          "not both. [NOSE_IPDOCTEST_TESTS]")
+        parser.add_option('--ipdoctest-extension', action="append",
+                          dest="ipdoctest_extension",
+                          help="Also look for doctests in files with "
+                          "this extension [NOSE_IPDOCTEST_EXTENSION]")
+        # Set the default as a list, if given in env; otherwise
+        # an additional value set on the command line will cause
+        # an error.
+        env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
+        if env_setting is not None:
+            parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
+    def configure(self, options, config):
         Plugin.configure(self, options, config)
-        self.doctest_tests = options.doctest_tests
-        self.extension = tolist(options.doctestExtension)
+        self.doctest_tests = options.ipdoctest_tests
+        self.extension = tolist(options.ipdoctest_extension)
 
         self.parser = IPDocTestParser()
         self.finder = DocTestFinder(parser=self.parser)
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
-

IPython/testing/plugin/show_refs.py

@@ -6,8 +6,9 b' This is used by a companion test case.'
 import gc
 
 class C(object):
-  def __del__(self):
-      print 'deleting object...'
+   def __del__(self):
+      pass
+      #print 'deleting object...'  # dbg
 
 c = C()
 

IPython/testing/plugin/test_refs.py

@@ -39,13 +39,10 b' def doctest_ivars():'
     Out[6]: 1
     """
     
[e-mail]_doctest
+#@dec.skip_doctest
 def doctest_refs():
     """DocTest reference holding issues when running scripts.
 
     In [32]: run show_refs.py
     c referrers: [<type 'dict'>]
-
-    In [33]: map(type,gc.get_referrers(c))
-    Out[33]: [<type 'dict'>]
     """

IPython/testing/tests/test_decorators.py

@@ -46,7 +46,7 b' def test_deliberately_broken():'
     """A deliberately broken test - we want to skip this one."""
     1/0
 
[e-mail]('foo')
[e-mail]('Testing the skip decorator')
 def test_deliberately_broken2():
     """Another deliberately broken test - we want to skip this one."""
     1/0

IPython/tests/obj_del.py

@@ -26,7 +26,7 b' import sys'
 
 class A(object):
     def __del__(self):
-      print 'object A deleted'
+        print 'obj_del.py: object A deleted'
 
 a = A()
 

IPython/tests/tclass.py

@@ -16,11 +16,12 b' class C(object):'
         self.name = name
         
     def __del__(self):
-        print 'Deleting object:',self.name
+        print 'tclass.py: deleting object:',self.name
 
 try:
     name = sys.argv[1]
 except IndexError:
     pass
 else:
-    c = C(name)
+    if name.startswith('C'):
+        c = C(name)

IPython/tests/test_genutils.py

@@ -210,7 +210,6 b' def test_get_home_dir_7():'
     home_dir = genutils.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
-
 #
 # Tests for get_ipython_dir
 #
@@ -239,7 +238,6 b' def test_get_ipython_dir_3():'
     ipdir = genutils.get_ipython_dir()
     nt.assert_equal(ipdir, os.path.abspath(os.path.join("someplace", "_ipython")))
 
-
 #
 # Tests for get_security_dir
 #
@@ -249,6 +247,14 b' def test_get_security_dir():'
     """Testcase to see if we can call get_security_dir without Exceptions."""
     sdir = genutils.get_security_dir()
 
+#
+# Tests for get_log_dir
+#
+
+@with_enivronment
+def test_get_log_dir():
+    """Testcase to see if we can call get_log_dir without Exceptions."""
+    sdir = genutils.get_log_dir()
 
 #
 # Tests for popkey
@@ -289,3 +295,12 b' def test_popkey_3():'
     nt.assert_equal(dct, dict(c=3))
     nt.assert_equal(genutils.popkey(dct, "c"), 3)
     nt.assert_equal(dct, dict())
+
+
+def test_filefind():
+    """Various tests for filefind"""
+    f = tempfile.NamedTemporaryFile()
+    print 'fname:',f.name
+    alt_dirs = genutils.get_ipython_dir()
+    t = genutils.filefind(f.name,alt_dirs)
+    print 'found:',t

IPython/tests/test_iplib.py

@@ -1,17 +1,80 b''
 """Tests for the key iplib module, where the main ipython class is defined.
 """
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
 
+# stdlib
+import os
+import shutil
+import tempfile
+
+# third party
 import nose.tools as nt
 
+# our own packages
+from IPython import ipapi, iplib
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Useful global ipapi object and main IPython one.  Unfortunately we have a
+# long precedent of carrying the 'ipapi' global object which is injected into
+# the system namespace as _ip, but that keeps a pointer to the actual IPython
+# InteractiveShell instance, which is named IP.  Since in testing we do need
+# access to the real thing (we want to probe beyond what ipapi exposes), make
+# here a global reference to each.  In general, things that are exposed by the
+# ipapi instance should be read from there, but we also will often need to use
+# the actual IPython one.
+
+# Get the public instance of IPython, and if it's None, make one so we can use
+# it for testing
+ip = ipapi.get()
+if ip is None:
+    # IPython not running yet,  make one from the testing machinery for
+    # consistency when the test suite is being run via iptest
+    from IPython.testing.plugin import ipdoctest
+    ip = ipapi.get()
+    
+IP = ip.IP  # This is the actual IPython shell 'raw' object.
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
 
 def test_reset():
     """reset must clear most namespaces."""
-    ip = _ip.IP
-    ip.reset()  # first, it should run without error
+    IP.reset()  # first, it should run without error
     # Then, check that most namespaces end up empty
-    for ns in ip.ns_refs_table:
-        if ns is ip.user_ns:
+    for ns in IP.ns_refs_table:
+        if ns is IP.user_ns:
             # The user namespace is reset with some data, so we can't check for
             # it being empty
             continue
         nt.assert_equals(len(ns),0)
+
+
+# make sure that user_setup can be run re-entrantly in 'install' mode.
+def test_user_setup():
+    # use a lambda to pass kwargs to the generator
+    user_setup = lambda a,k: iplib.user_setup(*a,**k)
+    kw = dict(mode='install', interactive=False)
+
+    # Call the user setup and verify that the directory exists
+    yield user_setup, (ip.options.ipythondir,''), kw
+    yield os.path.isdir, ip.options.ipythondir
+
+    # Now repeat the operation with a non-existent directory. Check both that
+    # the call succeeds and that the directory is created.
+    tmpdir = tempfile.mktemp(prefix='ipython-test-')
+    # Use a try with an empty except because try/finally doesn't work with a 
+    # yield in Python 2.4.
+    try:
+        yield user_setup, (tmpdir,''), kw
+        yield os.path.isdir, tmpdir
+    except:
+        pass
+    # Clean up the temp dir once done
+    shutil.rmtree(tmpdir)
+    
  No newline at end of file

IPython/tests/test_magic.py

@@ -3,15 +3,16 b''
 Needs to be run by nose (to make ipython session available).
 """
 
-# Standard library imports
 import os
 import sys
+import tempfile
+import types
 
-# Third-party imports
 import nose.tools as nt
 
-# From our own code
+from IPython.platutils import find_cmd, get_long_path_name
 from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
@@ -34,28 +35,6 b' def test_rehashx():'
     assert len(scoms) > 10
 
 
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [3]: run tclass.py
-
-    In [4]: run tclass first_pass
-
-    In [5]: run tclass second_pass
-    Deleting object: first_pass
-    """
-
-
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
@@ -79,13 +58,15 b' def doctest_hist_r():'
     hist -n -r 2  # random
     """
 
-
+# This test is known to fail on win32.
+# See ticket https://bugs.launchpad.net/bugs/366334
 def test_obj_del():
     """Test that object's __del__ methods are called on exit."""
     test_dir = os.path.dirname(__file__)
     del_file = os.path.join(test_dir,'obj_del.py')
-    out = _ip.IP.getoutput('ipython %s' % del_file)
-    nt.assert_equals(out,'object A deleted')
+    ipython_cmd = find_cmd('ipython')
+    out = _ip.IP.getoutput('%s %s' % (ipython_cmd, del_file))
+    nt.assert_equals(out,'obj_del.py: object A deleted')
 
 
 def test_shist():
@@ -133,3 +114,136 b' def test_fail_dec2(*a,**k):'
 def test_fail_dec3(*a,**k):
     yield nt.assert_true, False
 
+
+def doctest_refbug():
+    """Very nasty problem with references held by multiple runs of a script.
+    See: https://bugs.launchpad.net/ipython/+bug/269966
+
+    In [1]: _ip.IP.clear_main_mod_cache()
+    
+    In [2]: run refbug
+
+    In [3]: call_f()
+    lowercased: hello
+
+    In [4]: run refbug
+
+    In [5]: call_f()
+    lowercased: hello
+    lowercased: hello
+    """
+
+#-----------------------------------------------------------------------------
+# Tests for %run
+#-----------------------------------------------------------------------------
+
+# %run is critical enough that it's a good idea to have a solid collection of
+# tests for it, some as doctests and some as normal tests.
+
+def doctest_run_ns():
+    """Classes declared %run scripts must be instantiable afterwards.
+
+    In [11]: run tclass foo
+
+    In [12]: isinstance(f(),foo)
+    Out[12]: True
+    """
+
+    
+def doctest_run_ns2():
+    """Classes declared %run scripts must be instantiable afterwards.
+
+    In [4]: run tclass C-first_pass
+
+    In [5]: run tclass C-second_pass
+    tclass.py: deleting object: C-first_pass
+    """
+
[e-mail]_win32
+def doctest_run_builtins():
+    """Check that %run doesn't damage __builtins__ via a doctest.
+
+    This is similar to the test_run_builtins, but I want *both* forms of the
+    test to catch any possible glitches in our testing machinery, since that
+    modifies %run somewhat.  So for this, we have both a normal test (below)
+    and a doctest (this one).
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: f = tempfile.NamedTemporaryFile()
+
+    In [4]: f.write('pass\\n')
+
+    In [5]: f.flush()
+
+    In [6]: print 'B1:',type(__builtins__)
+    B1: <type 'module'>
+
+    In [7]: %run $f.name
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: print 'B2:',type(__builtins__)
+    B2: <type 'module'>
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TestMagicRun(object):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        f = tempfile.NamedTemporaryFile()
+        f.write('pass\n')
+        f.flush()
+        self.tmpfile = f
+
+    def run_tmpfile(self):
+        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
+        # See below and ticket https://bugs.launchpad.net/bugs/366353
+        _ip.magic('run %s' % self.tmpfile.name)
+
+    # See https://bugs.launchpad.net/bugs/366353
+    @dec.skip_if_not_win32
+    def test_run_tempfile_path(self):
+        tt.assert_equals(True,False,"%run doesn't work with tempfile paths on win32.")
+
+    # See https://bugs.launchpad.net/bugs/366353
+    @dec.skip_win32
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    # See https://bugs.launchpad.net/bugs/366353
+    @dec.skip_win32
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    # See https://bugs.launchpad.net/bugs/366353
+    @dec.skip_win32
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        p2 = str(_ip.IP.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+    def teardown(self):
+        self.tmpfile.close()

docs/autogen_api.py

@@ -25,6 +25,7 b" if __name__ == '__main__':"
                                         r'\.cocoa',
                                         r'\.ipdoctest',
                                         r'\.Gnuplot',
+                                        r'\.frontend.process.winprocess',
                                         ]
     docwriter.write_api_docs(outdir)
     docwriter.write_index(outdir, 'gen',

docs/source/changes.txt

@@ -75,6 +75,9 b' Bug fixes'
   The block is assigned to pasted_block even if code
   raises exception.
 
+* Bug #274067 'The code in get_home_dir is broken for py2exe' was
+  fixed.
+
 Backwards incompatible changes
 ------------------------------
 

docs/source/development/overview.txt

@@ -345,6 +345,37 b' nosetests option. For example, you can use ``--pdb`` or ``--pdb-failures`` to'
 automatically activate the interactive Pdb debugger on errors or failures.  See
 the nosetests documentation for further details.
 
+.. warning::
+
+   Note that right now we have a nasty interaction between ipdoctest and
+   twisted.  Until we figure this out, please use the following instructions to
+   ensure that at least you run all the tests.
+
+Right now, if you now run::
+
+    $ iptest [any options] [any submodules]
+
+it will NOT load ipdoctest but won't cause any Twisted problems.
+
+Once you're happy that you didn't break Twisted, run::
+
+    $ iptest --with-ipdoctest [any options] [any submodules]
+
+This MAY give a Twisted AlreadyCalledError exception at the end, but it will
+also correctly load up all of the ipython-specific tests and doctests.
+
+The above can be made easier with a trivial shell alias::
+
+    $ alias iptest2='iptest --with-ipdoctest'
+
+So that you can run::
+
+    $ iptest ...
+    # Twisted happy
+    # iptest2 ...
+    # ignore possible Twisted error, this checks all the rest.
+
+
 A few tips for writing tests
 ----------------------------
 

docs/source/parallel/parallel_process.txt

@@ -302,18 +302,33 b' The ``--furl-file`` flag works like this::'
 Make FURL files persistent
 ---------------------------
 
-At fist glance it may seem that that managing the FURL files is a bit annoying.  Going back to the house and key analogy, copying the FURL around each time you start the controller is like having to make a new key every time you want to unlock the door and enter your house.  As with your house, you want to be able to create the key (or FURL file) once, and then simply use it at any point in the future.
+At fist glance it may seem that that managing the FURL files is a bit
+annoying. Going back to the house and key analogy, copying the FURL around
+each time you start the controller is like having to make a new key every time
+you want to unlock the door and enter your house. As with your house, you want
+to be able to create the key (or FURL file) once, and then simply use it at
+any point in the future.
 
-This is possible.  The only thing you have to do is decide what ports the controller will listen on for the engines and clients.  This is done as follows::
+This is possible, but before you do this, you **must** remove any old FURL
+files in the :file:`~/.ipython/security` directory.
+
+.. warning::
+
+    You **must** remove old FURL files before using persistent FURL files.
+
+Then, The only thing you have to do is decide what ports the controller will
+listen on for the engines and clients. This is done as follows::
 
     $ ipcontroller -r --client-port=10101 --engine-port=10102
 
-These options also work with all of the various modes of 
+These options also work with all of the various modes of
 :command:`ipcluster`::
 
     $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102
 
-Then, just copy the furl files over the first time and you are set.  You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports.
+Then, just copy the furl files over the first time and you are set. You can
+start and stop the controller and engines any many times as you want in the
+future, just make sure to tell the controller to use the *same* ports.
 
 .. note::
 

docs/source/parallel/visionhpc.txt

@@ -2,6 +2,10 b''
  IPython/Vision Beam Pattern Demo
 ==================================
 
+.. note::
+
+    This page has not been updated to reflect the recent work on ipcluster.
+    This work makes it much easier to use IPython on a cluster.
 
 Installing and testing IPython at OSC systems
 =============================================

docs/sphinxext/only_directives.py

@@ -5,9 +5,15 b''
 
 from docutils.nodes import Body, Element
 from docutils.writers.html4css1 import HTMLTranslator
-from sphinx.latexwriter import LaTeXTranslator
 from docutils.parsers.rst import directives
 
+# The sphinx API has changed, so we try both the old and new import forms
+try:
+    from sphinx.latexwriter import LaTeXTranslator
+except ImportError:
+    from sphinx.writers.latex import LaTeXTranslator
+
+
 class html_only(Body, Element):
     pass
 

setupbase.py

@@ -109,7 +109,7 b' def find_packages():'
     add_package(packages, 'gui')
     add_package(packages, 'gui.wx')
     add_package(packages, 'frontend', tests=True)
-    add_package(packages, 'frontend._process')
+    add_package(packages, 'frontend.process')
     add_package(packages, 'frontend.wx')
     add_package(packages, 'frontend.cocoa', tests=True)
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)