Massive amount of work to improve the test suite, restores doctests.

After Brian's comments, I realized that our test machinery was NOT in
reality running all the ipython-syntax doctests we have.  This is now fixed.
The test suite isn't completely passing, but this commit is for the
underlying machinery.  I will now work on fixing as many broken tests as I
can.

Fixes https://bugs.launchpad.net/ipython/+bug/505071

IPython/core/tests/test_run.py

@@ -0,0 +1,185 b''
+"""Tests for code execution (%run and related), which is particularly tricky.
+
+Because of how %run manages namespaces, and the fact that we are trying here to
+verify subtle object deletion and reference counting issues, the %run tests
+will be kept in this separate file.  This makes it easier to aggregate in one
+place the tricks needed to handle it; most other magics are much easier to test
+and we do so in a common test_magic file.
+"""
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import os
+import sys
+import tempfile
+
+# third-party
+import nose.tools as nt
+
+# our own
+from IPython.utils.platutils import find_cmd
+from IPython.utils import genutils
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
+def doctest_refbug():
+    """Very nasty problem with references held by multiple runs of a script.
+    See: https://bugs.launchpad.net/ipython/+bug/269966
+
+    In [1]: _ip.clear_main_mod_cache()
+    # random
+    
+    In [2]: %run refbug
+
+    In [3]: call_f()
+    lowercased: hello
+
+    In [4]: %run refbug
+
+    In [5]: call_f()
+    lowercased: hello
+    lowercased: hello
+    """
+
+
+def doctest_run_builtins():
+    r"""Check that %run doesn't damage __builtins__.
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: fname = tempfile.mkstemp('.py')[1]
+
+    In [3]: f = open(fname,'w')
+
+    In [4]: f.write('pass\n')
+
+    In [5]: f.flush()
+
+    In [6]: t1 = type(__builtins__)
+
+    In [7]: %run "$fname"
+
+    In [7]: f.close()
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: t2 = type(__builtins__)
+
+    In [10]: t1 == t2
+    Out[10]: True
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+
+    In [12]: try:
+       ....:     os.unlink(fname)
+       ....: except:
+       ....:     pass
+       ....: 
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TempFileMixin(object):
+    def mktmp(self, src, ext='.py'):
+        """Make a valid python temp file."""
+        fname, f = tt.temp_pyfile(src, ext)
+        self.tmpfile = f
+        self.fname = fname
+
+    def teardown(self):
+        self.tmpfile.close()
+        try:
+            os.unlink(self.fname)
+        except:
+            # On Windows, even though we close the file, we still can't delete
+            # it.  I have no clue why
+            pass
+
+
+class TestMagicRunPass(TempFileMixin):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        self.mktmp('pass\n')
+
+    def run_tmpfile(self):
+        _ip = get_ipython()
+        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
+        # See below and ticket https://bugs.launchpad.net/bugs/366353
+        _ip.magic('run "%s"' % self.fname)
+
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+        _ip = get_ipython()
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        _ip = get_ipython()
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        _ip = get_ipython()
+        p2 = str(_ip.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+
+class TestMagicRunSimple(TempFileMixin):
+
+    def test_simpledef(self):
+        """Test that simple class definitions work."""
+        src = ("class foo: pass\n"
+               "def f(): return foo()")
+        self.mktmp(src)
+        _ip.magic('run "%s"' % self.fname)
+        _ip.runlines('t = isinstance(f(), foo)')
+        nt.assert_true(_ip.user_ns['t'])
+        
+    def test_obj_del(self):
+        """Test that object's __del__ methods are called on exit."""
+        
+        # This test is known to fail on win32.
+        # See ticket https://bugs.launchpad.net/bugs/366334
+        src = ("class A(object):\n"
+               "    def __del__(self):\n"
+               "        print 'object A deleted'\n"
+               "a = A()\n")
+        self.mktmp(src)
+        tt.ipexec_validate(self.fname, 'object A deleted')
+
+    def test_tclass(self):
+        mydir = os.path.dirname(__file__)
+        tc = os.path.join(mydir, 'tclass')
+        src = ("%%run '%s' C-first\n"
+               "%%run '%s' C-second\n") % (tc, tc)
+        self.mktmp(src, '.ipy')
+        out = """\
+ARGV 1-: ['C-first']
+ARGV 1-: ['C-second']
+tclass.py: deleting object: C-first
+"""
+        tt.ipexec_validate(self.fname, out)

IPython/core/magic.py

@@ -3506,6 +3506,7 b' Defaulting color scheme to \'NoColor\'"""'
         """Reload an IPython extension by its module name."""
         self.reload_extension(module_str)
 
+    @testdec.skip_doctest
     def magic_install_profiles(self, s):
         """Install the default IPython profiles into the .ipython dir.
 

IPython/core/prefilter.py

@@ -416,7 +416,7 b' class PrefilterManager(Component):'
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
-    def prefilter_lines(self, lines, continue_prompt):
+    def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of

IPython/core/tests/tclass.py

@@ -1,16 +1,12 b''
-"""Simple script to instantiate a class for testing %run"""
+"""Simple script to be run *twice*, to check reference counting bugs.
 
-import sys
-
-# An external test will check that calls to f() work after %run
-class foo: pass
+See test_run for details."""
 
-def f():
-    return foo()
+import sys
 
-# We also want to ensure that while objects remain available for immediate
-# access, objects from *previous* runs of the same script get collected, to
-# avoid accumulating massive amounts of old references.
+# We want to ensure that while objects remain available for immediate access,
+# objects from *previous* runs of the same script get collected, to avoid
+# accumulating massive amounts of old references.
 class C(object):
     def __init__(self,name):
         self.name = name
@@ -18,6 +14,7 b' class C(object):'
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
 
+
 try:
     name = sys.argv[1]
 except IndexError:
@@ -25,3 +22,8 b' except IndexError:'
 else:
     if name.startswith('C'):
         c = C(name)
+
+#print >> sys.stderr, "ARGV:", sys.argv  # dbg
+# This print statement is NOT debugging, we're making the check on a completely
+# separate process so we verify by capturing stdout.
+print 'ARGV 1-:', sys.argv[1:]

IPython/core/tests/test_magic.py

@@ -2,21 +2,31 b''
 
 Needs to be run by nose (to make ipython session available).
 """
+from __future__ import absolute_import
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
+# third-party
 import nose.tools as nt
 
+# our own
+from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
+#-----------------------------------------------------------------------------
 
 def test_rehashx():
     # clear up everything
@@ -63,17 +73,6 b' def doctest_hist_r():'
     hist -n -r 2  # random
     """
 
-# This test is known to fail on win32.
-# See ticket https://bugs.launchpad.net/bugs/366334
-def test_obj_del():
-    _ip = get_ipython()
-    """Test that object's __del__ methods are called on exit."""
-    test_dir = os.path.dirname(__file__)
-    del_file = os.path.join(test_dir,'obj_del.py')
-    ipython_cmd = find_cmd('ipython')
-    out = _ip.getoutput('%s %s' % (ipython_cmd, del_file))
-    nt.assert_equals(out,'obj_del.py: object A deleted')
-
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
@@ -113,161 +112,6 b' def test_numpy_clear_array_undec():'
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
[e-mail]()
-def test_fail_dec(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]('This one shouldn not run')
-def test_fail_dec2(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]
-def test_fail_dec3(*a,**k):
-    yield nt.assert_true, False
-
-
-def doctest_refbug():
-    """Very nasty problem with references held by multiple runs of a script.
-    See: https://bugs.launchpad.net/ipython/+bug/269966
-
-    In [1]: _ip.clear_main_mod_cache()
-    
-    In [2]: run refbug
-
-    In [3]: call_f()
-    lowercased: hello
-
-    In [4]: run refbug
-
-    In [5]: call_f()
-    lowercased: hello
-    lowercased: hello
-    """
-
-#-----------------------------------------------------------------------------
-# Tests for %run
-#-----------------------------------------------------------------------------
-
-# %run is critical enough that it's a good idea to have a solid collection of
-# tests for it, some as doctests and some as normal tests.
-
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass foo
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [4]: run tclass C-first_pass
-
-    In [5]: run tclass C-second_pass
-    tclass.py: deleting object: C-first_pass
-    """
-
-def doctest_run_builtins():
-    """Check that %run doesn't damage __builtins__ via a doctest.
-
-    This is similar to the test_run_builtins, but I want *both* forms of the
-    test to catch any possible glitches in our testing machinery, since that
-    modifies %run somewhat.  So for this, we have both a normal test (below)
-    and a doctest (this one).
-
-    In [1]: import tempfile
-
-    In [2]: bid1 = id(__builtins__)
-
-    In [3]: fname = tempfile.mkstemp()[1]
-
-    In [3]: f = open(fname,'w')
-
-    In [4]: f.write('pass\\n')
-
-    In [5]: f.flush()
-
-    In [6]: print type(__builtins__)
-    <type 'module'>
-
-    In [7]: %run "$fname"
-
-    In [7]: f.close()
-
-    In [8]: bid2 = id(__builtins__)
-
-    In [9]: print type(__builtins__)
-    <type 'module'>
-
-    In [10]: bid1 == bid2
-    Out[10]: True
-
-    In [12]: try:
-       ....:     os.unlink(fname)
-       ....: except:
-       ....:     pass
-       ....: 
-    """
-
-# For some tests, it will be handy to organize them in a class with a common
-# setup that makes a temp file
-
-class TestMagicRun(object):
-
-    def setup(self):
-        """Make a valid python temp file."""
-        fname = tempfile.mkstemp('.py')[1]
-        f = open(fname,'w')
-        f.write('pass\n')
-        f.flush()
-        self.tmpfile = f
-        self.fname = fname
-
-    def run_tmpfile(self):
-        _ip = get_ipython()
-        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
-        # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run "%s"' % self.fname)
-
-    def test_builtins_id(self):
-        """Check that %run doesn't damage __builtins__ """
-        _ip = get_ipython()
-        # Test that the id of __builtins__ is not modified by %run
-        bid1 = id(_ip.user_ns['__builtins__'])
-        self.run_tmpfile()
-        bid2 = id(_ip.user_ns['__builtins__'])
-        tt.assert_equals(bid1, bid2)
-
-    def test_builtins_type(self):
-        """Check that the type of __builtins__ doesn't change with %run.
-        
-        However, the above could pass if __builtins__ was already modified to
-        be a dict (it should be a module) by a previous use of %run.  So we
-        also check explicitly that it really is a module:
-        """
-        _ip = get_ipython()
-        self.run_tmpfile()
-        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
-
-    def test_prompts(self):
-        """Test that prompts correctly generate after %run"""
-        self.run_tmpfile()
-        _ip = get_ipython()
-        p2 = str(_ip.outputcache.prompt2).strip()
-        nt.assert_equals(p2[:3], '...')
-
-    def teardown(self):
-        self.tmpfile.close()
-        try:
-            os.unlink(self.fname)
-        except:
-            # On Windows, even though we close the file, we still can't delete
-            # it.  I have no clue why
-            pass
-
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():

IPython/testing/decorators.py

@@ -64,6 +64,9 b" if sys.version[0]=='2':"
 else:
     from _paramtestpy3 import parametric
 
+# Expose the unittest-driven decorators
+from ipunittest import ipdoctest, ipdocstring
+
 # Grab the numpy-specific decorators which we keep in a file that we
 # occasionally update from upstream: decorators.py is a copy of
 # numpy.testing.decorators, we expose all of it here.

IPython/testing/iptest.py

@@ -16,6 +16,8 b' For now, this script requires that both nose and twisted are installed. This'
 will change in the future.
 """
 
+from __future__ import absolute_import
+
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
@@ -34,6 +36,8 b' from nose.core import TestProgram'
 
 from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, FindCmdError
+from . import globalipapp
+from .plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
@@ -76,7 +80,11 b' def make_exclude():'
     # cause testing problems.  We should strive to minimize the number of
     # skipped modules, since this means untested code.  As the testing
     # machinery solidifies, this list should eventually become empty.
-    EXCLUDE = [pjoin('IPython', 'external'),
+
+    # Note that these exclusions only mean that the docstrings are not analyzed
+    # for examples to be run as tests, if there are other test functions in
+    # those modules, they do get run.
+    exclusions = [pjoin('IPython', 'external'),
                pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
                pjoin('IPython_doctest_plugin'),
                pjoin('IPython', 'quarantine'),
@@ -88,58 +96,58 b' def make_exclude():'
                ]
 
     if not have_wx:
-        EXCLUDE.append(pjoin('IPython', 'gui'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookwx'))
+        exclusions.append(pjoin('IPython', 'gui'))
+        exclusions.append(pjoin('IPython', 'frontend', 'wx'))
+        exclusions.append(pjoin('IPython', 'lib', 'inputhookwx'))
 
     if not have_gtk or not have_gobject:
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookgtk'))
+        exclusions.append(pjoin('IPython', 'lib', 'inputhookgtk'))
 
     if not have_wx_aui:
-        EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+        exclusions.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
 
     if not have_objc:
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
+        exclusions.append(pjoin('IPython', 'frontend', 'cocoa'))
 
     if not sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
+        exclusions.append(pjoin('IPython', 'utils', 'platutils_win32'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
+        exclusions.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
+        exclusions.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
 
     if not os.name == 'posix':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
+        exclusions.append(pjoin('IPython', 'utils', 'platutils_posix'))
 
     if not have_pexpect:
-        EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
+        exclusions.append(pjoin('IPython', 'scripts', 'irunner'))
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have_twisted and have_zi and have_foolscap):
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
+        exclusions.append(pjoin('IPython', 'frontend', 'frontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_linefrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 
+        exclusions.append(pjoin('IPython', 'frontend', 'tests', 
                              'test_frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_asyncfrontendbase')),
-        EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
+        exclusions.append(pjoin('IPython', 'testing', 'parametric'))
+        exclusions.append(pjoin('IPython', 'testing', 'util'))
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
-        EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+        exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
-    return EXCLUDE
+    return exclusions
 
 
 #-----------------------------------------------------------------------------
@@ -163,16 +171,16 b' class IPTester(object):'
         if runner == 'iptest':
             # Find our own 'iptest' script OS-level entry point
             try:
-                iptest_path = find_cmd('iptest')
+                iptest_path = os.path.abspath(find_cmd('iptest'))
             except FindCmdError:
                 # Script not installed (may be the case for testing situations
                 # that are running from a source tree only), pull from internal
                 # path:
                 iptest_path = pjoin(genutils.get_ipython_package_dir(),
                                     'scripts','iptest')
-            self.runner = [iptest_path,'-v']
+            self.runner = ['python', iptest_path, '-v']
         else:
-            self.runner = [find_cmd('trial')]
+            self.runner = ['python', os.path.abspath(find_cmd('trial'))]
         if params is None:
             params = []
         if isinstance(params,str):
@@ -238,11 +246,13 b' def make_runners():'
     nose_packages = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils']
     trial_packages = ['kernel']
-    #trial_packages = []  # dbg 
 
     if have_wx:
         nose_packages.append('gui')
 
+    #nose_packages = ['core']  # dbg
+    #trial_packages = []  # dbg 
+
     nose_packages = ['IPython.%s' % m for m in nose_packages ]
     trial_packages = ['IPython.%s' % m for m in trial_packages ]
 
@@ -268,15 +278,14 b' def run_iptest():'
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
-    argv = sys.argv + [ 
-                        # Loading ipdoctest causes problems with Twisted.
-                        # I am removing this as a temporary fix to get the 
-                        # test suite back into working shape.  Our nose
-                        # plugin needs to be gone through with a fine
-                        # toothed comb to find what is causing the problem.
-                        # '--with-ipdoctest',
-                        # '--ipdoctest-tests','--ipdoctest-extension=txt',
-                        # '--detailed-errors',
+    argv = sys.argv + [ '--detailed-errors',
+                        # Loading ipdoctest causes problems with Twisted, but
+                        # our test suite runner now separates things and runs
+                        # all Twisted tests with trial.
+                        '--with-ipdoctest',
+                        '--ipdoctest-tests','--ipdoctest-extension=txt',
+                        
+                        #'-x','-s',  # dbg
                        
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
@@ -300,17 +309,18 b' def run_iptest():'
     if not has_tests:
         argv.append('IPython')
 
-    # Construct list of plugins, omitting the existing doctest plugin, which
-    # ours replaces (and extends).
-    EXCLUDE = make_exclude()
-    plugins = []
-    # plugins = [IPythonDoctest(EXCLUDE)]
+    ## # Construct list of plugins, omitting the existing doctest plugin, which
+    ## # ours replaces (and extends).
+    plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
+    # We need a global ipython running in this process
+    globalipapp.start_ipython()
+    # Now nose can run
     TestProgram(argv=argv,plugins=plugins)
 
 

IPython/testing/ipunittest.py

@@ -22,6 +22,8 b' Authors'
 - Fernando Perez <[e-mail]>
 """
 
+from __future__ import absolute_import
+
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
@@ -40,14 +42,16 b' import sys'
 import unittest
 from doctest import DocTestFinder, DocTestRunner, TestResults
 
-# Our own
-import nosepatch
+# Our own, a nose monkeypatch
+from . import nosepatch
 
 # We already have python3-compliant code for parametric tests
 if sys.version[0]=='2':
-    from _paramtestpy2 import ParametricTestCase
+    from ._paramtestpy2 import ParametricTestCase
 else:
-    from _paramtestpy3 import ParametricTestCase
+    from ._paramtestpy3 import ParametricTestCase
+
+from . import globalipapp
 
 #-----------------------------------------------------------------------------
 # Classes and functions
@@ -68,9 +72,13 b' class IPython2PythonConverter(object):'
     implementation, but for now it only does prompt convertion."""
     
     def __init__(self):
-        self.ps1 = re.compile(r'In\ \[\d+\]: ')
-        self.ps2 = re.compile(r'\ \ \ \.\.\.+: ')
-        self.out = re.compile(r'Out\[\d+\]: \s*?\n?')
+        self.rps1 = re.compile(r'In\ \[\d+\]: ')
+        self.rps2 = re.compile(r'\ \ \ \.\.\.+: ')
+        self.rout = re.compile(r'Out\[\d+\]: \s*?\n?')
+        self.pyps1 = '>>> '
+        self.pyps2 = '... '
+        self.rpyps1 = re.compile ('(\s*%s)(.*)$' % self.pyps1)
+        self.rpyps2 = re.compile ('(\s*%s)(.*)$' % self.pyps2)
 
     def __call__(self, ds):
         """Convert IPython prompts to python ones in a string."""
@@ -79,10 +87,34 b' class IPython2PythonConverter(object):'
         pyout = ''
 
         dnew = ds
-        dnew = self.ps1.sub(pyps1, dnew)
-        dnew = self.ps2.sub(pyps2, dnew)
-        dnew = self.out.sub(pyout, dnew)
-        return dnew
+        dnew = self.rps1.sub(pyps1, dnew)
+        dnew = self.rps2.sub(pyps2, dnew)
+        dnew = self.rout.sub(pyout, dnew)
+        ip = globalipapp.get_ipython()
+
+        # Convert input IPython source into valid Python.
+        out = []
+        newline = out.append
+        for line in dnew.splitlines():
+
+            mps1 = self.rpyps1.match(line)
+            if mps1 is not None:
+                prompt, text = mps1.groups()
+                newline(prompt+ip.prefilter(text, False))
+                continue
+
+            mps2 = self.rpyps2.match(line)
+            if mps2 is not None:
+                prompt, text = mps2.groups()
+                newline(prompt+ip.prefilter(text, True))
+                continue
+            
+            newline(line)
+        newline('')  # ensure a closing newline, needed by doctest
+        #print "PYSRC:", '\n'.join(out)  # dbg
+        return '\n'.join(out)
+
+    #return dnew
 
 
 class Doc2UnitTester(object):

IPython/testing/plugin/ipdoctest.py

@@ -49,183 +49,14 b' from nose.util import anyp, getpackage, test_address, resolve_name, tolist'
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
+#-----------------------------------------------------------------------------
 
 log = logging.getLogger(__name__)
 
-###########################################################################
-# *** HACK ***
-# We must start our own ipython object and heavily muck with it so that all the
-# modifications IPython makes to system behavior don't send the doctest
-# machinery into a fit.  This code should be considered a gross hack, but it
-# gets the job done.
-
-def default_argv():
-    """Return a valid default argv for creating testing instances of ipython"""
-
-    # Get the install directory for the user configuration and tell ipython to
-    # use the default profile from there.
-    from IPython.config import default
-    ipcdir = os.path.dirname(default.__file__)
-    ipconf = os.path.join(ipcdir,'ipython_config.py')
-    #print 'conf:',ipconf # dbg
-    return ['--colors=NoColor', '--no-term-title','--no-banner',
-            '--config-file=%s' % ipconf]
-
-
-# Hack to modify the %run command so we can sync the user's namespace with the
-# test globals.  Once we move over to a clean magic system, this will be done
-# with much less ugliness.
-
-class py_file_finder(object):
-    def __init__(self,test_filename):
-        self.test_filename = test_filename
-        
-    def __call__(self,name):
-        from IPython.utils.genutils import get_py_filename
-        try:
-            return get_py_filename(name)
-        except IOError:
-            test_dir = os.path.dirname(self.test_filename)
-            new_path = os.path.join(test_dir,name)
-            return get_py_filename(new_path)
-    
-
-def _run_ns_sync(self,arg_s,runner=None):
-    """Modified version of %run that syncs testing namespaces.
-
-    This is strictly needed for running doctests that call %run.
-    """
-
-    # When tests call %run directly (not via doctest) these function attributes
-    # are not set
-    try:
-        fname = _run_ns_sync.test_filename
-    except AttributeError:
-        fname = arg_s
-
-    finder = py_file_finder(fname)
-    out = _ip.magic_run_ori(arg_s,runner,finder)
-    
-    # Simliarly, there is no test_globs when a test is NOT a doctest
-    if hasattr(_run_ns_sync,'test_globs'):
-        _run_ns_sync.test_globs.update(_ip.user_ns)
-    return out
-
-
-class ipnsdict(dict):
-    """A special subclass of dict for use as an IPython namespace in doctests.
-
-    This subclass adds a simple checkpointing capability so that when testing
-    machinery clears it (we use it as the test execution context), it doesn't
-    get completely destroyed.
-    """
-    
-    def __init__(self,*a):
-        dict.__init__(self,*a)
-        self._savedict = {}
-        
-    def clear(self):
-        dict.clear(self)
-        self.update(self._savedict)
-
-    def _checkpoint(self):
-        self._savedict.clear()
-        self._savedict.update(self)
-
-    def update(self,other):
-        self._checkpoint()
-        dict.update(self,other)
-
-        # If '_' is in the namespace, python won't set it when executing code,
-        # and we have examples that test it.  So we ensure that the namespace
-        # is always 'clean' of it before it's used for test code execution.
-        self.pop('_',None)
-
-        # The builtins namespace must *always* be the real __builtin__ module,
-        # else weird stuff happens.  The main ipython code does have provisions
-        # to ensure this after %run, but since in this class we do some
-        # aggressive low-level cleaning of the execution namespace, we need to
-        # correct for that ourselves, to ensure consitency with the 'real'
-        # ipython.
-        self['__builtins__'] = __builtin__
-        
-
-def start_ipython():
-    """Start a global IPython shell, which we need for IPython-specific syntax.
-    """
-
-    # This function should only ever run once!
-    if hasattr(start_ipython,'already_called'):
-        return
-    start_ipython.already_called = True
-
-    # Ok,  first time we're called, go ahead
-    import new
-
-    import IPython
-    from IPython.core import ipapp, iplib
-    
-    def xsys(cmd):
-        """Execute a command and print its output.
-
-        This is just a convenience function to replace the IPython system call
-        with one that is more doctest-friendly.
-        """
-        cmd = _ip.var_expand(cmd,depth=1)
-        sys.stdout.write(commands.getoutput(cmd))
-        sys.stdout.flush()
-
-    # Store certain global objects that IPython modifies
-    _displayhook = sys.displayhook
-    _excepthook = sys.excepthook
-    _main = sys.modules.get('__main__')
-
-    argv = default_argv()
-    
-    # Start IPython instance.  We customize it to start with minimal frills.
-    user_ns,global_ns = iplib.make_user_namespaces(ipnsdict(),{})
-    ip = ipapp.IPythonApp(argv, user_ns=user_ns, user_global_ns=global_ns)
-    ip.initialize()
-    ip.shell.builtin_trap.set()
-
-    # Deactivate the various python system hooks added by ipython for
-    # interactive convenience so we don't confuse the doctest system
-    sys.modules['__main__'] = _main
-    sys.displayhook = _displayhook
-    sys.excepthook = _excepthook
-
-    # So that ipython magics and aliases can be doctested (they work by making
-    # a call into a global _ip object)
-    __builtin__._ip = ip.shell
-
-    # Modify the IPython system call with one that uses getoutput, so that we
-    # can capture subcommands and print them to Python's stdout, otherwise the
-    # doctest machinery would miss them.
-    ip.shell.system = xsys
-
-    # Also patch our %run function in.
-    im = new.instancemethod(_run_ns_sync,_ip, _ip.__class__)
-    ip.shell.magic_run_ori = _ip.magic_run
-    ip.shell.magic_run = im
-
-    # XXX - For some very bizarre reason, the loading of %history by default is
-    # failing.  This needs to be fixed later, but for now at least this ensures
-    # that tests that use %hist run to completion.
-    from IPython.core import history
-    history.init_ipython(ip.shell)
-    if not hasattr(ip.shell,'magic_history'):
-        raise RuntimeError("Can't load magics, aborting")
-
-
-# The start call MUST be made here.  I'm not sure yet why it doesn't work if
-# it is made later, at plugin initialization time, but in all my tests, that's
-# the case.
-start_ipython()
-
-# *** END HACK ***
-###########################################################################
 
+#-----------------------------------------------------------------------------
 # Classes and functions
+#-----------------------------------------------------------------------------
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
@@ -288,7 +119,7 b' class DocTestFinder(doctest.DocTestFinder):'
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
-
+        #print '_find for:', obj, name, module  # dbg
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = DocTestSkip(obj)
@@ -396,8 +227,9 b' class DocTestCase(doctests.DocTestCase):'
         self._dt_runner = runner
 
 
-        # Each doctest should remember what directory it was loaded from...
-        self._ori_dir = os.getcwd()
+        # Each doctest should remember the directory it was loaded from, so
+        # things like %run work without too many contortions
+        self._ori_dir = os.path.dirname(test.filename)
 
     # Modified runTest from the default stdlib
     def runTest(self):
@@ -418,6 +250,7 b' class DocTestCase(doctests.DocTestCase):'
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
             curdir = os.getcwd()
+            #print 'runTest in dir:', self._ori_dir  # dbg
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
@@ -432,7 +265,7 b' class DocTestCase(doctests.DocTestCase):'
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
-        
+        #print "setUp test", self._dt_test.examples # dbg
         if isinstance(self._dt_test.examples[0],IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
@@ -731,8 +564,10 b' class IPDocTestRunner(doctest.DocTestRunner,object):'
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
-        _run_ns_sync.test_globs = test.globs
-        _run_ns_sync.test_filename = test.filename
+        #_ip._ipdoctest_test_globs = test.globs
+        #_ip._ipdoctest_test_filename = test.filename
+
+        test.globs.update(_ip.user_ns)
         
         return super(IPDocTestRunner,self).run(test,
                                                compileflags,out,clear_globs)
@@ -846,6 +681,7 b' class ExtensionDoctest(doctests.Doctest):'
 
 
     def loadTestsFromFile(self, filename):
+        #print "ipdoctest - from file", filename # dbg
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
@@ -895,6 +731,7 b' class IPythonDoctest(ExtensionDoctest):'
         """Look for doctests in the given object, which will be a
         function, method or class.
         """
+        #print 'Plugin analyzing:', obj, parent  # dbg
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
@@ -909,6 +746,7 b' class IPythonDoctest(ExtensionDoctest):'
                                   checker=self.checker)
 
     def options(self, parser, env=os.environ):
+        #print "Options for nose plugin:", self.name # dbg
         Plugin.options(self, parser, env)
         parser.add_option('--ipdoctest-tests', action='store_true',
                           dest='ipdoctest_tests',
@@ -929,6 +767,7 b' class IPythonDoctest(ExtensionDoctest):'
             parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
     def configure(self, options, config):
+        #print "Configuring nose plugin:", self.name # dbg
         Plugin.configure(self, options, config)
         self.doctest_tests = options.ipdoctest_tests
         self.extension = tolist(options.ipdoctest_extension)

IPython/testing/tools.py

@@ -29,10 +29,11 b' Authors'
 import os
 import re
 import sys
+import tempfile
 
 import nose.tools as nt
 
-from IPython.utils import genutils
+from IPython.utils import genutils, platutils
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -128,5 +129,93 b' def parse_test_output(txt):'
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
+
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
+
+
+def temp_pyfile(src, ext='.py'):
+    """Make a temporary python file, return filename and filehandle.
+
+    Parameters
+    ----------
+    src : string or list of strings (no need for ending newlines if list)
+      Source code to be written to the file.
+
+    ext : optional, string
+      Extension for the generated file.
+
+    Returns
+    -------
+    (filename, open filehandle)
+      It is the caller's responsibility to close the open file and unlink it.
+    """
+    fname = tempfile.mkstemp(ext)[1]
+    f = open(fname,'w')
+    f.write(src)
+    f.flush()
+    return fname, f
+
+
+def default_argv():
+    """Return a valid default argv for creating testing instances of ipython"""
+
+    # Get the install directory for the user configuration and tell ipython to
+    # use the default profile from there.
+    from IPython.config import default
+    ipcdir = os.path.dirname(default.__file__)
+    ipconf = os.path.join(ipcdir,'ipython_config.py')
+    #print 'conf:',ipconf # dbg
+    return ['--colors=NoColor', '--no-term-title','--no-banner',
+            '--config-file=%s' % ipconf, '--autocall=0', '--quick']
+    
+
+def ipexec(fname):
+    """Utility to call 'ipython filename'.
+
+    Starts IPython witha minimal and safe configuration to make startup as fast
+    as possible.
+
+    Note that this starts IPython in a subprocess!
+
+    Parameters
+    ----------
+    fname : str
+      Name of file to be executed (should have .py or .ipy extension).
+
+    Returns
+    -------
+    (stdout, stderr) of ipython subprocess.
+    """  
+    _ip = get_ipython()
+    test_dir = os.path.dirname(__file__)
+    full_fname = os.path.join(test_dir, fname)
+    ipython_cmd = platutils.find_cmd('ipython')
+    cmdargs = ' '.join(default_argv())
+    return genutils.getoutputerror('%s %s' % (ipython_cmd, full_fname))
+
+
+def ipexec_validate(fname, expected_out, expected_err=None):
+    """Utility to call 'ipython filename' and validate output/error.
+
+    This function raises an AssertionError if the validation fails.
+
+    Note that this starts IPython in a subprocess!
+
+    Parameters
+    ----------
+    fname : str
+      Name of the file to be executed (should have .py or .ipy extension).
+
+    expected_out : str
+      Expected stdout of the process.
+
+    Returns
+    -------
+    None
+    """
+
+    out, err = ipexec(fname)
+    nt.assert_equals(out.strip(), expected_out.strip())
+    if expected_err:
+        nt.assert_equals(err.strip(), expected_err.strip())