Massive amount of work to improve the test suite, restores doctests.

After Brian's comments, I realized that our test machinery was NOT in
reality running all the ipython-syntax doctests we have.  This is now fixed.
The test suite isn't completely passing, but this commit is for the
underlying machinery.  I will now work on fixing as many broken tests as I
can.

Fixes https://bugs.launchpad.net/ipython/+bug/505071

IPython/core/tests/test_run.py

@@ -0,0 +1,185 b''
+"""Tests for code execution (%run and related), which is particularly tricky.
+
+Because of how %run manages namespaces, and the fact that we are trying here to
+verify subtle object deletion and reference counting issues, the %run tests
+will be kept in this separate file.  This makes it easier to aggregate in one
+place the tricks needed to handle it; most other magics are much easier to test
+and we do so in a common test_magic file.
+"""
+from __future__ import absolute_import
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import os
+import sys
+import tempfile
+
+# third-party
+import nose.tools as nt
+
+# our own
+from IPython.utils.platutils import find_cmd
+from IPython.utils import genutils
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
+def doctest_refbug():
+    """Very nasty problem with references held by multiple runs of a script.
+    See: https://bugs.launchpad.net/ipython/+bug/269966
+
+    In [1]: _ip.clear_main_mod_cache()
+    # random
+    
+    In [2]: %run refbug
+
+    In [3]: call_f()
+    lowercased: hello
+
+    In [4]: %run refbug
+
+    In [5]: call_f()
+    lowercased: hello
+    lowercased: hello
+    """
+
+
+def doctest_run_builtins():
+    r"""Check that %run doesn't damage __builtins__.
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: fname = tempfile.mkstemp('.py')[1]
+
+    In [3]: f = open(fname,'w')
+
+    In [4]: f.write('pass\n')
+
+    In [5]: f.flush()
+
+    In [6]: t1 = type(__builtins__)
+
+    In [7]: %run "$fname"
+
+    In [7]: f.close()
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: t2 = type(__builtins__)
+
+    In [10]: t1 == t2
+    Out[10]: True
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+
+    In [12]: try:
+       ....:     os.unlink(fname)
+       ....: except:
+       ....:     pass
+       ....: 
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TempFileMixin(object):
+    def mktmp(self, src, ext='.py'):
+        """Make a valid python temp file."""
+        fname, f = tt.temp_pyfile(src, ext)
+        self.tmpfile = f
+        self.fname = fname
+
+    def teardown(self):
+        self.tmpfile.close()
+        try:
+            os.unlink(self.fname)
+        except:
+            # On Windows, even though we close the file, we still can't delete
+            # it.  I have no clue why
+            pass
+
+
+class TestMagicRunPass(TempFileMixin):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        self.mktmp('pass\n')
+
+    def run_tmpfile(self):
+        _ip = get_ipython()
+        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
+        # See below and ticket https://bugs.launchpad.net/bugs/366353
+        _ip.magic('run "%s"' % self.fname)
+
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+        _ip = get_ipython()
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        _ip = get_ipython()
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        _ip = get_ipython()
+        p2 = str(_ip.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+
+class TestMagicRunSimple(TempFileMixin):
+
+    def test_simpledef(self):
+        """Test that simple class definitions work."""
+        src = ("class foo: pass\n"
+               "def f(): return foo()")
+        self.mktmp(src)
+        _ip.magic('run "%s"' % self.fname)
+        _ip.runlines('t = isinstance(f(), foo)')
+        nt.assert_true(_ip.user_ns['t'])
+        
+    def test_obj_del(self):
+        """Test that object's __del__ methods are called on exit."""
+        
+        # This test is known to fail on win32.
+        # See ticket https://bugs.launchpad.net/bugs/366334
+        src = ("class A(object):\n"
+               "    def __del__(self):\n"
+               "        print 'object A deleted'\n"
+               "a = A()\n")
+        self.mktmp(src)
+        tt.ipexec_validate(self.fname, 'object A deleted')
+
+    def test_tclass(self):
+        mydir = os.path.dirname(__file__)
+        tc = os.path.join(mydir, 'tclass')
+        src = ("%%run '%s' C-first\n"
+               "%%run '%s' C-second\n") % (tc, tc)
+        self.mktmp(src, '.ipy')
+        out = """\
+ARGV 1-: ['C-first']
+ARGV 1-: ['C-second']
+tclass.py: deleting object: C-first
+"""
+        tt.ipexec_validate(self.fname, out)

IPython/core/magic.py

@@ -3506,6 +3506,7 b' Defaulting color scheme to \'NoColor\'"""'
         """Reload an IPython extension by its module name."""
         self.reload_extension(module_str)
 
+    @testdec.skip_doctest
     def magic_install_profiles(self, s):
         """Install the default IPython profiles into the .ipython dir.
 

IPython/core/prefilter.py

@@ -416,7 +416,7 b' class PrefilterManager(Component):'
         # print "prefiltered line: %r" % prefiltered
         return prefiltered
 
-    def prefilter_lines(self, lines, continue_prompt):
+    def prefilter_lines(self, lines, continue_prompt=False):
         """Prefilter multiple input lines of text.
 
         This is the main entry point for prefiltering multiple lines of

IPython/core/tests/tclass.py

@@ -1,16 +1,12 b''
-"""Simple script to instantiate a class for testing %run"""
+"""Simple script to be run *twice*, to check reference counting bugs.
 
-import sys
-
-# An external test will check that calls to f() work after %run
-class foo: pass
+See test_run for details."""
 
-def f():
-    return foo()
+import sys
 
-# We also want to ensure that while objects remain available for immediate
-# access, objects from *previous* runs of the same script get collected, to
-# avoid accumulating massive amounts of old references.
+# We want to ensure that while objects remain available for immediate access,
+# objects from *previous* runs of the same script get collected, to avoid
+# accumulating massive amounts of old references.
 class C(object):
     def __init__(self,name):
         self.name = name
@@ -18,6 +14,7 b' class C(object):'
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
 
+
 try:
     name = sys.argv[1]
 except IndexError:
@@ -25,3 +22,8 b' except IndexError:'
 else:
     if name.startswith('C'):
         c = C(name)
+
+#print >> sys.stderr, "ARGV:", sys.argv  # dbg
+# This print statement is NOT debugging, we're making the check on a completely
+# separate process so we verify by capturing stdout.
+print 'ARGV 1-:', sys.argv[1:]

IPython/core/tests/test_magic.py

@@ -2,21 +2,31 b''
 
 Needs to be run by nose (to make ipython session available).
 """
+from __future__ import absolute_import
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
+# third-party
 import nose.tools as nt
 
+# our own
+from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
+#-----------------------------------------------------------------------------
 
 def test_rehashx():
     # clear up everything
@@ -63,17 +73,6 b' def doctest_hist_r():'
     hist -n -r 2  # random
     """
 
-# This test is known to fail on win32.
-# See ticket https://bugs.launchpad.net/bugs/366334
-def test_obj_del():
-    _ip = get_ipython()
-    """Test that object's __del__ methods are called on exit."""
-    test_dir = os.path.dirname(__file__)
-    del_file = os.path.join(test_dir,'obj_del.py')
-    ipython_cmd = find_cmd('ipython')
-    out = _ip.getoutput('%s %s' % (ipython_cmd, del_file))
-    nt.assert_equals(out,'obj_del.py: object A deleted')
-
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
@@ -113,161 +112,6 b' def test_numpy_clear_array_undec():'
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
[e-mail]()
-def test_fail_dec(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]('This one shouldn not run')
-def test_fail_dec2(*a,**k):
-    yield nt.assert_true, False
-
[e-mail]
-def test_fail_dec3(*a,**k):
-    yield nt.assert_true, False
-
-
-def doctest_refbug():
-    """Very nasty problem with references held by multiple runs of a script.
-    See: https://bugs.launchpad.net/ipython/+bug/269966
-
-    In [1]: _ip.clear_main_mod_cache()
-    
-    In [2]: run refbug
-
-    In [3]: call_f()
-    lowercased: hello
-
-    In [4]: run refbug
-
-    In [5]: call_f()
-    lowercased: hello
-    lowercased: hello
-    """
-
-#-----------------------------------------------------------------------------
-# Tests for %run
-#-----------------------------------------------------------------------------
-
-# %run is critical enough that it's a good idea to have a solid collection of
-# tests for it, some as doctests and some as normal tests.
-
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass foo
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [4]: run tclass C-first_pass
-
-    In [5]: run tclass C-second_pass
-    tclass.py: deleting object: C-first_pass
-    """
-
-def doctest_run_builtins():
-    """Check that %run doesn't damage __builtins__ via a doctest.
-
-    This is similar to the test_run_builtins, but I want *both* forms of the
-    test to catch any possible glitches in our testing machinery, since that
-    modifies %run somewhat.  So for this, we have both a normal test (below)
-    and a doctest (this one).
-
-    In [1]: import tempfile
-
-    In [2]: bid1 = id(__builtins__)
-
-    In [3]: fname = tempfile.mkstemp()[1]
-
-    In [3]: f = open(fname,'w')
-
-    In [4]: f.write('pass\\n')
-
-    In [5]: f.flush()
-
-    In [6]: print type(__builtins__)
-    <type 'module'>
-
-    In [7]: %run "$fname"
-
-    In [7]: f.close()
-
-    In [8]: bid2 = id(__builtins__)
-
-    In [9]: print type(__builtins__)
-    <type 'module'>
-
-    In [10]: bid1 == bid2
-    Out[10]: True
-
-    In [12]: try:
-       ....:     os.unlink(fname)
-       ....: except:
-       ....:     pass
-       ....: 
-    """
-
-# For some tests, it will be handy to organize them in a class with a common
-# setup that makes a temp file
-
-class TestMagicRun(object):
-
-    def setup(self):
-        """Make a valid python temp file."""
-        fname = tempfile.mkstemp('.py')[1]
-        f = open(fname,'w')
-        f.write('pass\n')
-        f.flush()
-        self.tmpfile = f
-        self.fname = fname
-
-    def run_tmpfile(self):
-        _ip = get_ipython()
-        # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
-        # See below and ticket https://bugs.launchpad.net/bugs/366353
-        _ip.magic('run "%s"' % self.fname)
-
-    def test_builtins_id(self):
-        """Check that %run doesn't damage __builtins__ """
-        _ip = get_ipython()
-        # Test that the id of __builtins__ is not modified by %run
-        bid1 = id(_ip.user_ns['__builtins__'])
-        self.run_tmpfile()
-        bid2 = id(_ip.user_ns['__builtins__'])
-        tt.assert_equals(bid1, bid2)
-
-    def test_builtins_type(self):
-        """Check that the type of __builtins__ doesn't change with %run.
-        
-        However, the above could pass if __builtins__ was already modified to
-        be a dict (it should be a module) by a previous use of %run.  So we
-        also check explicitly that it really is a module:
-        """
-        _ip = get_ipython()
-        self.run_tmpfile()
-        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
-
-    def test_prompts(self):
-        """Test that prompts correctly generate after %run"""
-        self.run_tmpfile()
-        _ip = get_ipython()
-        p2 = str(_ip.outputcache.prompt2).strip()
-        nt.assert_equals(p2[:3], '...')
-
-    def teardown(self):
-        self.tmpfile.close()
-        try:
-            os.unlink(self.fname)
-        except:
-            # On Windows, even though we close the file, we still can't delete
-            # it.  I have no clue why
-            pass
-
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():

IPython/testing/decorators.py

@@ -64,6 +64,9 b" if sys.version[0]=='2':"
 else:
     from _paramtestpy3 import parametric
 
+# Expose the unittest-driven decorators
+from ipunittest import ipdoctest, ipdocstring
+
 # Grab the numpy-specific decorators which we keep in a file that we
 # occasionally update from upstream: decorators.py is a copy of
 # numpy.testing.decorators, we expose all of it here.

IPython/testing/iptest.py

@@ -16,6 +16,8 b' For now, this script requires that both nose and twisted are installed. This'
 will change in the future.
 """
 
+from __future__ import absolute_import
+
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
@@ -34,6 +36,8 b' from nose.core import TestProgram'
 
 from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, FindCmdError
+from . import globalipapp
+from .plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
@@ -76,7 +80,11 b' def make_exclude():'
     # cause testing problems.  We should strive to minimize the number of
     # skipped modules, since this means untested code.  As the testing
     # machinery solidifies, this list should eventually become empty.
-    EXCLUDE = [pjoin('IPython', 'external'),
+
+    # Note that these exclusions only mean that the docstrings are not analyzed
+    # for examples to be run as tests, if there are other test functions in
+    # those modules, they do get run.
+    exclusions = [pjoin('IPython', 'external'),
                pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
                pjoin('IPython_doctest_plugin'),
                pjoin('IPython', 'quarantine'),
@@ -88,58 +96,58 b' def make_exclude():'
                ]
 
     if not have_wx:
-        EXCLUDE.append(pjoin('IPython', 'gui'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookwx'))
+        exclusions.append(pjoin('IPython', 'gui'))
+        exclusions.append(pjoin('IPython', 'frontend', 'wx'))
+        exclusions.append(pjoin('IPython', 'lib', 'inputhookwx'))
 
     if not have_gtk or not have_gobject:
-        EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookgtk'))
+        exclusions.append(pjoin('IPython', 'lib', 'inputhookgtk'))
 
     if not have_wx_aui:
-        EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+        exclusions.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
 
     if not have_objc:
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
+        exclusions.append(pjoin('IPython', 'frontend', 'cocoa'))
 
     if not sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
+        exclusions.append(pjoin('IPython', 'utils', 'platutils_win32'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
+        exclusions.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
+        exclusions.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
 
     if not os.name == 'posix':
-        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
+        exclusions.append(pjoin('IPython', 'utils', 'platutils_posix'))
 
     if not have_pexpect:
-        EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
+        exclusions.append(pjoin('IPython', 'scripts', 'irunner'))
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have_twisted and have_zi and have_foolscap):
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
+        exclusions.append(pjoin('IPython', 'frontend', 'frontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_linefrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 
+        exclusions.append(pjoin('IPython', 'frontend', 'tests', 
                              'test_frontendbase'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_prefilterfrontend'))
-        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+        exclusions.append(pjoin('IPython', 'frontend', 'tests',
                              'test_asyncfrontendbase')),
-        EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
+        exclusions.append(pjoin('IPython', 'testing', 'parametric'))
+        exclusions.append(pjoin('IPython', 'testing', 'util'))
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
-        EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+        exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
-    return EXCLUDE
+    return exclusions
 
 
 #-----------------------------------------------------------------------------
@@ -163,16 +171,16 b' class IPTester(object):'
         if runner == 'iptest':
             # Find our own 'iptest' script OS-level entry point
             try:
-                iptest_path = find_cmd('iptest')
+                iptest_path = os.path.abspath(find_cmd('iptest'))
             except FindCmdError:
                 # Script not installed (may be the case for testing situations
                 # that are running from a source tree only), pull from internal
                 # path:
                 iptest_path = pjoin(genutils.get_ipython_package_dir(),
                                     'scripts','iptest')
-            self.runner = [iptest_path,'-v']
+            self.runner = ['python', iptest_path, '-v']
         else:
-            self.runner = [find_cmd('trial')]
+            self.runner = ['python', os.path.abspath(find_cmd('trial'))]
         if params is None:
             params = []
         if isinstance(params,str):
@@ -238,11 +246,13 b' def make_runners():'
     nose_packages = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils']
     trial_packages = ['kernel']
-    #trial_packages = []  # dbg 
 
     if have_wx:
         nose_packages.append('gui')
 
+    #nose_packages = ['core']  # dbg
+    #trial_packages = []  # dbg 
+
     nose_packages = ['IPython.%s' % m for m in nose_packages ]
     trial_packages = ['IPython.%s' % m for m in trial_packages ]
 
@@ -268,16 +278,15 b' def run_iptest():'
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
-    argv = sys.argv + [ 
-                        # Loading ipdoctest causes problems with Twisted.
-                        # I am removing this as a temporary fix to get the 
-                        # test suite back into working shape.  Our nose
-                        # plugin needs to be gone through with a fine
-                        # toothed comb to find what is causing the problem.
-                        # '--with-ipdoctest',
-                        # '--ipdoctest-tests','--ipdoctest-extension=txt',
-                        # '--detailed-errors',
-                       
+    argv = sys.argv + [ '--detailed-errors',
+                        # Loading ipdoctest causes problems with Twisted, but
+                        # our test suite runner now separates things and runs
+                        # all Twisted tests with trial.
+                        '--with-ipdoctest',
+                        '--ipdoctest-tests','--ipdoctest-extension=txt',
+                        
+                        #'-x','-s',  # dbg
+                        
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
@@ -300,17 +309,18 b' def run_iptest():'
     if not has_tests:
         argv.append('IPython')
 
-    # Construct list of plugins, omitting the existing doctest plugin, which
-    # ours replaces (and extends).
-    EXCLUDE = make_exclude()
-    plugins = []
-    # plugins = [IPythonDoctest(EXCLUDE)]
+    ## # Construct list of plugins, omitting the existing doctest plugin, which
+    ## # ours replaces (and extends).
+    plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
+    # We need a global ipython running in this process
+    globalipapp.start_ipython()
+    # Now nose can run
     TestProgram(argv=argv,plugins=plugins)
 
 

IPython/testing/ipunittest.py

@@ -22,6 +22,8 b' Authors'
 - Fernando Perez <[e-mail]>
 """
 
+from __future__ import absolute_import
+
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
@@ -40,14 +42,16 b' import sys'
 import unittest
 from doctest import DocTestFinder, DocTestRunner, TestResults
 
-# Our own
-import nosepatch
+# Our own, a nose monkeypatch
+from . import nosepatch
 
 # We already have python3-compliant code for parametric tests
 if sys.version[0]=='2':
-    from _paramtestpy2 import ParametricTestCase
+    from ._paramtestpy2 import ParametricTestCase
 else:
-    from _paramtestpy3 import ParametricTestCase
+    from ._paramtestpy3 import ParametricTestCase
+
+from . import globalipapp
 
 #-----------------------------------------------------------------------------
 # Classes and functions
@@ -68,9 +72,13 b' class IPython2PythonConverter(object):'
     implementation, but for now it only does prompt convertion."""
     
     def __init__(self):
-        self.ps1 = re.compile(r'In\ \[\d+\]: ')
-        self.ps2 = re.compile(r'\ \ \ \.\.\.+: ')
-        self.out = re.compile(r'Out\[\d+\]: \s*?\n?')
+        self.rps1 = re.compile(r'In\ \[\d+\]: ')
+        self.rps2 = re.compile(r'\ \ \ \.\.\.+: ')
+        self.rout = re.compile(r'Out\[\d+\]: \s*?\n?')
+        self.pyps1 = '>>> '
+        self.pyps2 = '... '
+        self.rpyps1 = re.compile ('(\s*%s)(.*)$' % self.pyps1)
+        self.rpyps2 = re.compile ('(\s*%s)(.*)$' % self.pyps2)
 
     def __call__(self, ds):
         """Convert IPython prompts to python ones in a string."""
@@ -79,10 +87,34 b' class IPython2PythonConverter(object):'
         pyout = ''
 
         dnew = ds
-        dnew = self.ps1.sub(pyps1, dnew)
-        dnew = self.ps2.sub(pyps2, dnew)
-        dnew = self.out.sub(pyout, dnew)
-        return dnew
+        dnew = self.rps1.sub(pyps1, dnew)
+        dnew = self.rps2.sub(pyps2, dnew)
+        dnew = self.rout.sub(pyout, dnew)
+        ip = globalipapp.get_ipython()
+
+        # Convert input IPython source into valid Python.
+        out = []
+        newline = out.append
+        for line in dnew.splitlines():
+
+            mps1 = self.rpyps1.match(line)
+            if mps1 is not None:
+                prompt, text = mps1.groups()
+                newline(prompt+ip.prefilter(text, False))
+                continue
+
+            mps2 = self.rpyps2.match(line)
+            if mps2 is not None:
+                prompt, text = mps2.groups()
+                newline(prompt+ip.prefilter(text, True))
+                continue
+            
+            newline(line)
+        newline('')  # ensure a closing newline, needed by doctest
+        #print "PYSRC:", '\n'.join(out)  # dbg
+        return '\n'.join(out)
+
+    #return dnew
 
 
 class Doc2UnitTester(object):

IPython/testing/plugin/ipdoctest.py

@@ -49,183 +49,14 b' from nose.util import anyp, getpackage, test_address, resolve_name, tolist'
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
+#-----------------------------------------------------------------------------
52