Merging Fernando's fixes from his trunk-dev and fixing testing things.

I have merged his changes with a few modifications.  I have also
removed IPython.kernel from the list of packages that nose will test.

IPython/testing/plugin/test_ipdoctest.py

@@ -0,0 +1,94 b''
+"""Tests for the ipdoctest machinery itself.
+
+Note: in a file named test_X, functions whose only test is their docstring (as
+a doctest) and which have no test functionality of their own, should be called
+'doctest_foo' instead of 'test_foo', otherwise they get double-counted (the
+empty function call is counted as a test, which just inflates tests numbers
+artificially).
+"""
+
+def doctest_simple():
+    """ipdoctest must handle simple inputs
+    
+    In [1]: 1
+    Out[1]: 1
+
+    In [2]: print 1
+    1
+    """
+
+
+def doctest_run_builtins():
+    """Check that %run doesn't damage __builtins__ via a doctest.
+
+    This is similar to the test_run_builtins, but I want *both* forms of the
+    test to catch any possible glitches in our testing machinery, since that
+    modifies %run somewhat.  So for this, we have both a normal test (below)
+    and a doctest (this one).
+
+    In [1]: import tempfile
+
+    In [3]: f = tempfile.NamedTemporaryFile()
+
+    In [4]: f.write('pass\\n')
+
+    In [5]: f.flush()
+
+    In [7]: %run $f.name
+    """
+
+def doctest_multiline1():
+    """The ipdoctest machinery must handle multiline examples gracefully.
+
+    In [2]: for i in range(10):
+       ...:     print i,
+       ...:      
+    0 1 2 3 4 5 6 7 8 9
+    """
+
+    
+def doctest_multiline2():
+    """Multiline examples that define functions and print output.
+
+    In [7]: def f(x):
+       ...:     return x+1
+       ...: 
+
+    In [8]: f(1)
+    Out[8]: 2
+
+    In [9]: def g(x):
+       ...:     print 'x is:',x
+       ...:      
+
+    In [10]: g(1)
+    x is: 1
+
+    In [11]: g('hello')
+    x is: hello
+    """
+
+
+def doctest_multiline3():
+    """Multiline examples with blank lines.
+
+    In [12]: def h(x):
+       ....:     if x>1:
+       ....:         return x**2
+       ....:     # To leave a blank line in the input, you must mark it
+       ....:     # with a comment character:
+       ....:     #
+       ....:     # otherwise the doctest parser gets confused.
+       ....:     else:
+       ....:         return -1
+       ....:      
+
+    In [13]: h(5)
+    Out[13]: 25
+
+    In [14]: h(1)
+    Out[14]: -1
+
+    In [15]: h(0)
+    Out[15]: -1
+   """

IPython/testing/tools.py

@@ -0,0 +1,90 b''
+"""Generic testing tools that do NOT depend on Twisted.
+
+In particular, this module exposes a set of top-level assert* functions that
+can be used in place of nose.tools.assert* in method generators (the ones in
+nose can not, at least as of nose 0.10.4).
+
+Note: our testing package contains testing.util, which does depend on Twisted
+and provides utilities for tests that manage Deferreds.  All testing support
+tools that only depend on nose, IPython or the standard library should go here
+instead.
+
+
+Authors
+-------
+- Fernando Perez <[e-mail]>
+"""
+
+#*****************************************************************************
+#       Copyright (C) 2009 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+#-----------------------------------------------------------------------------
+# Required modules and packages
+#-----------------------------------------------------------------------------
+
+# Standard Python lib
+import os
+import sys
+
+# Third-party
+import nose.tools as nt
+
+# From this project
+from IPython.tools import utils
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Make a bunch of nose.tools assert wrappers that can be used in test
+# generators.  This will expose an assert* function for each one in nose.tools.
+
+_tpl = """
+def %(name)s(*a,**kw):
+    return nt.%(name)s(*a,**kw)
+"""
+
+for _x in [a for a in dir(nt) if a.startswith('assert')]:
+    exec _tpl % dict(name=_x)
+
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
+
+def full_path(startPath,files):
+    """Make full paths for all the listed files, based on startPath.
+
+    Only the base part of startPath is kept, since this routine is typically
+    used with a script's __file__ variable as startPath.  The base of startPath
+    is then prepended to all the listed files, forming the output list.
+
+    Parameters
+    ----------
+      startPath : string
+        Initial path to use as the base for the results.  This path is split
+      using os.path.split() and only its first component is kept.
+
+      files : string or list
+        One or more files.
+
+    Examples
+    --------
+
+    >>> full_path('/foo/bar.py',['a.txt','b.txt'])
+    ['/foo/a.txt', '/foo/b.txt']
+
+    >>> full_path('/foo',['a.txt','b.txt'])
+    ['/a.txt', '/b.txt']
+
+    If a single file is given, the output is still a list:
+    >>> full_path('/foo','a.txt')
+    ['/a.txt']
+    """
+
+    files = utils.list_strings(files)
+    base = os.path.split(startPath)[0]
+    return [ os.path.join(base,f) for f in files ]

IPython/Magic.py

@@ -1709,6 +1709,16 b' Currently the magic system has the following functions:\\n"""'
                     del prog_ns['__name__']
                     self.shell.user_ns.update(prog_ns)
         finally:
+            # It's a bit of a mystery why, but __builtins__ can change from
+            # being a module to becoming a dict missing some key data after
+            # %run.  As best I can see, this is NOT something IPython is doing
+            # at all, and similar problems have been reported before:
+            # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
+            # Since this seems to be done by the interpreter itself, the best
+            # we can do is to at least restore __builtins__ for the user on
+            # exit.
+            self.shell.user_ns['__builtins__'] = __builtin__
+            
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:

IPython/Prompts.py

@@ -126,9 +126,13 b' prompt_specials_color = {'
     # Just the prompt counter number, WITHOUT any coloring wrappers, so users
     # can get numbers displayed in whatever color they want.
     r'\N': '${self.cache.prompt_count}',
+
     # Prompt/history count, with the actual digits replaced by dots.  Used
     # mainly in continuation prompts (prompt_in2)
+    #r'\D': '${"."*len(str(self.cache.prompt_count))}',
+    # More robust form of the above expression, that uses __builtins__
     r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}',
+
     # Current working directory
     r'\w': '${os.getcwd()}',
     # Current time

IPython/kernel/core/notification.py

@@ -15,6 +15,8 b' __docformat__ = "restructuredtext en"'
 #  the file COPYING, distributed as part of this software.                    
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
 
 class NotificationCenter(object):
     """Synchronous notification center

IPython/kernel/engineservice.py

@@ -34,6 +34,9 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
+
 import os, sys, copy
 import cPickle as pickle
 from new import instancemethod

IPython/kernel/task.py

@@ -16,6 +16,9 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
+
 import copy, time
 from types import FunctionType
 

IPython/testing/iptest.py

@@ -167,7 +167,8 b' def make_runners():'
     else:
         top_mod.append('platutils_dummy.py')
 
-    top_pack = ['config','Extensions','frontend','gui','kernel',
+    # These are tested by nose, so skip IPython.kernel
+    top_pack = ['config','Extensions','frontend','gui',
                 'testing','tests','tools','UserConfig']
 
     modules  = ['IPython.%s' % m[:-3] for m in top_mod ]
@@ -176,6 +177,7 b' def make_runners():'
     # Make runners
     runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
 
+    # Test IPython.kernel using trial if twisted is installed
     try:
         import zope.interface
         import twisted

IPython/testing/plugin/ipdoctest.py

@@ -97,8 +97,18 b' def _run_ns_sync(self,arg_s,runner=None):'
     This is strictly needed for running doctests that call %run.
     """
 
-    finder = py_file_finder(_run_ns_sync.test_filename)
+    # When tests call %run directly (not via doctest) these function attributes
+    # are not set
+    try:
+        fname = _run_ns_sync.test_filename
+    except AttributeError:
+        fname = arg_s
+
+    finder = py_file_finder(fname)
     out = _ip.IP.magic_run_ori(arg_s,runner,finder)
+    
+    # Simliarly, there is no test_globs when a test is NOT a doctest
+    if hasattr(_run_ns_sync,'test_globs'):
     _run_ns_sync.test_globs.update(_ip.user_ns)
     return out
 
@@ -126,11 +136,20 b' class ipnsdict(dict):'
     def update(self,other):
         self._checkpoint()
         dict.update(self,other)
+
         # If '_' is in the namespace, python won't set it when executing code,
         # and we have examples that test it.  So we ensure that the namespace
         # is always 'clean' of it before it's used for test code execution.
         self.pop('_',None)
         
+        # The builtins namespace must *always* be the real __builtin__ module,
+        # else weird stuff happens.  The main ipython code does have provisions
+        # to ensure this after %run, but since in this class we do some
+        # aggressive low-level cleaning of the execution namespace, we need to
+        # correct for that ourselves, to ensure consitency with the 'real'
+        # ipython.
+        self['__builtins__'] = __builtin__
+        
 
 def start_ipython():
     """Start a global IPython shell, which we need for IPython-specific syntax.

IPython/tests/test_magic.py

@@ -6,12 +6,15 b' Needs to be run by nose (to make ipython session available).'
 # Standard library imports
 import os
 import sys
+import tempfile
+import types
 
 # Third-party imports
 import nose.tools as nt
 
 # From our own code
 from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
@@ -34,26 +37,6 b' def test_rehashx():'
     assert len(scoms) > 10
 
 
-def doctest_run_ns():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [11]: run tclass foo
-
-    In [12]: isinstance(f(),foo)
-    Out[12]: True
-    """
-
-    
-def doctest_run_ns2():
-    """Classes declared %run scripts must be instantiable afterwards.
-
-    In [4]: run tclass C-first_pass
-
-    In [5]: run tclass C-second_pass
-    tclass.py: deleting object: C-first_pass
-    """
-
-
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
@@ -149,3 +132,104 b' def doctest_refbug():'
     lowercased: hello
     lowercased: hello
     """
+
+#-----------------------------------------------------------------------------
+# Tests for %run
+#-----------------------------------------------------------------------------
+
+# %run is critical enough that it's a good idea to have a solid collection of
+# tests for it, some as doctests and some as normal tests.
+
+def doctest_run_ns():
+    """Classes declared %run scripts must be instantiable afterwards.
+
+    In [11]: run tclass foo
+
+    In [12]: isinstance(f(),foo)
+    Out[12]: True
+    """
+
+    
+def doctest_run_ns2():
+    """Classes declared %run scripts must be instantiable afterwards.
+
+    In [4]: run tclass C-first_pass
+
+    In [5]: run tclass C-second_pass
+    tclass.py: deleting object: C-first_pass
+    """
+
+def doctest_run_builtins():
+    """Check that %run doesn't damage __builtins__ via a doctest.
+
+    This is similar to the test_run_builtins, but I want *both* forms of the
+    test to catch any possible glitches in our testing machinery, since that
+    modifies %run somewhat.  So for this, we have both a normal test (below)
+    and a doctest (this one).
+
+    In [1]: import tempfile
+
+    In [2]: bid1 = id(__builtins__)
+
+    In [3]: f = tempfile.NamedTemporaryFile()
+
+    In [4]: f.write('pass\\n')
+
+    In [5]: f.flush()
+
+    In [6]: print 'B1:',type(__builtins__)
+    B1: <type 'module'>
+
+    In [7]: %run $f.name
+
+    In [8]: bid2 = id(__builtins__)
+
+    In [9]: print 'B2:',type(__builtins__)
+    B2: <type 'module'>
+
+    In [10]: bid1 == bid2
+    Out[10]: True
+    """
+
+# For some tests, it will be handy to organize them in a class with a common
+# setup that makes a temp file
+
+class TestMagicRun(object):
+
+    def setup(self):
+        """Make a valid python temp file."""
+        f = tempfile.NamedTemporaryFile()
+        f.write('pass\n')
+        f.flush()
+        self.tmpfile = f
+
+    def run_tmpfile(self):
+        _ip.magic('run %s' % self.tmpfile.name)
+
+    def test_builtins_id(self):
+        """Check that %run doesn't damage __builtins__ """
+
+        # Test that the id of __builtins__ is not modified by %run
+        bid1 = id(_ip.user_ns['__builtins__'])
+        self.run_tmpfile()
+        bid2 = id(_ip.user_ns['__builtins__'])
+        tt.assert_equals(bid1, bid2)
+
+    def test_builtins_type(self):
+        """Check that the type of __builtins__ doesn't change with %run.
+        
+        However, the above could pass if __builtins__ was already modified to
+        be a dict (it should be a module) by a previous use of %run.  So we
+        also check explicitly that it really is a module:
+        """
+        self.run_tmpfile()
+        tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
+
+    def test_prompts(self):
+        """Test that prompts correctly generate after %run"""
+        self.run_tmpfile()
+        p2 = str(_ip.IP.outputcache.prompt2).strip()
+        nt.assert_equals(p2[:3], '...')
+
+    def teardown(self):
+        self.tmpfile.close()