Add new testing support machinery with better parametric tests.

Also included are new tools for doctests with ipython syntax.

IPython/testing/ipunittest.py

@@ -0,0 +1,156 b''
+"""Experimental code for cleaner support of IPython syntax with unittest.
+
+In IPython up until 0.10, we've used very hacked up nose machinery for running
+tests with IPython special syntax, and this has proved to be extremely slow.
+This module provides decorators to try a different approach, stemming from a
+conversation Brian and I (FP) had about this problem Sept/09.
+
+The goal is to be able to easily write simple functions that can be seen by
+unittest as tests, and ultimately for these to support doctests with full
+IPython syntax.  Nose already offers this based on naming conventions and our
+hackish plugins, but we are seeking to move away from nose dependencies if
+possible.
+
+This module follows a different approach, based on decorators.
+
+- A decorator called @ipdoctest can mark any function as having a docstring
+  that should be viewed as a doctest, but after syntax conversion.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import re
+import sys
+import unittest
+from doctest import DocTestFinder, DocTestRunner, TestResults
+
+# Our own
+import nosepatch
+
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from _paramtestpy2 import ParametricTestCase
+else:
+    from _paramtestpy3 import ParametricTestCase
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+def count_failures(runner):
+    """Count number of failures in a doctest runner.
+
+    Code modeled after the summarize() method in doctest.
+    """
+    return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0 ]
+
+
+class IPython2PythonConverter(object):
+    """Convert IPython 'syntax' to valid Python.
+
+    Eventually this code may grow to be the full IPython syntax conversion
+    implementation, but for now it only does prompt convertion."""
+    
+    def __init__(self):
+        self.ps1 = re.compile(r'In\ \[\d+\]: ')
+        self.ps2 = re.compile(r'\ \ \ \.\.\.+: ')
+        self.out = re.compile(r'Out\[\d+\]: \s*?\n?')
+
+    def __call__(self, ds):
+        """Convert IPython prompts to python ones in a string."""
+        pyps1 = '>>> '
+        pyps2 = '... '
+        pyout = ''
+
+        dnew = ds
+        dnew = self.ps1.sub(pyps1, dnew)
+        dnew = self.ps2.sub(pyps2, dnew)
+        dnew = self.out.sub(pyout, dnew)
+        return dnew
+
+
+class Doc2UnitTester(object):
+    """Class whose instances act as a decorator for docstring testing.
+
+    In practice we're only likely to need one instance ever, made below (though
+    no attempt is made at turning it into a singleton, there is no need for
+    that).
+    """
+    def __init__(self, verbose=False):
+        """New decorator.
+
+        Parameters
+        ----------
+
+        verbose : boolean, optional (False)
+          Passed to the doctest finder and runner to control verbosity.
+        """
+        self.verbose = verbose
+        # We can reuse the same finder for all instances
+        self.finder = DocTestFinder(verbose=verbose, recurse=False)
+
+    def __call__(self, func):
+        """Use as a decorator: doctest a function's docstring as a unittest.
+        
+        This version runs normal doctests, but the idea is to make it later run
+        ipython syntax instead."""
+
+        # Capture the enclosing instance with a different name, so the new
+        # class below can see it without confusion regarding its own 'self'
+        # that will point to the test instance at runtime
+        d2u = self
+
+        # Rewrite the function's docstring to have python syntax
+        if func.__doc__ is not None:
+            func.__doc__ = ip2py(func.__doc__)
+
+        # Now, create a tester object that is a real unittest instance, so
+        # normal unittest machinery (or Nose, or Trial) can find it.
+        class Tester(unittest.TestCase):
+            def test(self):
+                # Make a new runner per function to be tested
+                runner = DocTestRunner(verbose=d2u.verbose)
+                map(runner.run, d2u.finder.find(func, func.__name__))
+                failed = count_failures(runner)
+                if failed:
+                    # Since we only looked at a single function's docstring,
+                    # failed should contain at most one item.  More than that
+                    # is a case we can't handle and should error out on
+                    if len(failed) > 1:
+                        err = "Invalid number of test results:" % failed
+                        raise ValueError(err)
+                    # Report a normal failure.
+                    self.fail('failed doctests: %s' % str(failed[0]))
+                    
+        # Rename it so test reports have the original signature.
+        Tester.__name__ = func.__name__
+        return Tester
+
+
+def ipdocstring(func):
+    """Change the function docstring via ip2py.
+    """
+    if func.__doc__ is not None:
+        func.__doc__ = ip2py(func.__doc__)
+    return func
+
+        
+# Make an instance of the classes for public use
+ipdoctest = Doc2UnitTester()
+ip2py = IPython2PythonConverter()

IPython/testing/tests/test_ipunittest.py

@@ -0,0 +1,122 b''
+"""Tests for IPyhton's test support utilities.
+
+These are decorators that allow standalone functions and docstrings to be seen
+as tests by unittest, replicating some of nose's functionality.  Additionally,
+IPython-syntax docstrings can be auto-converted to '>>>' so that ipython
+sessions can be copy-pasted as tests.
+
+This file can be run as a script, and it will call unittest.main().  We must
+check that it works with unittest as well as with nose...
+
+
+Notes:
+
+- Using nosetests --with-doctest --doctest-tests testfile.py
+  will find docstrings as tests wherever they are, even in methods.  But
+  if we use ipython syntax in the docstrings, they must be decorated with
+  @ipdocstring.  This is OK for test-only code, but not for user-facing
+  docstrings where we want to keep the ipython syntax.
+
+- Using nosetests --with-doctest file.py
+  also finds doctests if the file name doesn't have 'test' in it, because it is
+  treated like a normal module.  But if nose treats the file like a test file,
+  then for normal classes to be doctested the extra --doctest-tests is
+  necessary.
+
+- running this script with python (it has a __main__ section at the end) misses
+  one docstring test, the one embedded in the Foo object method.  Since our
+  approach relies on using decorators that create standalone TestCase
+  instances, it can only be used for functions, not for methods of objects.  
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython.testing.ipunittest import ipdoctest, ipdocstring
+
+#-----------------------------------------------------------------------------
+# Test classes and functions
+#-----------------------------------------------------------------------------
+@ipdoctest
+def simple_dt():
+    """
+    >>> print 1+1
+    2
+    """
+
+
+@ipdoctest
+def ipdt_flush():
+    """
+In [20]: print 1
+1
+
+In [26]: for i in range(10):
+   ....:     print i,
+   ....:     
+   ....:     
+0 1 2 3 4 5 6 7 8 9
+
+In [27]: 3+4
+Out[27]: 7
+"""
+
+
+@ipdoctest
+def ipdt_indented_test():
+    """
+    In [20]: print 1
+    1
+
+    In [26]: for i in range(10):
+       ....:     print i,
+       ....:     
+       ....:     
+    0 1 2 3 4 5 6 7 8 9
+
+    In [27]: 3+4
+    Out[27]: 7
+    """
+
+
+class Foo(object):
+    """For methods, the normal decorator doesn't work.
+
+    But rewriting the docstring with ip2py does, *but only if using nose
+    --with-doctest*.  Do we want to have that as a dependency?
+    """
+
+    @ipdocstring
+    def ipdt_method(self):
+        """
+        In [20]: print 1
+        2
+
+        In [26]: for i in range(10):
+           ....:     print i,
+           ....:     
+           ....:     
+        0 1 2 3 4 5 6 7 8 9
+
+        In [27]: 3+4
+        Out[27]: 7
+        """
+
+    def normaldt_method(self):
+        """
+        >>> print 1+1
+        2
+        """

IPython/kernel/core/tests/test_redirectors.py

@@ -21,13 +21,12 b' import os'
 
 from twisted.trial import unittest
 
-from IPython.testing import decorators_trial as dec
+from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class TestRedirector(unittest.TestCase):
 
     @dec.skip_win32

IPython/testing/decorators.py

@@ -10,27 +10,79 b' This module provides a set of useful decorators meant to be ready to use in'
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
+Included decorators:
+
+
+Lightweight testing that remains unittest-compatible.
+
+- @parametric, for parametric test support that is vastly easier to use than
+  nose's for debugging. With ours, if a test fails, the stack under inspection
+  is that of the test and not that of the test framework.
+
+- An @as_unittest decorator can be used to tag any normal parameter-less
+  function as a unittest TestCase.  Then, both nose and normal unittest will
+  recognize it as such.  This will make it easier to migrate away from Nose if
+  we ever need/want to while maintaining very lightweight tests.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009-2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 # Stdlib imports
 import inspect
 import sys
+import unittest
 
 # Third-party imports
 
-# This is Michele Simionato's decorator module, also kept verbatim.
+# This is Michele Simionato's decorator module, kept verbatim.
 from IPython.external.decorator import decorator, update_wrapper
 
+# Our own modules
+import nosepatch  # monkeypatch nose
+
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from _paramtestpy2 import parametric
+else:
+    from _paramtestpy3 import parametric
+
 # Grab the numpy-specific decorators which we keep in a file that we
-# occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
-# of numpy.testing.decorators.
-from decorators_numpy import *
+# occasionally update from upstream: decorators.py is a copy of
+# numpy.testing.decorators, we expose all of it here.
+from IPython.external.decorators import *
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
 
-##############################################################################
-# Local code begins
+# Simple example of the basic idea
+def as_unittest(func):
+    """Decorator to make a simple function into a normal test via unittest."""
+    class Tester(unittest.TestCase):
+        def test(self):
+            func()
+
+    Tester.__name__ = func.__name__
+
+    return Tester
 
 # Utility functions
 
@@ -51,21 +103,23 b' def apply_wrapper(wrapper,func):'
 def make_label_dec(label,ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
-    :Parameters:
+    Parameters
+    ----------
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
-    :Keywords:
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
-    :Returns:
+    Returns
+    -------
       A decorator.
 
-    :Examples:
+    Examples
+    --------
 
     A simple labeling decorator:
     >>> slow = make_label_dec('slow')
@@ -193,11 +247,13 b' def skipif(skip_condition, msg=None):'
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
-    :Parameters:
+    Parameters
+    ----------
       msg : string
         Optional message to be added.
 
-    :Returns:
+    Returns
+    -------
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.

IPython/testing/iptest.py

@@ -125,8 +125,6 b' def make_exclude():'
                              'test_asyncfrontendbase')),
         EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
         EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 
-                             'test_decorators_trial'))
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':

IPython/testing/parametric.py

@@ -1,5 +1,8 b''
 """Parametric testing on top of twisted.trial.unittest.
 
+XXX - It may be possbile to deprecate this in favor of the new, cleaner
+parametric code.  We just need to double-check that the new code doesn't clash
+with Twisted (we know it works with nose and unittest).
 """
 
 __all__ = ['parametric','Parametric']

IPython/testing/tests/test_decorators.py

@@ -5,13 +5,14 b''
 # Std lib
 import inspect
 import sys
+import unittest
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.testing import decorators as dec
-
+from IPython.testing.ipunittest import ParametricTestCase
 
 #-----------------------------------------------------------------------------
 # Utilities
@@ -41,6 +42,30 b' def getargspec(obj):'
 #-----------------------------------------------------------------------------
 # Testing functions
 
[e-mail]_unittest
+def trivial():
+    """A trivial test"""
+    pass
+
+# Some examples of parametric tests.
+
+def is_smaller(i,j):
+    assert i<j,"%s !< %s" % (i,j)
+
+class Tester(ParametricTestCase):
+
+    def test_parametric(self):
+        yield is_smaller(3, 4)
+        x, y = 1, 2
+        yield is_smaller(x, y)
+
[e-mail]
+def test_par_standalone():
+    yield is_smaller(3, 4)
+    x, y = 1, 2
+    yield is_smaller(x, y)
+
+
 @dec.skip
 def test_deliberately_broken():
     """A deliberately broken test - we want to skip this one."""
@@ -159,3 +184,36 b' def test_win32():'
 @dec.skip_osx
 def test_osx():
     nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")
+
+
+# Verify that the same decorators work for methods.
+# Note: this code is identical to that in test_decorators_trial, but that one
+# uses twisted's unittest, not the one from the stdlib, which we are using
+# here. While somewhat redundant, we want to check both with the stdlib and
+# with twisted, so the duplication is OK.
+class TestDecoratorsTrial(unittest.TestCase):
+    
+    @dec.skip()
+    def test_deliberately_broken(self):
+        """A deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip('Testing the skip decorator')
+    def test_deliberately_broken2(self):
+        """Another deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip_linux
+    def test_linux(self):
+        self.assertNotEquals(sys.platform, 'linux2',
+                             "This test can't run under linux")
+
+    @dec.skip_win32
+    def test_win32(self):
+        self.assertNotEquals(sys.platform, 'win32',
+                             "This test can't run under windows")
+
+    @dec.skip_osx
+    def test_osx(self):
+        self.assertNotEquals(sys.platform, 'darwin',
+                             "This test can't run under osx")

IPython/testing/tools.py

@@ -33,7 +33,6 b' import sys'
 import nose.tools as nt
 
 from IPython.utils import genutils
-from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Globals