Add new testing support machinery with better parametric tests.

Also included are new tools for doctests with ipython syntax.

IPython/testing/ipunittest.py

@@ -0,0 +1,156 b''
+"""Experimental code for cleaner support of IPython syntax with unittest.
+
+In IPython up until 0.10, we've used very hacked up nose machinery for running
+tests with IPython special syntax, and this has proved to be extremely slow.
+This module provides decorators to try a different approach, stemming from a
+conversation Brian and I (FP) had about this problem Sept/09.
+
+The goal is to be able to easily write simple functions that can be seen by
+unittest as tests, and ultimately for these to support doctests with full
+IPython syntax.  Nose already offers this based on naming conventions and our
+hackish plugins, but we are seeking to move away from nose dependencies if
+possible.
+
+This module follows a different approach, based on decorators.
+
+- A decorator called @ipdoctest can mark any function as having a docstring
+  that should be viewed as a doctest, but after syntax conversion.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib
+import re
+import sys
+import unittest
+from doctest import DocTestFinder, DocTestRunner, TestResults
+
+# Our own
+import nosepatch
+
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from _paramtestpy2 import ParametricTestCase
+else:
+    from _paramtestpy3 import ParametricTestCase
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+def count_failures(runner):
+    """Count number of failures in a doctest runner.
+
+    Code modeled after the summarize() method in doctest.
+    """
+    return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0 ]
+
+
+class IPython2PythonConverter(object):
+    """Convert IPython 'syntax' to valid Python.
+
+    Eventually this code may grow to be the full IPython syntax conversion
+    implementation, but for now it only does prompt convertion."""
+    
+    def __init__(self):
+        self.ps1 = re.compile(r'In\ \[\d+\]: ')
+        self.ps2 = re.compile(r'\ \ \ \.\.\.+: ')
+        self.out = re.compile(r'Out\[\d+\]: \s*?\n?')
+
+    def __call__(self, ds):
+        """Convert IPython prompts to python ones in a string."""
+        pyps1 = '>>> '
+        pyps2 = '... '
+        pyout = ''
+
+        dnew = ds
+        dnew = self.ps1.sub(pyps1, dnew)
+        dnew = self.ps2.sub(pyps2, dnew)
+        dnew = self.out.sub(pyout, dnew)
+        return dnew
+
+
+class Doc2UnitTester(object):
+    """Class whose instances act as a decorator for docstring testing.
+
+    In practice we're only likely to need one instance ever, made below (though
+    no attempt is made at turning it into a singleton, there is no need for
+    that).
+    """
+    def __init__(self, verbose=False):
+        """New decorator.
+
+        Parameters
+        ----------
+
+        verbose : boolean, optional (False)
+          Passed to the doctest finder and runner to control verbosity.
+        """
+        self.verbose = verbose
+        # We can reuse the same finder for all instances
+        self.finder = DocTestFinder(verbose=verbose, recurse=False)
+
+    def __call__(self, func):
+        """Use as a decorator: doctest a function's docstring as a unittest.
+        
+        This version runs normal doctests, but the idea is to make it later run
+        ipython syntax instead."""
+
+        # Capture the enclosing instance with a different name, so the new
+        # class below can see it without confusion regarding its own 'self'
+        # that will point to the test instance at runtime
+        d2u = self
+
+        # Rewrite the function's docstring to have python syntax
+        if func.__doc__ is not None:
+            func.__doc__ = ip2py(func.__doc__)
+
+        # Now, create a tester object that is a real unittest instance, so
+        # normal unittest machinery (or Nose, or Trial) can find it.
+        class Tester(unittest.TestCase):
+            def test(self):
+                # Make a new runner per function to be tested
+                runner = DocTestRunner(verbose=d2u.verbose)
+                map(runner.run, d2u.finder.find(func, func.__name__))
+                failed = count_failures(runner)
+                if failed:
+                    # Since we only looked at a single function's docstring,
+                    # failed should contain at most one item.  More than that
+                    # is a case we can't handle and should error out on
+                    if len(failed) > 1:
+                        err = "Invalid number of test results:" % failed
+                        raise ValueError(err)
+                    # Report a normal failure.
+                    self.fail('failed doctests: %s' % str(failed[0]))
+                    
+        # Rename it so test reports have the original signature.
+        Tester.__name__ = func.__name__
+        return Tester
+
+
+def ipdocstring(func):
+    """Change the function docstring via ip2py.
+    """
+    if func.__doc__ is not None:
+        func.__doc__ = ip2py(func.__doc__)
+    return func
+
+        
+# Make an instance of the classes for public use
+ipdoctest = Doc2UnitTester()
+ip2py = IPython2PythonConverter()

IPython/testing/tests/test_ipunittest.py

@@ -0,0 +1,122 b''
+"""Tests for IPyhton's test support utilities.
+
+These are decorators that allow standalone functions and docstrings to be seen
+as tests by unittest, replicating some of nose's functionality.  Additionally,
+IPython-syntax docstrings can be auto-converted to '>>>' so that ipython
+sessions can be copy-pasted as tests.
+
+This file can be run as a script, and it will call unittest.main().  We must
+check that it works with unittest as well as with nose...
+
+
+Notes:
+
+- Using nosetests --with-doctest --doctest-tests testfile.py
+  will find docstrings as tests wherever they are, even in methods.  But
+  if we use ipython syntax in the docstrings, they must be decorated with
+  @ipdocstring.  This is OK for test-only code, but not for user-facing
+  docstrings where we want to keep the ipython syntax.
+
+- Using nosetests --with-doctest file.py
+  also finds doctests if the file name doesn't have 'test' in it, because it is
+  treated like a normal module.  But if nose treats the file like a test file,
+  then for normal classes to be doctested the extra --doctest-tests is
+  necessary.
+
+- running this script with python (it has a __main__ section at the end) misses
+  one docstring test, the one embedded in the Foo object method.  Since our
+  approach relies on using decorators that create standalone TestCase
+  instances, it can only be used for functions, not for methods of objects.  
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython.testing.ipunittest import ipdoctest, ipdocstring
+
+#-----------------------------------------------------------------------------
+# Test classes and functions
+#-----------------------------------------------------------------------------
+@ipdoctest
+def simple_dt():
+    """
+    >>> print 1+1
+    2
+    """
+
+
+@ipdoctest
+def ipdt_flush():
+    """
+In [20]: print 1
+1
+
+In [26]: for i in range(10):
+   ....:     print i,
+   ....:     
+   ....:     
+0 1 2 3 4 5 6 7 8 9
+
+In [27]: 3+4
+Out[27]: 7
+"""
+
+
+@ipdoctest
+def ipdt_indented_test():
+    """
+    In [20]: print 1
+    1
+
+    In [26]: for i in range(10):
+       ....:     print i,
+       ....:     
+       ....:     
+    0 1 2 3 4 5 6 7 8 9
+
+    In [27]: 3+4
+    Out[27]: 7
+    """
+
+
+class Foo(object):
+    """For methods, the normal decorator doesn't work.
+
+    But rewriting the docstring with ip2py does, *but only if using nose
+    --with-doctest*.  Do we want to have that as a dependency?
+    """
+
+    @ipdocstring
+    def ipdt_method(self):
+        """
+        In [20]: print 1
+        2
+
+        In [26]: for i in range(10):
+           ....:     print i,
+           ....:     
+           ....:     
+        0 1 2 3 4 5 6 7 8 9
+
+        In [27]: 3+4
+        Out[27]: 7
+        """
+
+    def normaldt_method(self):
+        """
+        >>> print 1+1
+        2
+        """

IPython/kernel/core/tests/test_redirectors.py

@@ -1,78 +1,77 b''
 # encoding: utf-8
 """
 Test the output capture at the OS level, using file descriptors.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Tell nose to skip this module
 __test__ = {}
 
 from cStringIO import StringIO
 import os
 
 from twisted.trial import unittest
 
-from IPython.testing import decorators_trial as dec
+from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
-
 class TestRedirector(unittest.TestCase):
 
     @dec.skip_win32
     def test_redirector(self):
         """Checks that the redirector can be used to do synchronous capture.
         """
         from IPython.kernel.core.fd_redirector import FDRedirector
         r = FDRedirector()
         out = StringIO()
         try:
             r.start()
             for i in range(10):
                 os.system('echo %ic' % i)
                 print >>out, r.getvalue(),
                 print >>out, i
         except:
             r.stop()
             raise
         r.stop()
         result1 = out.getvalue()
         result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
         self.assertEquals(result1, result2)
 
     @dec.skip_win32
     def test_redirector_output_trap(self):
         """Check the greedy trapping behavior of the traps.
         
         This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
         """
         from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
         out = StringIO()
         trap = RedirectorOutputTrap(out.write, out.write)
         try:
             trap.set()
             for i in range(10):
                 os.system('echo %ic' % i)
                 print "%ip" % i
                 print >>out, i
         except:
             trap.unset()
             raise
         trap.unset()
         result1 = out.getvalue()
         result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
         self.assertEquals(result1, result2)
 

IPython/testing/decorators.py

@@ -1,254 +1,310 b''
 """Decorators for labeling test objects.
 
 Decorators that merely return a modified version of the original function
 object are straightforward.  Decorators that return a new function object need
 to use nose.tools.make_decorator(original_function)(decorator) in returning the
 decorator, in order to preserve metadata such as function name, setup and
 teardown functions and so on - see nose.tools for more information.
 
 This module provides a set of useful decorators meant to be ready to use in
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
+Included decorators:
+
+
+Lightweight testing that remains unittest-compatible.
+
+- @parametric, for parametric test support that is vastly easier to use than
+  nose's for debugging. With ours, if a test fails, the stack under inspection
+  is that of the test and not that of the test framework.
+
+- An @as_unittest decorator can be used to tag any normal parameter-less
+  function as a unittest TestCase.  Then, both nose and normal unittest will
+  recognize it as such.  This will make it easier to migrate away from Nose if
+  we ever need/want to while maintaining very lightweight tests.
+
 NOTE: This file contains IPython-specific decorators and imports the
 numpy.testing.decorators file, which we've copied verbatim.  Any of our own
 code will be added at the bottom if we end up extending this.
+
+Authors
+-------
+
+- Fernando Perez <[e-mail]>
 """
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2009-2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
 # Stdlib imports
 import inspect
 import sys
+import unittest
 
 # Third-party imports
 
-# This is Michele Simionato's decorator module, also kept verbatim.
+# This is Michele Simionato's decorator module, kept verbatim.
 from IPython.external.decorator import decorator, update_wrapper
 
+# Our own modules
+import nosepatch  # monkeypatch nose
+
+# We already have python3-compliant code for parametric tests
+if sys.version[0]=='2':
+    from _paramtestpy2 import parametric
+else:
+    from _paramtestpy3 import parametric
+
 # Grab the numpy-specific decorators which we keep in a file that we
-# occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
-# of numpy.testing.decorators.
-from decorators_numpy import *
+# occasionally update from upstream: decorators.py is a copy of
+# numpy.testing.decorators, we expose all of it here.
+from IPython.external.decorators import *
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+# Simple example of the basic idea
+def as_unittest(func):
+    """Decorator to make a simple function into a normal test via unittest."""
+    class Tester(unittest.TestCase):
+        def test(self):
+            func()
 
-##############################################################################
-# Local code begins
+    Tester.__name__ = func.__name__
+
+    return Tester
 
 # Utility functions
 
 def apply_wrapper(wrapper,func):
     """Apply a wrapper to a function for decoration.
 
     This mixes Michele Simionato's decorator tool with nose's make_decorator,
     to apply a wrapper in a decorator so that all nose attributes, as well as
     function signature and other properties, survive the decoration cleanly.
     This will ensure that wrapped functions can still be well introspected via
     IPython, for example.
     """
     import nose.tools
 
     return decorator(wrapper,nose.tools.make_decorator(func)(wrapper))
 
 
 def make_label_dec(label,ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
-    :Parameters:
+    Parameters
+    ----------
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
-    :Keywords:
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
-    :Returns:
+    Returns
+    -------
       A decorator.
 
-    :Examples:
+    Examples
+    --------
 
     A simple labeling decorator:
     >>> slow = make_label_dec('slow')
     >>> print slow.__doc__
     Labels a test as 'slow'.
 
     And one that uses multiple labels and a custom docstring:
     >>> rare = make_label_dec(['slow','hard'],
     ... "Mix labels 'slow' and 'hard' for rare tests.")
     >>> print rare.__doc__
     Mix labels 'slow' and 'hard' for rare tests.
 
     Now, let's test using this one:
     >>> @rare
     ... def f(): pass
     ...
     >>>
     >>> f.slow
     True
     >>> f.hard
     True
     """
 
     if isinstance(label,basestring):
         labels = [label]
     else:
         labels = label
         
     # Validate that the given label(s) are OK for use in setattr() by doing a
     # dry run on a dummy function.
     tmp = lambda : None
     for label in labels:
         setattr(tmp,label,True)
 
     # This is the actual decorator we'll return
     def decor(f):
         for label in labels:
             setattr(f,label,True)
         return f
     
     # Apply the user's docstring, or autogenerate a basic one
     if ds is None:
         ds = "Labels a test as %r." % label
     decor.__doc__ = ds
     
     return decor
 
 
 # Inspired by numpy's skipif, but uses the full apply_wrapper utility to
 # preserve function metadata better and allows the skip condition to be a
 # callable.
 def skipif(skip_condition, msg=None):
     ''' Make function raise SkipTest exception if skip_condition is true
 
     Parameters
     ----------
     skip_condition : bool or callable. 
     Flag to determine whether to skip test.  If the condition is a 
     callable, it is used at runtime to dynamically make the decision.  This 
     is useful for tests that may require costly imports, to delay the cost 
     until the test suite is actually executed.        
     msg : string
         Message to give on raising a SkipTest exception
 
    Returns
    -------
    decorator : function
        Decorator, which, when applied to a function, causes SkipTest
        to be raised when the skip_condition was True, and the function
        to be called normally otherwise.
 
     Notes
     -----
     You will see from the code that we had to further decorate the
     decorator with the nose.tools.make_decorator function in order to
     transmit function name, and various other metadata.
     '''
 
     def skip_decorator(f):
         # Local import to avoid a hard nose dependency and only incur the 
         # import time overhead at actual test-time. 
         import nose
 
         # Allow for both boolean or callable skip conditions.
         if callable(skip_condition):
             skip_val = lambda : skip_condition()
         else:
             skip_val = lambda : skip_condition
 
         def get_msg(func,msg=None):
             """Skip message with information about function being skipped."""
             if msg is None: out = 'Test skipped due to test condition.'
             else: out = msg
             return "Skipping test: %s. %s" % (func.__name__,out)
 
         # We need to define *two* skippers because Python doesn't allow both
         # return with value and yield inside the same function.
         def skipper_func(*args, **kwargs):
             """Skipper for normal test functions."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 return f(*args, **kwargs) 
 
         def skipper_gen(*args, **kwargs):
             """Skipper for test generators."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 for x in f(*args, **kwargs):
                     yield x
 
         # Choose the right skipper to use when building the actual generator.
         if nose.util.isgenerator(f):
             skipper = skipper_gen
         else:
             skipper = skipper_func
             
         return nose.tools.make_decorator(f)(skipper)
 
     return skip_decorator
 
 # A version with the condition set to true, common case just to attacha message
 # to a skip decorator
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
-    :Parameters:
+    Parameters
+    ----------
       msg : string
         Optional message to be added.
 
-    :Returns:
+    Returns
+    -------
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
       """
 
     return skipif(True,msg)
 
 
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
 def numpy_not_available():
     """Can numpy be imported?  Returns true if numpy does NOT import.
 
     This is used to make a decorator to skip tests that require numpy to be
     available, but delay the 'import numpy' to test execution time.
     """
     try:
         import numpy
         np_not_avail = False
     except ImportError:
         np_not_avail = True
 
     return np_not_avail
 
 #-----------------------------------------------------------------------------
 # Decorators for public use
 
 skip_doctest = make_label_dec('skip_doctest',
     """Decorator - mark a function or method for skipping its doctest.
 
     This decorator allows you to mark a function whose docstring you wish to
     omit from testing, while preserving the docstring for introspection, help,
     etc.""")                              
 
 # Decorators to skip certain tests on specific platforms.
 skip_win32 = skipif(sys.platform == 'win32',
                     "This test does not run under Windows")
 skip_linux = skipif(sys.platform == 'linux2',
                     "This test does not run under Linux")
 skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
 
 
 # Decorators to skip tests if not on specific platforms.
 skip_if_not_win32 = skipif(sys.platform != 'win32',
                            "This test only runs under Windows")
 skip_if_not_linux = skipif(sys.platform != 'linux2',
                            "This test only runs under Linux")
 skip_if_not_osx = skipif(sys.platform != 'darwin',
                          "This test only runs under OSX")
 
 # Other skip decorators
 skipif_not_numpy = skipif(numpy_not_available,"This test requires numpy")
 
 skipknownfailure = skip('This test is known to fail')

IPython/testing/iptest.py

@@ -1,324 +1,322 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) or trial recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 For now, this script requires that both nose and twisted are installed.  This
 will change in the future.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 import os
 import os.path as path
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
 import nose.plugins.builtin
 from nose.core import TestProgram
 
 from IPython.utils.platutils import find_cmd
 # from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
 #-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
 def test_for(mod):
     """Test to see if mod is importable."""
     try:
         __import__(mod)
     except ImportError:
         return False
     else:
         return True
 
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
 have_wx_aui = test_for('wx.aui')
 have_zi = test_for('zope.interface')
 have_twisted = test_for('twisted')
 have_foolscap = test_for('foolscap')
 have_objc = test_for('objc')
 have_pexpect = test_for('pexpect')
 have_gtk = test_for('gtk')
 have_gobject = test_for('gobject')
 
 
 def make_exclude():
 
     # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
     # testing problems.  We should strive to minimize the number of skipped
     # modules, since this means untested code.  As the testing machinery
     # solidifies, this list should eventually become empty.
     EXCLUDE = [pjoin('IPython', 'external'),
                pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
                pjoin('IPython_doctest_plugin'),
                pjoin('IPython', 'quarantine'),
                pjoin('IPython', 'deathrow'),
                pjoin('IPython', 'testing', 'attic'),
                pjoin('IPython', 'testing', 'tools'),
                pjoin('IPython', 'testing', 'mkdoctests'),
                pjoin('IPython', 'lib', 'inputhook')
                ]
 
     if not have_wx:
         EXCLUDE.append(pjoin('IPython', 'gui'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
         EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookwx'))
 
     if not have_gtk or not have_gobject:
         EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookgtk'))
 
     if not have_wx_aui:
         EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
 
     if not have_objc:
         EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
 
     if not sys.platform == 'win32':
         EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
         EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
 
     if not os.name == 'posix':
         EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
 
     if not have_pexpect:
         EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have_twisted and have_zi and have_foolscap):
         EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
                              'test_linefrontend'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 
                              'test_frontendbase'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
                              'test_prefilterfrontend'))
         EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
                              'test_asyncfrontendbase')),
         EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
         EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
-        EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 
-                             'test_decorators_trial'))
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
         EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
 
     return EXCLUDE
 
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def run_iptest():
     """Run the IPython test suite using nose.
     
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ 
                         # Loading ipdoctest causes problems with Twisted.
                         # I am removing this as a temporary fix to get the 
                         # test suite back into working shape.  Our nose
                         # plugin needs to be gone through with a fine
                         # toothed comb to find what is causing the problem.
                         # '--with-ipdoctest',
                         # '--ipdoctest-tests','--ipdoctest-extension=txt',
                         # '--detailed-errors',
                        
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
 
     # Detect if any tests were required by explicitly calling an IPython
     # submodule or giving a specific path
     has_tests = False
     for arg in sys.argv:
         if 'IPython' in arg or arg.endswith('.py') or \
            (':' in arg and  '.py' in arg):
             has_tests = True
             break
         
     # If nothing was specifically requested, test full IPython
     if not has_tests:
         argv.append('IPython')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
     EXCLUDE = make_exclude()
     plugins = []
     # plugins = [IPythonDoctest(EXCLUDE)]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
     TestProgram(argv=argv,plugins=plugins)
 
 
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
     def __init__(self,runner='iptest',params=None):
         """ """
         if runner == 'iptest':
             self.runner = ['iptest','-v']
         else:
             self.runner = [find_cmd('trial')]
         if params is None:
             params = []
         if isinstance(params,str):
             params = [params]
         self.params = params
 
         # Assemble call
         self.call_args = self.runner+self.params
 
     if sys.platform == 'win32':
         def run(self):
             """Run the stored commands"""
             # On Windows, cd to temporary directory to run tests.  Otherwise,
             # Twisted's trial may not be able to execute 'trial IPython', since
             # it will confuse the IPython module name with the ipython
             # execution scripts, because the windows file system isn't case
             # sensitive.
             # We also use os.system instead of subprocess.call, because I was
             # having problems with subprocess and I just don't know enough
             # about win32 to debug this reliably.  Os.system may be the 'old
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
             curdir = os.getcwd()
             os.chdir(tempfile.gettempdir())
             stat = os.system(' '.join(self.call_args))
             os.chdir(curdir)
             return stat
     else:
         def run(self):
             """Run the stored commands"""
             return subprocess.call(self.call_args)
 
 
 def make_runners():
     """Define the top-level packages that need to be tested.
     """
 
     nose_packages = ['config', 'core', 'extensions',
                      'frontend', 'lib',
                      'scripts', 'testing', 'utils']
     trial_packages = ['kernel']
 
     if have_wx:
         nose_packages.append('gui')
 
     nose_packages = ['IPython.%s' % m for m in nose_packages ]
     trial_packages = ['IPython.%s' % m for m in trial_packages ]
 
     # Make runners
     runners = dict()
     
     nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages]))
     if have_zi and have_twisted and have_foolscap:
         trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages]))
     runners.update(nose_runners)
     runners.update(trial_runners)
 
     return runners
 
 
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
     This function constructs :class:`IPTester` instances for all IPython
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
 
     runners = make_runners()
 
     # Run all test runners, tracking execution time
     failed = {}
     t_start = time.time()
     for name,runner in runners.iteritems():
         print '*'*77
         print 'IPython test group:',name
         res = runner.run()
         if res:
             failed[name] = res
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)
     nfail = len(failed)
     # summarize results
     print
     print '*'*77
     print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
         print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name in failed:
             failed_runner = runners[name]
             print '-'*40
             print 'Runner failed:',name
             print 'You may wish to rerun this one individually, with:'
             print ' '.join(failed_runner.call_args)
             print
 
 
 def main():
     if len(sys.argv) == 1:
         run_iptestall()
     else:
         if sys.argv[1] == 'all':
             run_iptestall()
         else:
             run_iptest()
 
 
 if __name__ == '__main__':
     main()

IPython/testing/parametric.py

@@ -1,55 +1,58 b''
 """Parametric testing on top of twisted.trial.unittest.
 
+XXX - It may be possbile to deprecate this in favor of the new, cleaner
+parametric code.  We just need to double-check that the new code doesn't clash
+with Twisted (we know it works with nose and unittest).
 """
 
 __all__ = ['parametric','Parametric']
 
 from twisted.trial.unittest import TestCase
 
 def partial(f, *partial_args, **partial_kwargs):
     """Generate a partial class method.
 
     """
     def partial_func(self, *args, **kwargs):
         dikt = dict(kwargs)
         dikt.update(partial_kwargs)
         return f(self, *(partial_args+args), **dikt)
 
     return partial_func
 
 def parametric(f):
     """Mark f as a parametric test.
 
     """
     f._parametric = True
     return classmethod(f)
 
 def Parametric(cls):
     """Register parametric tests with a class.
 
     """
     # Walk over all tests marked with @parametric
     test_generators = [getattr(cls,f) for f in dir(cls)
                     if f.startswith('test')]
     test_generators = [m for m in test_generators if hasattr(m,'_parametric')]
     for test_gen in test_generators:
         test_name = test_gen.func_name
 
         # Insert a new test for each parameter
         for n,test_and_params in enumerate(test_gen()):
             test_method = test_and_params[0]
             test_params = test_and_params[1:]
 
             # Here we use partial (defined above), which returns a
             # class method of type ``types.FunctionType``, unlike
             # functools.partial which returns a function of type
             # ``functools.partial``.
             partial_func = partial(test_method,*test_params)
             # rename the test to look like a testcase
             partial_func.__name__ = 'test_' + partial_func.__name__
 
             # insert the new function into the class as a test
             setattr(cls, test_name + '_%s' % n, partial_func)
 
         # rename test generator so it isn't called again by nose
         test_gen.im_func.func_name = '__done_' + test_name

IPython/testing/tests/test_decorators.py

@@ -1,161 +1,219 b''
 """Tests for the decorators we've created for IPython.
 """
 
 # Module imports
 # Std lib
 import inspect
 import sys
+import unittest
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.testing import decorators as dec
-
+from IPython.testing.ipunittest import ParametricTestCase
 
 #-----------------------------------------------------------------------------
 # Utilities
 
 # Note: copied from OInspect, kept here so the testing stuff doesn't create
 # circular dependencies and is easier to reuse.
 def getargspec(obj):
     """Get the names and default values of a function's arguments.
 
     A tuple of four things is returned: (args, varargs, varkw, defaults).
     'args' is a list of the argument names (it may contain nested lists).
     'varargs' and 'varkw' are the names of the * and ** arguments or None.
     'defaults' is an n-tuple of the default values of the last n arguments.
 
     Modified version of inspect.getargspec from the Python Standard
     Library."""
 
     if inspect.isfunction(obj):
         func_obj = obj
     elif inspect.ismethod(obj):
         func_obj = obj.im_func
     else:
         raise TypeError, 'arg is not a Python function'
     args, varargs, varkw = inspect.getargs(func_obj.func_code)
     return args, varargs, varkw, func_obj.func_defaults
 
 #-----------------------------------------------------------------------------
 # Testing functions
 
[e-mail]_unittest
+def trivial():
+    """A trivial test"""
+    pass
+
+# Some examples of parametric tests.
+
+def is_smaller(i,j):
+    assert i<j,"%s !< %s" % (i,j)
+
+class Tester(ParametricTestCase):
+
+    def test_parametric(self):
+        yield is_smaller(3, 4)
+        x, y = 1, 2
+        yield is_smaller(x, y)
+
[e-mail]
+def test_par_standalone():
+    yield is_smaller(3, 4)
+    x, y = 1, 2
+    yield is_smaller(x, y)
+
+
 @dec.skip
 def test_deliberately_broken():
     """A deliberately broken test - we want to skip this one."""
     1/0
 
 @dec.skip('Testing the skip decorator')
 def test_deliberately_broken2():
     """Another deliberately broken test - we want to skip this one."""
     1/0
 
 
 # Verify that we can correctly skip the doctest for a function at will, but
 # that the docstring itself is NOT destroyed by the decorator.
 @dec.skip_doctest
 def doctest_bad(x,y=1,**k):
     """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     print 'x:',x
     print 'y:',y
     print 'k:',k
 
 
 def call_doctest_bad():
     """Check that we can still call the decorated functions.
     
     >>> doctest_bad(3,y=4)
     x: 3
     y: 4
     k: {}
     """
     pass
 
 
 def test_skip_dt_decorator():
     """Doctest-skipping decorator should preserve the docstring.
     """
     # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
     check = """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     # Fetch the docstring from doctest_bad after decoration.
     val = doctest_bad.__doc__
     
     assert check==val,"doctest_bad docstrings don't match"
 
 # Doctest skipping should work for class methods too
 class foo(object):
     """Foo
 
     Example:
 
     >>> 1+1
     2
     """
 
     @dec.skip_doctest
     def __init__(self,x):
         """Make a foo.
 
         Example:
 
         >>> f = foo(3)
         junk
         """
         print 'Making a foo.'
         self.x = x
         
     @dec.skip_doctest
     def bar(self,y):
         """Example:
 
         >>> f = foo(3)
         >>> f.bar(0)
         boom!
         >>> 1/0
         bam!
         """
         return 1/y
 
     def baz(self,y):
         """Example:
 
         >>> f = foo(3)
         Making a foo.
         >>> f.baz(3)
         True
         """
         return self.x==y
 
 
 
 def test_skip_dt_decorator2():
     """Doctest-skipping decorator should preserve function signature.
     """
     # Hardcoded correct answer
     dtargs = (['x', 'y'], None, 'k', (1,))
     # Introspect out the value
     dtargsr = getargspec(doctest_bad)
     assert dtargsr==dtargs, \
            "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
 
 
 @dec.skip_linux
 def test_linux():
     nt.assert_not_equals(sys.platform,'linux2',"This test can't run under linux")
 
 @dec.skip_win32
 def test_win32():
     nt.assert_not_equals(sys.platform,'win32',"This test can't run under windows")
 
 @dec.skip_osx
 def test_osx():
     nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")
+
+
+# Verify that the same decorators work for methods.
+# Note: this code is identical to that in test_decorators_trial, but that one
+# uses twisted's unittest, not the one from the stdlib, which we are using
+# here. While somewhat redundant, we want to check both with the stdlib and
+# with twisted, so the duplication is OK.
+class TestDecoratorsTrial(unittest.TestCase):
+    
+    @dec.skip()
+    def test_deliberately_broken(self):
+        """A deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip('Testing the skip decorator')
+    def test_deliberately_broken2(self):
+        """Another deliberately broken test - we want to skip this one."""
+        1/0
+
+    @dec.skip_linux
+    def test_linux(self):
+        self.assertNotEquals(sys.platform, 'linux2',
+                             "This test can't run under linux")
+
+    @dec.skip_win32
+    def test_win32(self):
+        self.assertNotEquals(sys.platform, 'win32',
+                             "This test can't run under windows")
+
+    @dec.skip_osx
+    def test_osx(self):
+        self.assertNotEquals(sys.platform, 'darwin',
+                             "This test can't run under osx")

IPython/testing/tools.py

@@ -1,133 +1,132 b''
 """Generic testing tools that do NOT depend on Twisted.
 
 In particular, this module exposes a set of top-level assert* functions that
 can be used in place of nose.tools.assert* in method generators (the ones in
 nose can not, at least as of nose 0.10.4).
 
 Note: our testing package contains testing.util, which does depend on Twisted
 and provides utilities for tests that manage Deferreds.  All testing support
 tools that only depend on nose, IPython or the standard library should go here
 instead.
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 #*****************************************************************************
 #       Copyright (C) 2009 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #-----------------------------------------------------------------------------
 # Required modules and packages
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 
 import nose.tools as nt
 
 from IPython.utils import genutils
-from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
 _tpl = """
 def %(name)s(*a,**kw):
     return nt.%(name)s(*a,**kw)
 """
 
 for _x in [a for a in dir(nt) if a.startswith('assert')]:
     exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = genutils.list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
     
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
     
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False