From 277cf879d45c91dbfff8dc7d17e79759d7edc94e 2010-01-01 03:26:22 From: Fernando Perez Date: 2010-01-01 03:26:22 Subject: [PATCH] Add new testing support machinery with better parametric tests. Also included are new tools for doctests with ipython syntax. --- diff --git a/IPython/testing/decorators_numpy.py b/IPython/external/decorators.py similarity index 100% rename from IPython/testing/decorators_numpy.py rename to IPython/external/decorators.py diff --git a/IPython/kernel/core/tests/test_redirectors.py b/IPython/kernel/core/tests/test_redirectors.py index 3b0e416..57d103c 100644 --- a/IPython/kernel/core/tests/test_redirectors.py +++ b/IPython/kernel/core/tests/test_redirectors.py @@ -21,13 +21,12 @@ import os from twisted.trial import unittest -from IPython.testing import decorators_trial as dec +from IPython.testing import decorators as dec #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- - class TestRedirector(unittest.TestCase): @dec.skip_win32 diff --git a/IPython/testing/decorators.py b/IPython/testing/decorators.py index d2ecbd8..3ac7004 100644 --- a/IPython/testing/decorators.py +++ b/IPython/testing/decorators.py @@ -10,27 +10,79 @@ This module provides a set of useful decorators meant to be ready to use in your own tests. See the bottom of the file for the ready-made ones, and if you find yourself writing a new one that may be of generic use, add it here. +Included decorators: + + +Lightweight testing that remains unittest-compatible. + +- @parametric, for parametric test support that is vastly easier to use than + nose's for debugging. With ours, if a test fails, the stack under inspection + is that of the test and not that of the test framework. + +- An @as_unittest decorator can be used to tag any normal parameter-less + function as a unittest TestCase. Then, both nose and normal unittest will + recognize it as such. This will make it easier to migrate away from Nose if + we ever need/want to while maintaining very lightweight tests. + NOTE: This file contains IPython-specific decorators and imports the numpy.testing.decorators file, which we've copied verbatim. Any of our own code will be added at the bottom if we end up extending this. + +Authors +------- + +- Fernando Perez """ +#----------------------------------------------------------------------------- +# Copyright (C) 2009-2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + # Stdlib imports import inspect import sys +import unittest # Third-party imports -# This is Michele Simionato's decorator module, also kept verbatim. +# This is Michele Simionato's decorator module, kept verbatim. from IPython.external.decorator import decorator, update_wrapper +# Our own modules +import nosepatch # monkeypatch nose + +# We already have python3-compliant code for parametric tests +if sys.version[0]=='2': + from _paramtestpy2 import parametric +else: + from _paramtestpy3 import parametric + # Grab the numpy-specific decorators which we keep in a file that we -# occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy -# of numpy.testing.decorators. -from decorators_numpy import * +# occasionally update from upstream: decorators.py is a copy of +# numpy.testing.decorators, we expose all of it here. +from IPython.external.decorators import * + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- -############################################################################## -# Local code begins +# Simple example of the basic idea +def as_unittest(func): + """Decorator to make a simple function into a normal test via unittest.""" + class Tester(unittest.TestCase): + def test(self): + func() + + Tester.__name__ = func.__name__ + + return Tester # Utility functions @@ -51,21 +103,23 @@ def apply_wrapper(wrapper,func): def make_label_dec(label,ds=None): """Factory function to create a decorator that applies one or more labels. - :Parameters: + Parameters + ---------- label : string or sequence One or more labels that will be applied by the decorator to the functions it decorates. Labels are attributes of the decorated function with their value set to True. - :Keywords: ds : string An optional docstring for the resulting decorator. If not given, a default docstring is auto-generated. - :Returns: + Returns + ------- A decorator. - :Examples: + Examples + -------- A simple labeling decorator: >>> slow = make_label_dec('slow') @@ -193,11 +247,13 @@ def skipif(skip_condition, msg=None): def skip(msg=None): """Decorator factory - mark a test function for skipping from test suite. - :Parameters: + Parameters + ---------- msg : string Optional message to be added. - :Returns: + Returns + ------- decorator : function Decorator, which, when applied to a function, causes SkipTest to be raised, with the optional message added. diff --git a/IPython/testing/iptest.py b/IPython/testing/iptest.py index e229787..c2cf104 100644 --- a/IPython/testing/iptest.py +++ b/IPython/testing/iptest.py @@ -125,8 +125,6 @@ def make_exclude(): 'test_asyncfrontendbase')), EXCLUDE.append(pjoin('IPython', 'testing', 'parametric')) EXCLUDE.append(pjoin('IPython', 'testing', 'util')) - EXCLUDE.append(pjoin('IPython', 'testing', 'tests', - 'test_decorators_trial')) # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. if sys.platform == 'win32': diff --git a/IPython/testing/ipunittest.py b/IPython/testing/ipunittest.py new file mode 100644 index 0000000..2687fd0 --- /dev/null +++ b/IPython/testing/ipunittest.py @@ -0,0 +1,156 @@ +"""Experimental code for cleaner support of IPython syntax with unittest. + +In IPython up until 0.10, we've used very hacked up nose machinery for running +tests with IPython special syntax, and this has proved to be extremely slow. +This module provides decorators to try a different approach, stemming from a +conversation Brian and I (FP) had about this problem Sept/09. + +The goal is to be able to easily write simple functions that can be seen by +unittest as tests, and ultimately for these to support doctests with full +IPython syntax. Nose already offers this based on naming conventions and our +hackish plugins, but we are seeking to move away from nose dependencies if +possible. + +This module follows a different approach, based on decorators. + +- A decorator called @ipdoctest can mark any function as having a docstring + that should be viewed as a doctest, but after syntax conversion. + +Authors +------- + +- Fernando Perez +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Stdlib +import re +import sys +import unittest +from doctest import DocTestFinder, DocTestRunner, TestResults + +# Our own +import nosepatch + +# We already have python3-compliant code for parametric tests +if sys.version[0]=='2': + from _paramtestpy2 import ParametricTestCase +else: + from _paramtestpy3 import ParametricTestCase + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +def count_failures(runner): + """Count number of failures in a doctest runner. + + Code modeled after the summarize() method in doctest. + """ + return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0 ] + + +class IPython2PythonConverter(object): + """Convert IPython 'syntax' to valid Python. + + Eventually this code may grow to be the full IPython syntax conversion + implementation, but for now it only does prompt convertion.""" + + def __init__(self): + self.ps1 = re.compile(r'In\ \[\d+\]: ') + self.ps2 = re.compile(r'\ \ \ \.\.\.+: ') + self.out = re.compile(r'Out\[\d+\]: \s*?\n?') + + def __call__(self, ds): + """Convert IPython prompts to python ones in a string.""" + pyps1 = '>>> ' + pyps2 = '... ' + pyout = '' + + dnew = ds + dnew = self.ps1.sub(pyps1, dnew) + dnew = self.ps2.sub(pyps2, dnew) + dnew = self.out.sub(pyout, dnew) + return dnew + + +class Doc2UnitTester(object): + """Class whose instances act as a decorator for docstring testing. + + In practice we're only likely to need one instance ever, made below (though + no attempt is made at turning it into a singleton, there is no need for + that). + """ + def __init__(self, verbose=False): + """New decorator. + + Parameters + ---------- + + verbose : boolean, optional (False) + Passed to the doctest finder and runner to control verbosity. + """ + self.verbose = verbose + # We can reuse the same finder for all instances + self.finder = DocTestFinder(verbose=verbose, recurse=False) + + def __call__(self, func): + """Use as a decorator: doctest a function's docstring as a unittest. + + This version runs normal doctests, but the idea is to make it later run + ipython syntax instead.""" + + # Capture the enclosing instance with a different name, so the new + # class below can see it without confusion regarding its own 'self' + # that will point to the test instance at runtime + d2u = self + + # Rewrite the function's docstring to have python syntax + if func.__doc__ is not None: + func.__doc__ = ip2py(func.__doc__) + + # Now, create a tester object that is a real unittest instance, so + # normal unittest machinery (or Nose, or Trial) can find it. + class Tester(unittest.TestCase): + def test(self): + # Make a new runner per function to be tested + runner = DocTestRunner(verbose=d2u.verbose) + map(runner.run, d2u.finder.find(func, func.__name__)) + failed = count_failures(runner) + if failed: + # Since we only looked at a single function's docstring, + # failed should contain at most one item. More than that + # is a case we can't handle and should error out on + if len(failed) > 1: + err = "Invalid number of test results:" % failed + raise ValueError(err) + # Report a normal failure. + self.fail('failed doctests: %s' % str(failed[0])) + + # Rename it so test reports have the original signature. + Tester.__name__ = func.__name__ + return Tester + + +def ipdocstring(func): + """Change the function docstring via ip2py. + """ + if func.__doc__ is not None: + func.__doc__ = ip2py(func.__doc__) + return func + + +# Make an instance of the classes for public use +ipdoctest = Doc2UnitTester() +ip2py = IPython2PythonConverter() diff --git a/IPython/testing/parametric.py b/IPython/testing/parametric.py index cd2bf81..b1bf243 100644 --- a/IPython/testing/parametric.py +++ b/IPython/testing/parametric.py @@ -1,5 +1,8 @@ """Parametric testing on top of twisted.trial.unittest. +XXX - It may be possbile to deprecate this in favor of the new, cleaner +parametric code. We just need to double-check that the new code doesn't clash +with Twisted (we know it works with nose and unittest). """ __all__ = ['parametric','Parametric'] diff --git a/IPython/testing/tests/test_decorators.py b/IPython/testing/tests/test_decorators.py index 5f7e688..e548f5d 100644 --- a/IPython/testing/tests/test_decorators.py +++ b/IPython/testing/tests/test_decorators.py @@ -5,13 +5,14 @@ # Std lib import inspect import sys +import unittest # Third party import nose.tools as nt # Our own from IPython.testing import decorators as dec - +from IPython.testing.ipunittest import ParametricTestCase #----------------------------------------------------------------------------- # Utilities @@ -41,6 +42,30 @@ def getargspec(obj): #----------------------------------------------------------------------------- # Testing functions +@dec.as_unittest +def trivial(): + """A trivial test""" + pass + +# Some examples of parametric tests. + +def is_smaller(i,j): + assert i>>' so that ipython +sessions can be copy-pasted as tests. + +This file can be run as a script, and it will call unittest.main(). We must +check that it works with unittest as well as with nose... + + +Notes: + +- Using nosetests --with-doctest --doctest-tests testfile.py + will find docstrings as tests wherever they are, even in methods. But + if we use ipython syntax in the docstrings, they must be decorated with + @ipdocstring. This is OK for test-only code, but not for user-facing + docstrings where we want to keep the ipython syntax. + +- Using nosetests --with-doctest file.py + also finds doctests if the file name doesn't have 'test' in it, because it is + treated like a normal module. But if nose treats the file like a test file, + then for normal classes to be doctested the extra --doctest-tests is + necessary. + +- running this script with python (it has a __main__ section at the end) misses + one docstring test, the one embedded in the Foo object method. Since our + approach relies on using decorators that create standalone TestCase + instances, it can only be used for functions, not for methods of objects. +Authors +------- + +- Fernando Perez +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2009 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from IPython.testing.ipunittest import ipdoctest, ipdocstring + +#----------------------------------------------------------------------------- +# Test classes and functions +#----------------------------------------------------------------------------- +@ipdoctest +def simple_dt(): + """ + >>> print 1+1 + 2 + """ + + +@ipdoctest +def ipdt_flush(): + """ +In [20]: print 1 +1 + +In [26]: for i in range(10): + ....: print i, + ....: + ....: +0 1 2 3 4 5 6 7 8 9 + +In [27]: 3+4 +Out[27]: 7 +""" + + +@ipdoctest +def ipdt_indented_test(): + """ + In [20]: print 1 + 1 + + In [26]: for i in range(10): + ....: print i, + ....: + ....: + 0 1 2 3 4 5 6 7 8 9 + + In [27]: 3+4 + Out[27]: 7 + """ + + +class Foo(object): + """For methods, the normal decorator doesn't work. + + But rewriting the docstring with ip2py does, *but only if using nose + --with-doctest*. Do we want to have that as a dependency? + """ + + @ipdocstring + def ipdt_method(self): + """ + In [20]: print 1 + 2 + + In [26]: for i in range(10): + ....: print i, + ....: + ....: + 0 1 2 3 4 5 6 7 8 9 + + In [27]: 3+4 + Out[27]: 7 + """ + + def normaldt_method(self): + """ + >>> print 1+1 + 2 + """ diff --git a/IPython/testing/tools.py b/IPython/testing/tools.py index 89d1504..192b390 100644 --- a/IPython/testing/tools.py +++ b/IPython/testing/tools.py @@ -33,7 +33,6 @@ import sys import nose.tools as nt from IPython.utils import genutils -from IPython.testing import decorators as dec #----------------------------------------------------------------------------- # Globals