##// END OF EJS Templates
Add new testing support machinery with better parametric tests....
Fernando Perez -
Show More
@@ -0,0 +1,156 b''
1 """Experimental code for cleaner support of IPython syntax with unittest.
2
3 In IPython up until 0.10, we've used very hacked up nose machinery for running
4 tests with IPython special syntax, and this has proved to be extremely slow.
5 This module provides decorators to try a different approach, stemming from a
6 conversation Brian and I (FP) had about this problem Sept/09.
7
8 The goal is to be able to easily write simple functions that can be seen by
9 unittest as tests, and ultimately for these to support doctests with full
10 IPython syntax. Nose already offers this based on naming conventions and our
11 hackish plugins, but we are seeking to move away from nose dependencies if
12 possible.
13
14 This module follows a different approach, based on decorators.
15
16 - A decorator called @ipdoctest can mark any function as having a docstring
17 that should be viewed as a doctest, but after syntax conversion.
18
19 Authors
20 -------
21
22 - Fernando Perez <Fernando.Perez@berkeley.edu>
23 """
24
25 #-----------------------------------------------------------------------------
26 # Copyright (C) 2009 The IPython Development Team
27 #
28 # Distributed under the terms of the BSD License. The full license is in
29 # the file COPYING, distributed as part of this software.
30 #-----------------------------------------------------------------------------
31
32
33 #-----------------------------------------------------------------------------
34 # Imports
35 #-----------------------------------------------------------------------------
36
37 # Stdlib
38 import re
39 import sys
40 import unittest
41 from doctest import DocTestFinder, DocTestRunner, TestResults
42
43 # Our own
44 import nosepatch
45
46 # We already have python3-compliant code for parametric tests
47 if sys.version[0]=='2':
48 from _paramtestpy2 import ParametricTestCase
49 else:
50 from _paramtestpy3 import ParametricTestCase
51
52 #-----------------------------------------------------------------------------
53 # Classes and functions
54 #-----------------------------------------------------------------------------
55
56 def count_failures(runner):
57 """Count number of failures in a doctest runner.
58
59 Code modeled after the summarize() method in doctest.
60 """
61 return [TestResults(f, t) for f, t in runner._name2ft.values() if f > 0 ]
62
63
64 class IPython2PythonConverter(object):
65 """Convert IPython 'syntax' to valid Python.
66
67 Eventually this code may grow to be the full IPython syntax conversion
68 implementation, but for now it only does prompt convertion."""
69
70 def __init__(self):
71 self.ps1 = re.compile(r'In\ \[\d+\]: ')
72 self.ps2 = re.compile(r'\ \ \ \.\.\.+: ')
73 self.out = re.compile(r'Out\[\d+\]: \s*?\n?')
74
75 def __call__(self, ds):
76 """Convert IPython prompts to python ones in a string."""
77 pyps1 = '>>> '
78 pyps2 = '... '
79 pyout = ''
80
81 dnew = ds
82 dnew = self.ps1.sub(pyps1, dnew)
83 dnew = self.ps2.sub(pyps2, dnew)
84 dnew = self.out.sub(pyout, dnew)
85 return dnew
86
87
88 class Doc2UnitTester(object):
89 """Class whose instances act as a decorator for docstring testing.
90
91 In practice we're only likely to need one instance ever, made below (though
92 no attempt is made at turning it into a singleton, there is no need for
93 that).
94 """
95 def __init__(self, verbose=False):
96 """New decorator.
97
98 Parameters
99 ----------
100
101 verbose : boolean, optional (False)
102 Passed to the doctest finder and runner to control verbosity.
103 """
104 self.verbose = verbose
105 # We can reuse the same finder for all instances
106 self.finder = DocTestFinder(verbose=verbose, recurse=False)
107
108 def __call__(self, func):
109 """Use as a decorator: doctest a function's docstring as a unittest.
110
111 This version runs normal doctests, but the idea is to make it later run
112 ipython syntax instead."""
113
114 # Capture the enclosing instance with a different name, so the new
115 # class below can see it without confusion regarding its own 'self'
116 # that will point to the test instance at runtime
117 d2u = self
118
119 # Rewrite the function's docstring to have python syntax
120 if func.__doc__ is not None:
121 func.__doc__ = ip2py(func.__doc__)
122
123 # Now, create a tester object that is a real unittest instance, so
124 # normal unittest machinery (or Nose, or Trial) can find it.
125 class Tester(unittest.TestCase):
126 def test(self):
127 # Make a new runner per function to be tested
128 runner = DocTestRunner(verbose=d2u.verbose)
129 map(runner.run, d2u.finder.find(func, func.__name__))
130 failed = count_failures(runner)
131 if failed:
132 # Since we only looked at a single function's docstring,
133 # failed should contain at most one item. More than that
134 # is a case we can't handle and should error out on
135 if len(failed) > 1:
136 err = "Invalid number of test results:" % failed
137 raise ValueError(err)
138 # Report a normal failure.
139 self.fail('failed doctests: %s' % str(failed[0]))
140
141 # Rename it so test reports have the original signature.
142 Tester.__name__ = func.__name__
143 return Tester
144
145
146 def ipdocstring(func):
147 """Change the function docstring via ip2py.
148 """
149 if func.__doc__ is not None:
150 func.__doc__ = ip2py(func.__doc__)
151 return func
152
153
154 # Make an instance of the classes for public use
155 ipdoctest = Doc2UnitTester()
156 ip2py = IPython2PythonConverter()
@@ -0,0 +1,122 b''
1 """Tests for IPyhton's test support utilities.
2
3 These are decorators that allow standalone functions and docstrings to be seen
4 as tests by unittest, replicating some of nose's functionality. Additionally,
5 IPython-syntax docstrings can be auto-converted to '>>>' so that ipython
6 sessions can be copy-pasted as tests.
7
8 This file can be run as a script, and it will call unittest.main(). We must
9 check that it works with unittest as well as with nose...
10
11
12 Notes:
13
14 - Using nosetests --with-doctest --doctest-tests testfile.py
15 will find docstrings as tests wherever they are, even in methods. But
16 if we use ipython syntax in the docstrings, they must be decorated with
17 @ipdocstring. This is OK for test-only code, but not for user-facing
18 docstrings where we want to keep the ipython syntax.
19
20 - Using nosetests --with-doctest file.py
21 also finds doctests if the file name doesn't have 'test' in it, because it is
22 treated like a normal module. But if nose treats the file like a test file,
23 then for normal classes to be doctested the extra --doctest-tests is
24 necessary.
25
26 - running this script with python (it has a __main__ section at the end) misses
27 one docstring test, the one embedded in the Foo object method. Since our
28 approach relies on using decorators that create standalone TestCase
29 instances, it can only be used for functions, not for methods of objects.
30 Authors
31 -------
32
33 - Fernando Perez <Fernando.Perez@berkeley.edu>
34 """
35
36 #-----------------------------------------------------------------------------
37 # Copyright (C) 2009 The IPython Development Team
38 #
39 # Distributed under the terms of the BSD License. The full license is in
40 # the file COPYING, distributed as part of this software.
41 #-----------------------------------------------------------------------------
42
43
44 #-----------------------------------------------------------------------------
45 # Imports
46 #-----------------------------------------------------------------------------
47
48 from IPython.testing.ipunittest import ipdoctest, ipdocstring
49
50 #-----------------------------------------------------------------------------
51 # Test classes and functions
52 #-----------------------------------------------------------------------------
53 @ipdoctest
54 def simple_dt():
55 """
56 >>> print 1+1
57 2
58 """
59
60
61 @ipdoctest
62 def ipdt_flush():
63 """
64 In [20]: print 1
65 1
66
67 In [26]: for i in range(10):
68 ....: print i,
69 ....:
70 ....:
71 0 1 2 3 4 5 6 7 8 9
72
73 In [27]: 3+4
74 Out[27]: 7
75 """
76
77
78 @ipdoctest
79 def ipdt_indented_test():
80 """
81 In [20]: print 1
82 1
83
84 In [26]: for i in range(10):
85 ....: print i,
86 ....:
87 ....:
88 0 1 2 3 4 5 6 7 8 9
89
90 In [27]: 3+4
91 Out[27]: 7
92 """
93
94
95 class Foo(object):
96 """For methods, the normal decorator doesn't work.
97
98 But rewriting the docstring with ip2py does, *but only if using nose
99 --with-doctest*. Do we want to have that as a dependency?
100 """
101
102 @ipdocstring
103 def ipdt_method(self):
104 """
105 In [20]: print 1
106 2
107
108 In [26]: for i in range(10):
109 ....: print i,
110 ....:
111 ....:
112 0 1 2 3 4 5 6 7 8 9
113
114 In [27]: 3+4
115 Out[27]: 7
116 """
117
118 def normaldt_method(self):
119 """
120 >>> print 1+1
121 2
122 """
1 NO CONTENT: file renamed from IPython/testing/decorators_numpy.py to IPython/external/decorators.py
NO CONTENT: file renamed from IPython/testing/decorators_numpy.py to IPython/external/decorators.py
@@ -21,13 +21,12 b' import os'
21
21
22 from twisted.trial import unittest
22 from twisted.trial import unittest
23
23
24 from IPython.testing import decorators_trial as dec
24 from IPython.testing import decorators as dec
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Tests
27 # Tests
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29
29
30
31 class TestRedirector(unittest.TestCase):
30 class TestRedirector(unittest.TestCase):
32
31
33 @dec.skip_win32
32 @dec.skip_win32
@@ -10,27 +10,79 b' This module provides a set of useful decorators meant to be ready to use in'
10 your own tests. See the bottom of the file for the ready-made ones, and if you
10 your own tests. See the bottom of the file for the ready-made ones, and if you
11 find yourself writing a new one that may be of generic use, add it here.
11 find yourself writing a new one that may be of generic use, add it here.
12
12
13 Included decorators:
14
15
16 Lightweight testing that remains unittest-compatible.
17
18 - @parametric, for parametric test support that is vastly easier to use than
19 nose's for debugging. With ours, if a test fails, the stack under inspection
20 is that of the test and not that of the test framework.
21
22 - An @as_unittest decorator can be used to tag any normal parameter-less
23 function as a unittest TestCase. Then, both nose and normal unittest will
24 recognize it as such. This will make it easier to migrate away from Nose if
25 we ever need/want to while maintaining very lightweight tests.
26
13 NOTE: This file contains IPython-specific decorators and imports the
27 NOTE: This file contains IPython-specific decorators and imports the
14 numpy.testing.decorators file, which we've copied verbatim. Any of our own
28 numpy.testing.decorators file, which we've copied verbatim. Any of our own
15 code will be added at the bottom if we end up extending this.
29 code will be added at the bottom if we end up extending this.
30
31 Authors
32 -------
33
34 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
35 """
17
36
37 #-----------------------------------------------------------------------------
38 # Copyright (C) 2009-2010 The IPython Development Team
39 #
40 # Distributed under the terms of the BSD License. The full license is in
41 # the file COPYING, distributed as part of this software.
42 #-----------------------------------------------------------------------------
43
44 #-----------------------------------------------------------------------------
45 # Imports
46 #-----------------------------------------------------------------------------
47
18 # Stdlib imports
48 # Stdlib imports
19 import inspect
49 import inspect
20 import sys
50 import sys
51 import unittest
21
52
22 # Third-party imports
53 # Third-party imports
23
54
24 # This is Michele Simionato's decorator module, also kept verbatim.
55 # This is Michele Simionato's decorator module, kept verbatim.
25 from IPython.external.decorator import decorator, update_wrapper
56 from IPython.external.decorator import decorator, update_wrapper
26
57
58 # Our own modules
59 import nosepatch # monkeypatch nose
60
61 # We already have python3-compliant code for parametric tests
62 if sys.version[0]=='2':
63 from _paramtestpy2 import parametric
64 else:
65 from _paramtestpy3 import parametric
66
27 # Grab the numpy-specific decorators which we keep in a file that we
67 # Grab the numpy-specific decorators which we keep in a file that we
28 # occasionally update from upstream: decorators_numpy.py is an IDENTICAL copy
68 # occasionally update from upstream: decorators.py is a copy of
29 # of numpy.testing.decorators.
69 # numpy.testing.decorators, we expose all of it here.
30 from decorators_numpy import *
70 from IPython.external.decorators import *
71
72 #-----------------------------------------------------------------------------
73 # Classes and functions
74 #-----------------------------------------------------------------------------
75
76 # Simple example of the basic idea
77 def as_unittest(func):
78 """Decorator to make a simple function into a normal test via unittest."""
79 class Tester(unittest.TestCase):
80 def test(self):
81 func()
31
82
32 ##############################################################################
83 Tester.__name__ = func.__name__
33 # Local code begins
84
85 return Tester
34
86
35 # Utility functions
87 # Utility functions
36
88
@@ -51,21 +103,23 b' def apply_wrapper(wrapper,func):'
51 def make_label_dec(label,ds=None):
103 def make_label_dec(label,ds=None):
52 """Factory function to create a decorator that applies one or more labels.
104 """Factory function to create a decorator that applies one or more labels.
53
105
54 :Parameters:
106 Parameters
107 ----------
55 label : string or sequence
108 label : string or sequence
56 One or more labels that will be applied by the decorator to the functions
109 One or more labels that will be applied by the decorator to the functions
57 it decorates. Labels are attributes of the decorated function with their
110 it decorates. Labels are attributes of the decorated function with their
58 value set to True.
111 value set to True.
59
112
60 :Keywords:
61 ds : string
113 ds : string
62 An optional docstring for the resulting decorator. If not given, a
114 An optional docstring for the resulting decorator. If not given, a
63 default docstring is auto-generated.
115 default docstring is auto-generated.
64
116
65 :Returns:
117 Returns
118 -------
66 A decorator.
119 A decorator.
67
120
68 :Examples:
121 Examples
122 --------
69
123
70 A simple labeling decorator:
124 A simple labeling decorator:
71 >>> slow = make_label_dec('slow')
125 >>> slow = make_label_dec('slow')
@@ -193,11 +247,13 b' def skipif(skip_condition, msg=None):'
193 def skip(msg=None):
247 def skip(msg=None):
194 """Decorator factory - mark a test function for skipping from test suite.
248 """Decorator factory - mark a test function for skipping from test suite.
195
249
196 :Parameters:
250 Parameters
251 ----------
197 msg : string
252 msg : string
198 Optional message to be added.
253 Optional message to be added.
199
254
200 :Returns:
255 Returns
256 -------
201 decorator : function
257 decorator : function
202 Decorator, which, when applied to a function, causes SkipTest
258 Decorator, which, when applied to a function, causes SkipTest
203 to be raised, with the optional message added.
259 to be raised, with the optional message added.
@@ -125,8 +125,6 b' def make_exclude():'
125 'test_asyncfrontendbase')),
125 'test_asyncfrontendbase')),
126 EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
126 EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
127 EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
127 EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
128 EXCLUDE.append(pjoin('IPython', 'testing', 'tests',
129 'test_decorators_trial'))
130
128
131 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
129 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
132 if sys.platform == 'win32':
130 if sys.platform == 'win32':
@@ -1,5 +1,8 b''
1 """Parametric testing on top of twisted.trial.unittest.
1 """Parametric testing on top of twisted.trial.unittest.
2
2
3 XXX - It may be possbile to deprecate this in favor of the new, cleaner
4 parametric code. We just need to double-check that the new code doesn't clash
5 with Twisted (we know it works with nose and unittest).
3 """
6 """
4
7
5 __all__ = ['parametric','Parametric']
8 __all__ = ['parametric','Parametric']
@@ -5,13 +5,14 b''
5 # Std lib
5 # Std lib
6 import inspect
6 import inspect
7 import sys
7 import sys
8 import unittest
8
9
9 # Third party
10 # Third party
10 import nose.tools as nt
11 import nose.tools as nt
11
12
12 # Our own
13 # Our own
13 from IPython.testing import decorators as dec
14 from IPython.testing import decorators as dec
14
15 from IPython.testing.ipunittest import ParametricTestCase
15
16
16 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
17 # Utilities
18 # Utilities
@@ -41,6 +42,30 b' def getargspec(obj):'
41 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
42 # Testing functions
43 # Testing functions
43
44
45 @dec.as_unittest
46 def trivial():
47 """A trivial test"""
48 pass
49
50 # Some examples of parametric tests.
51
52 def is_smaller(i,j):
53 assert i<j,"%s !< %s" % (i,j)
54
55 class Tester(ParametricTestCase):
56
57 def test_parametric(self):
58 yield is_smaller(3, 4)
59 x, y = 1, 2
60 yield is_smaller(x, y)
61
62 @dec.parametric
63 def test_par_standalone():
64 yield is_smaller(3, 4)
65 x, y = 1, 2
66 yield is_smaller(x, y)
67
68
44 @dec.skip
69 @dec.skip
45 def test_deliberately_broken():
70 def test_deliberately_broken():
46 """A deliberately broken test - we want to skip this one."""
71 """A deliberately broken test - we want to skip this one."""
@@ -159,3 +184,36 b' def test_win32():'
159 @dec.skip_osx
184 @dec.skip_osx
160 def test_osx():
185 def test_osx():
161 nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")
186 nt.assert_not_equals(sys.platform,'darwin',"This test can't run under osx")
187
188
189 # Verify that the same decorators work for methods.
190 # Note: this code is identical to that in test_decorators_trial, but that one
191 # uses twisted's unittest, not the one from the stdlib, which we are using
192 # here. While somewhat redundant, we want to check both with the stdlib and
193 # with twisted, so the duplication is OK.
194 class TestDecoratorsTrial(unittest.TestCase):
195
196 @dec.skip()
197 def test_deliberately_broken(self):
198 """A deliberately broken test - we want to skip this one."""
199 1/0
200
201 @dec.skip('Testing the skip decorator')
202 def test_deliberately_broken2(self):
203 """Another deliberately broken test - we want to skip this one."""
204 1/0
205
206 @dec.skip_linux
207 def test_linux(self):
208 self.assertNotEquals(sys.platform, 'linux2',
209 "This test can't run under linux")
210
211 @dec.skip_win32
212 def test_win32(self):
213 self.assertNotEquals(sys.platform, 'win32',
214 "This test can't run under windows")
215
216 @dec.skip_osx
217 def test_osx(self):
218 self.assertNotEquals(sys.platform, 'darwin',
219 "This test can't run under osx")
@@ -33,7 +33,6 b' import sys'
33 import nose.tools as nt
33 import nose.tools as nt
34
34
35 from IPython.utils import genutils
35 from IPython.utils import genutils
36 from IPython.testing import decorators as dec
37
36
38 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
39 # Globals
38 # Globals
General Comments 0
You need to be logged in to leave comments. Login now