##// END OF EJS Templates
make know failures report as 'K'...
Paul Ivanov -
Show More
@@ -0,0 +1,41 b''
1 # IPython: modified copy of numpy.testing.noseclasses, so
2 # IPython.external._decorators works without numpy being installed.
3
4 # These classes implement a "known failure" error class.
5
6 import os
7
8 from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin
9
10 class KnownFailureTest(Exception):
11 '''Raise this exception to mark a test as a known failing test.'''
12 pass
13
14
15 class KnownFailure(ErrorClassPlugin):
16 '''Plugin that installs a KNOWNFAIL error class for the
17 KnownFailureClass exception. When KnownFailureTest is raised,
18 the exception will be logged in the knownfail attribute of the
19 result, 'K' or 'KNOWNFAIL' (verbose) will be output, and the
20 exception will not be counted as an error or failure.'''
21 enabled = True
22 knownfail = ErrorClass(KnownFailureTest,
23 label='KNOWNFAIL',
24 isfailure=False)
25
26 def options(self, parser, env=os.environ):
27 env_opt = 'NOSE_WITHOUT_KNOWNFAIL'
28 parser.add_option('--no-knownfail', action='store_true',
29 dest='noKnownFail', default=env.get(env_opt, False),
30 help='Disable special handling of KnownFailureTest '
31 'exceptions')
32
33 def configure(self, options, conf):
34 if not self.can_configure:
35 return
36 self.conf = conf
37 disable = getattr(options, 'noKnownFail', False)
38 if disable:
39 self.enabled = False
40
41
@@ -1,4 +1,7 b''
1 1 try:
2 2 from numpy.testing.decorators import *
3 from numpy.testing.noseclasses import KnownFailure
3 4 except ImportError:
4 5 from _decorators import *
6 # KnownFailure imported in _decorators from local version of
7 # noseclasses which is in _numpy_testing_noseclasses
@@ -1,284 +1,285 b''
1 1 """
2 2 Decorators for labeling and modifying behavior of test objects.
3 3
4 4 Decorators that merely return a modified version of the original
5 5 function object are straightforward. Decorators that return a new
6 6 function object need to use
7 7 ::
8 8
9 9 nose.tools.make_decorator(original_function)(decorator)
10 10
11 11 in returning the decorator, in order to preserve meta-data such as
12 12 function name, setup and teardown functions and so on - see
13 13 ``nose.tools`` for more information.
14 14
15 15 """
16 16 import warnings
17 import sys
18 17
19 18 # IPython changes: make this work if numpy not available
20 19 # Original code:
21 20 #from numpy.testing.utils import \
22 21 # WarningManager, WarningMessage
23 22 # Our version:
24 23 try:
25 24 from numpy.testing.utils import WarningManager, WarningMessage
26 25 except ImportError:
27 26 from _numpy_testing_utils import WarningManager, WarningMessage
28
27
28 try:
29 from numpy.testing.noseclasses import KnownFailure, KnownFailureTest
30 except ImportError:
31 from _numpy_testing_noseclasses import KnownFailure, KnownFailureTest
32
29 33 # End IPython changes
30 34
31 35 def slow(t):
32 36 """
33 37 Label a test as 'slow'.
34 38
35 39 The exact definition of a slow test is obviously both subjective and
36 40 hardware-dependent, but in general any individual test that requires more
37 than a second or two should be labeled as slow (the whole suite consits of
41 than a second or two should be labeled as slow (the whole suite consists of
38 42 thousands of tests, so even a second is significant).
39 43
40 44 Parameters
41 45 ----------
42 46 t : callable
43 47 The test to label as slow.
44 48
45 49 Returns
46 50 -------
47 51 t : callable
48 52 The decorated test `t`.
49 53
50 54 Examples
51 55 --------
52 56 The `numpy.testing` module includes ``import decorators as dec``.
53 57 A test can be decorated as slow like this::
54 58
55 59 from numpy.testing import *
56 60
57 61 @dec.slow
58 62 def test_big(self):
59 63 print 'Big, slow test'
60 64
61 65 """
62 66
63 67 t.slow = True
64 68 return t
65 69
66 70 def setastest(tf=True):
67 71 """
68 72 Signals to nose that this function is or is not a test.
69 73
70 74 Parameters
71 75 ----------
72 76 tf : bool
73 77 If True, specifies that the decorated callable is a test.
74 78 If False, specifies that the decorated callable is not a test.
75 79 Default is True.
76 80
77 81 Notes
78 82 -----
79 83 This decorator can't use the nose namespace, because it can be
80 84 called from a non-test module. See also ``istest`` and ``nottest`` in
81 85 ``nose.tools``.
82 86
83 87 Examples
84 88 --------
85 89 `setastest` can be used in the following way::
86 90
87 91 from numpy.testing.decorators import setastest
88 92
89 93 @setastest(False)
90 94 def func_with_test_in_name(arg1, arg2):
91 95 pass
92 96
93 97 """
94 98 def set_test(t):
95 99 t.__test__ = tf
96 100 return t
97 101 return set_test
98 102
99 103 def skipif(skip_condition, msg=None):
100 104 """
101 105 Make function raise SkipTest exception if a given condition is true.
102 106
103 107 If the condition is a callable, it is used at runtime to dynamically
104 108 make the decision. This is useful for tests that may require costly
105 109 imports, to delay the cost until the test suite is actually executed.
106 110
107 111 Parameters
108 112 ----------
109 113 skip_condition : bool or callable
110 114 Flag to determine whether to skip the decorated test.
111 115 msg : str, optional
112 116 Message to give on raising a SkipTest exception. Default is None.
113 117
114 118 Returns
115 119 -------
116 120 decorator : function
117 121 Decorator which, when applied to a function, causes SkipTest
118 122 to be raised when `skip_condition` is True, and the function
119 123 to be called normally otherwise.
120 124
121 125 Notes
122 126 -----
123 127 The decorator itself is decorated with the ``nose.tools.make_decorator``
124 128 function in order to transmit function name, and various other metadata.
125 129
126 130 """
127 131
128 132 def skip_decorator(f):
129 133 # Local import to avoid a hard nose dependency and only incur the
130 134 # import time overhead at actual test-time.
131 135 import nose
132 136
133 137 # Allow for both boolean or callable skip conditions.
134 138 if callable(skip_condition):
135 139 skip_val = lambda : skip_condition()
136 140 else:
137 141 skip_val = lambda : skip_condition
138 142
139 143 def get_msg(func,msg=None):
140 144 """Skip message with information about function being skipped."""
141 145 if msg is None:
142 146 out = 'Test skipped due to test condition'
143 147 else:
144 148 out = '\n'+msg
145 149
146 150 return "Skipping test: %s%s" % (func.__name__,out)
147 151
148 152 # We need to define *two* skippers because Python doesn't allow both
149 153 # return with value and yield inside the same function.
150 154 def skipper_func(*args, **kwargs):
151 155 """Skipper for normal test functions."""
152 156 if skip_val():
153 157 raise nose.SkipTest(get_msg(f,msg))
154 158 else:
155 159 return f(*args, **kwargs)
156 160
157 161 def skipper_gen(*args, **kwargs):
158 162 """Skipper for test generators."""
159 163 if skip_val():
160 164 raise nose.SkipTest(get_msg(f,msg))
161 165 else:
162 166 for x in f(*args, **kwargs):
163 167 yield x
164 168
165 169 # Choose the right skipper to use when building the actual decorator.
166 170 if nose.util.isgenerator(f):
167 171 skipper = skipper_gen
168 172 else:
169 173 skipper = skipper_func
170 174
171 175 return nose.tools.make_decorator(f)(skipper)
172 176
173 177 return skip_decorator
174 178
175
176 179 def knownfailureif(fail_condition, msg=None):
177 180 """
178 181 Make function raise KnownFailureTest exception if given condition is true.
179 182
180 183 If the condition is a callable, it is used at runtime to dynamically
181 184 make the decision. This is useful for tests that may require costly
182 185 imports, to delay the cost until the test suite is actually executed.
183 186
184 187 Parameters
185 188 ----------
186 189 fail_condition : bool or callable
187 190 Flag to determine whether to mark the decorated test as a known
188 191 failure (if True) or not (if False).
189 192 msg : str, optional
190 193 Message to give on raising a KnownFailureTest exception.
191 194 Default is None.
192 195
193 196 Returns
194 197 -------
195 198 decorator : function
196 199 Decorator, which, when applied to a function, causes SkipTest
197 200 to be raised when `skip_condition` is True, and the function
198 201 to be called normally otherwise.
199 202
200 203 Notes
201 204 -----
202 205 The decorator itself is decorated with the ``nose.tools.make_decorator``
203 206 function in order to transmit function name, and various other metadata.
204 207
205 208 """
206 209 if msg is None:
207 210 msg = 'Test skipped due to known failure'
208 211
209 212 # Allow for both boolean or callable known failure conditions.
210 213 if callable(fail_condition):
211 214 fail_val = lambda : fail_condition()
212 215 else:
213 216 fail_val = lambda : fail_condition
214 217
215 218 def knownfail_decorator(f):
216 219 # Local import to avoid a hard nose dependency and only incur the
217 220 # import time overhead at actual test-time.
218 221 import nose
219 from noseclasses import KnownFailureTest
220 222 def knownfailer(*args, **kwargs):
221 223 if fail_val():
222 224 raise KnownFailureTest, msg
223 225 else:
224 226 return f(*args, **kwargs)
225 227 return nose.tools.make_decorator(f)(knownfailer)
226 228
227 229 return knownfail_decorator
228 230
229 231 def deprecated(conditional=True):
230 232 """
231 233 Filter deprecation warnings while running the test suite.
232 234
233 235 This decorator can be used to filter DeprecationWarning's, to avoid
234 236 printing them during the test suite run, while checking that the test
235 237 actually raises a DeprecationWarning.
236 238
237 239 Parameters
238 240 ----------
239 241 conditional : bool or callable, optional
240 242 Flag to determine whether to mark test as deprecated or not. If the
241 243 condition is a callable, it is used at runtime to dynamically make the
242 244 decision. Default is True.
243 245
244 246 Returns
245 247 -------
246 248 decorator : function
247 249 The `deprecated` decorator itself.
248 250
249 251 Notes
250 252 -----
251 253 .. versionadded:: 1.4.0
252 254
253 255 """
254 256 def deprecate_decorator(f):
255 257 # Local import to avoid a hard nose dependency and only incur the
256 258 # import time overhead at actual test-time.
257 259 import nose
258 from noseclasses import KnownFailureTest
259 260
260 261 def _deprecated_imp(*args, **kwargs):
261 262 # Poor man's replacement for the with statement
262 263 ctx = WarningManager(record=True)
263 264 l = ctx.__enter__()
264 265 warnings.simplefilter('always')
265 266 try:
266 267 f(*args, **kwargs)
267 268 if not len(l) > 0:
268 269 raise AssertionError("No warning raised when calling %s"
269 270 % f.__name__)
270 271 if not l[0].category is DeprecationWarning:
271 272 raise AssertionError("First warning for %s is not a " \
272 273 "DeprecationWarning( is %s)" % (f.__name__, l[0]))
273 274 finally:
274 275 ctx.__exit__()
275 276
276 277 if callable(conditional):
277 278 cond = conditional()
278 279 else:
279 280 cond = conditional
280 281 if cond:
281 282 return nose.tools.make_decorator(f)(_deprecated_imp)
282 283 else:
283 284 return f
284 285 return deprecate_decorator
@@ -1,120 +1,116 b''
1 # IPython: modified copy of numpy.testing.utils, so numpy.testing.decorators
2 # works without numpy being installed.
1 # IPython: modified copy of numpy.testing.utils, so
2 # IPython.external._decorators works without numpy being installed.
3 3 """
4 4 Utility function to facilitate testing.
5 5 """
6 6
7 import os
8 7 import sys
9 import re
10 import operator
11 import types
12 8 import warnings
13 9
14 10 # The following two classes are copied from python 2.6 warnings module (context
15 11 # manager)
16 12 class WarningMessage(object):
17 13
18 14 """
19 15 Holds the result of a single showwarning() call.
20 16
21 17 Notes
22 18 -----
23 19 `WarningMessage` is copied from the Python 2.6 warnings module,
24 20 so it can be used in NumPy with older Python versions.
25 21
26 22 """
27 23
28 24 _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
29 25 "line")
30 26
31 27 def __init__(self, message, category, filename, lineno, file=None,
32 28 line=None):
33 29 local_values = locals()
34 30 for attr in self._WARNING_DETAILS:
35 31 setattr(self, attr, local_values[attr])
36 32 if category:
37 33 self._category_name = category.__name__
38 34 else:
39 35 self._category_name = None
40 36
41 37 def __str__(self):
42 38 return ("{message : %r, category : %r, filename : %r, lineno : %s, "
43 39 "line : %r}" % (self.message, self._category_name,
44 40 self.filename, self.lineno, self.line))
45 41
46 42 class WarningManager:
47 43 """
48 44 A context manager that copies and restores the warnings filter upon
49 45 exiting the context.
50 46
51 47 The 'record' argument specifies whether warnings should be captured by a
52 48 custom implementation of ``warnings.showwarning()`` and be appended to a
53 49 list returned by the context manager. Otherwise None is returned by the
54 50 context manager. The objects appended to the list are arguments whose
55 51 attributes mirror the arguments to ``showwarning()``.
56 52
57 53 The 'module' argument is to specify an alternative module to the module
58 54 named 'warnings' and imported under that name. This argument is only useful
59 55 when testing the warnings module itself.
60 56
61 57 Notes
62 58 -----
63 59 `WarningManager` is a copy of the ``catch_warnings`` context manager
64 60 from the Python 2.6 warnings module, with slight modifications.
65 61 It is copied so it can be used in NumPy with older Python versions.
66 62
67 63 """
68 64 def __init__(self, record=False, module=None):
69 65 self._record = record
70 66 if module is None:
71 67 self._module = sys.modules['warnings']
72 68 else:
73 69 self._module = module
74 70 self._entered = False
75 71
76 72 def __enter__(self):
77 73 if self._entered:
78 74 raise RuntimeError("Cannot enter %r twice" % self)
79 75 self._entered = True
80 76 self._filters = self._module.filters
81 77 self._module.filters = self._filters[:]
82 78 self._showwarning = self._module.showwarning
83 79 if self._record:
84 80 log = []
85 81 def showwarning(*args, **kwargs):
86 82 log.append(WarningMessage(*args, **kwargs))
87 83 self._module.showwarning = showwarning
88 84 return log
89 85 else:
90 86 return None
91 87
92 88 def __exit__(self):
93 89 if not self._entered:
94 90 raise RuntimeError("Cannot exit %r without entering first" % self)
95 91 self._module.filters = self._filters
96 92 self._module.showwarning = self._showwarning
97 93
98 94 def assert_warns(warning_class, func, *args, **kw):
99 95 """Fail unless a warning of class warning_class is thrown by callable when
100 96 invoked with arguments args and keyword arguments kwargs.
101 97
102 98 If a different type of warning is thrown, it will not be caught, and the
103 99 test case will be deemed to have suffered an error.
104 100 """
105 101
106 102 # XXX: once we may depend on python >= 2.6, this can be replaced by the
107 103 # warnings module context manager.
108 104 ctx = WarningManager(record=True)
109 105 l = ctx.__enter__()
110 106 warnings.simplefilter('always')
111 107 try:
112 108 func(*args, **kw)
113 109 if not len(l) > 0:
114 110 raise AssertionError("No warning raised when calling %s"
115 111 % func.__name__)
116 112 if not l[0].category is warning_class:
117 113 raise AssertionError("First warning for %s is not a " \
118 114 "%s( is %s)" % (func.__name__, warning_class, l[0]))
119 115 finally:
120 116 ctx.__exit__()
@@ -1,326 +1,327 b''
1 1 """Decorators for labeling test objects.
2 2
3 3 Decorators that merely return a modified version of the original function
4 4 object are straightforward. Decorators that return a new function object need
5 5 to use nose.tools.make_decorator(original_function)(decorator) in returning the
6 6 decorator, in order to preserve metadata such as function name, setup and
7 7 teardown functions and so on - see nose.tools for more information.
8 8
9 9 This module provides a set of useful decorators meant to be ready to use in
10 10 your own tests. See the bottom of the file for the ready-made ones, and if you
11 11 find yourself writing a new one that may be of generic use, add it here.
12 12
13 13 Included decorators:
14 14
15 15
16 16 Lightweight testing that remains unittest-compatible.
17 17
18 18 - @parametric, for parametric test support that is vastly easier to use than
19 19 nose's for debugging. With ours, if a test fails, the stack under inspection
20 20 is that of the test and not that of the test framework.
21 21
22 22 - An @as_unittest decorator can be used to tag any normal parameter-less
23 23 function as a unittest TestCase. Then, both nose and normal unittest will
24 24 recognize it as such. This will make it easier to migrate away from Nose if
25 25 we ever need/want to while maintaining very lightweight tests.
26 26
27 NOTE: This file contains IPython-specific decorators and imports the
28 numpy.testing.decorators file, which we've copied verbatim. Any of our own
29 code will be added at the bottom if we end up extending this.
27 NOTE: This file contains IPython-specific decorators. Using the machinery in
28 IPython.external.decorators, we import either numpy.testing.decorators if numpy is
29 available, OR use equivalent code in IPython.external._decorators, which
30 we've copied verbatim from numpy.
30 31
31 32 Authors
32 33 -------
33 34
34 35 - Fernando Perez <Fernando.Perez@berkeley.edu>
35 36 """
36 37
37 38 #-----------------------------------------------------------------------------
38 39 # Copyright (C) 2009-2010 The IPython Development Team
39 40 #
40 41 # Distributed under the terms of the BSD License. The full license is in
41 42 # the file COPYING, distributed as part of this software.
42 43 #-----------------------------------------------------------------------------
43 44
44 45 #-----------------------------------------------------------------------------
45 46 # Imports
46 47 #-----------------------------------------------------------------------------
47 48
48 49 # Stdlib imports
49 50 import inspect
50 51 import sys
51 52 import unittest
52 53
53 54 # Third-party imports
54 55
55 56 # This is Michele Simionato's decorator module, kept verbatim.
56 57 from IPython.external.decorator import decorator, update_wrapper
57 58
58 59 # We already have python3-compliant code for parametric tests
59 60 if sys.version[0]=='2':
60 61 from _paramtestpy2 import parametric, ParametricTestCase
61 62 else:
62 63 from _paramtestpy3 import parametric, ParametricTestCase
63 64
64 65 # Expose the unittest-driven decorators
65 66 from ipunittest import ipdoctest, ipdocstring
66 67
67 68 # Grab the numpy-specific decorators which we keep in a file that we
68 69 # occasionally update from upstream: decorators.py is a copy of
69 70 # numpy.testing.decorators, we expose all of it here.
70 71 from IPython.external.decorators import *
71 72
72 73 #-----------------------------------------------------------------------------
73 74 # Classes and functions
74 75 #-----------------------------------------------------------------------------
75 76
76 77 # Simple example of the basic idea
77 78 def as_unittest(func):
78 79 """Decorator to make a simple function into a normal test via unittest."""
79 80 class Tester(unittest.TestCase):
80 81 def test(self):
81 82 func()
82 83
83 84 Tester.__name__ = func.__name__
84 85
85 86 return Tester
86 87
87 88 # Utility functions
88 89
89 90 def apply_wrapper(wrapper,func):
90 91 """Apply a wrapper to a function for decoration.
91 92
92 93 This mixes Michele Simionato's decorator tool with nose's make_decorator,
93 94 to apply a wrapper in a decorator so that all nose attributes, as well as
94 95 function signature and other properties, survive the decoration cleanly.
95 96 This will ensure that wrapped functions can still be well introspected via
96 97 IPython, for example.
97 98 """
98 99 import nose.tools
99 100
100 101 return decorator(wrapper,nose.tools.make_decorator(func)(wrapper))
101 102
102 103
103 104 def make_label_dec(label,ds=None):
104 105 """Factory function to create a decorator that applies one or more labels.
105 106
106 107 Parameters
107 108 ----------
108 109 label : string or sequence
109 110 One or more labels that will be applied by the decorator to the functions
110 111 it decorates. Labels are attributes of the decorated function with their
111 112 value set to True.
112 113
113 114 ds : string
114 115 An optional docstring for the resulting decorator. If not given, a
115 116 default docstring is auto-generated.
116 117
117 118 Returns
118 119 -------
119 120 A decorator.
120 121
121 122 Examples
122 123 --------
123 124
124 125 A simple labeling decorator:
125 126 >>> slow = make_label_dec('slow')
126 127 >>> print slow.__doc__
127 128 Labels a test as 'slow'.
128 129
129 130 And one that uses multiple labels and a custom docstring:
130 131 >>> rare = make_label_dec(['slow','hard'],
131 132 ... "Mix labels 'slow' and 'hard' for rare tests.")
132 133 >>> print rare.__doc__
133 134 Mix labels 'slow' and 'hard' for rare tests.
134 135
135 136 Now, let's test using this one:
136 137 >>> @rare
137 138 ... def f(): pass
138 139 ...
139 140 >>>
140 141 >>> f.slow
141 142 True
142 143 >>> f.hard
143 144 True
144 145 """
145 146
146 147 if isinstance(label,basestring):
147 148 labels = [label]
148 149 else:
149 150 labels = label
150 151
151 152 # Validate that the given label(s) are OK for use in setattr() by doing a
152 153 # dry run on a dummy function.
153 154 tmp = lambda : None
154 155 for label in labels:
155 156 setattr(tmp,label,True)
156 157
157 158 # This is the actual decorator we'll return
158 159 def decor(f):
159 160 for label in labels:
160 161 setattr(f,label,True)
161 162 return f
162 163
163 164 # Apply the user's docstring, or autogenerate a basic one
164 165 if ds is None:
165 166 ds = "Labels a test as %r." % label
166 167 decor.__doc__ = ds
167 168
168 169 return decor
169 170
170 171
171 172 # Inspired by numpy's skipif, but uses the full apply_wrapper utility to
172 173 # preserve function metadata better and allows the skip condition to be a
173 174 # callable.
174 175 def skipif(skip_condition, msg=None):
175 176 ''' Make function raise SkipTest exception if skip_condition is true
176 177
177 178 Parameters
178 179 ----------
179 180 skip_condition : bool or callable.
180 181 Flag to determine whether to skip test. If the condition is a
181 182 callable, it is used at runtime to dynamically make the decision. This
182 183 is useful for tests that may require costly imports, to delay the cost
183 184 until the test suite is actually executed.
184 185 msg : string
185 186 Message to give on raising a SkipTest exception
186 187
187 188 Returns
188 189 -------
189 190 decorator : function
190 191 Decorator, which, when applied to a function, causes SkipTest
191 192 to be raised when the skip_condition was True, and the function
192 193 to be called normally otherwise.
193 194
194 195 Notes
195 196 -----
196 197 You will see from the code that we had to further decorate the
197 198 decorator with the nose.tools.make_decorator function in order to
198 199 transmit function name, and various other metadata.
199 200 '''
200 201
201 202 def skip_decorator(f):
202 203 # Local import to avoid a hard nose dependency and only incur the
203 204 # import time overhead at actual test-time.
204 205 import nose
205 206
206 207 # Allow for both boolean or callable skip conditions.
207 208 if callable(skip_condition):
208 209 skip_val = skip_condition
209 210 else:
210 211 skip_val = lambda : skip_condition
211 212
212 213 def get_msg(func,msg=None):
213 214 """Skip message with information about function being skipped."""
214 215 if msg is None: out = 'Test skipped due to test condition.'
215 216 else: out = msg
216 217 return "Skipping test: %s. %s" % (func.__name__,out)
217 218
218 219 # We need to define *two* skippers because Python doesn't allow both
219 220 # return with value and yield inside the same function.
220 221 def skipper_func(*args, **kwargs):
221 222 """Skipper for normal test functions."""
222 223 if skip_val():
223 224 raise nose.SkipTest(get_msg(f,msg))
224 225 else:
225 226 return f(*args, **kwargs)
226 227
227 228 def skipper_gen(*args, **kwargs):
228 229 """Skipper for test generators."""
229 230 if skip_val():
230 231 raise nose.SkipTest(get_msg(f,msg))
231 232 else:
232 233 for x in f(*args, **kwargs):
233 234 yield x
234 235
235 236 # Choose the right skipper to use when building the actual generator.
236 237 if nose.util.isgenerator(f):
237 238 skipper = skipper_gen
238 239 else:
239 240 skipper = skipper_func
240 241
241 242 return nose.tools.make_decorator(f)(skipper)
242 243
243 244 return skip_decorator
244 245
245 246 # A version with the condition set to true, common case just to attacha message
246 247 # to a skip decorator
247 248 def skip(msg=None):
248 249 """Decorator factory - mark a test function for skipping from test suite.
249 250
250 251 Parameters
251 252 ----------
252 253 msg : string
253 254 Optional message to be added.
254 255
255 256 Returns
256 257 -------
257 258 decorator : function
258 259 Decorator, which, when applied to a function, causes SkipTest
259 260 to be raised, with the optional message added.
260 261 """
261 262
262 263 return skipif(True,msg)
263 264
264 265
265 266 def onlyif(condition, msg):
266 267 """The reverse from skipif, see skipif for details."""
267 268
268 269 if callable(condition):
269 270 skip_condition = lambda : not condition()
270 271 else:
271 272 skip_condition = lambda : not condition
272 273
273 274 return skipif(skip_condition, msg)
274 275
275 276 #-----------------------------------------------------------------------------
276 277 # Utility functions for decorators
277 278 def module_not_available(module):
278 279 """Can module be imported? Returns true if module does NOT import.
279 280
280 281 This is used to make a decorator to skip tests that require module to be
281 282 available, but delay the 'import numpy' to test execution time.
282 283 """
283 284 try:
284 285 mod = __import__(module)
285 286 mod_not_avail = False
286 287 except ImportError:
287 288 mod_not_avail = True
288 289
289 290 return mod_not_avail
290 291
291 292 #-----------------------------------------------------------------------------
292 293 # Decorators for public use
293 294
294 295 skip_doctest = make_label_dec('skip_doctest',
295 296 """Decorator - mark a function or method for skipping its doctest.
296 297
297 298 This decorator allows you to mark a function whose docstring you wish to
298 299 omit from testing, while preserving the docstring for introspection, help,
299 300 etc.""")
300 301
301 302 # Decorators to skip certain tests on specific platforms.
302 303 skip_win32 = skipif(sys.platform == 'win32',
303 304 "This test does not run under Windows")
304 305 skip_linux = skipif(sys.platform == 'linux2',
305 306 "This test does not run under Linux")
306 307 skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
307 308
308 309
309 310 # Decorators to skip tests if not on specific platforms.
310 311 skip_if_not_win32 = skipif(sys.platform != 'win32',
311 312 "This test only runs under Windows")
312 313 skip_if_not_linux = skipif(sys.platform != 'linux2',
313 314 "This test only runs under Linux")
314 315 skip_if_not_osx = skipif(sys.platform != 'darwin',
315 316 "This test only runs under OSX")
316 317
317 318 # Other skip decorators
318 319 skipif_not_numpy = skipif(module_not_available('numpy'),"This test requires numpy")
319 320
320 321 skipif_not_sympy = skipif(module_not_available('sympy'),"This test requires sympy")
321 322
322 skip_known_failure = skip('This test is known to fail')
323 skip_known_failure = knownfailureif(True,'This test is known to fail')
323 324
324 325 # A null 'decorator', useful to make more readable code that needs to pick
325 326 # between different decorators based on OS or other conditions
326 327 null_deco = lambda f: f
@@ -1,441 +1,442 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) or trial recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 For now, this script requires that both nose and twisted are installed. This
16 16 will change in the future.
17 17 """
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Copyright (C) 2009 The IPython Development Team
21 21 #
22 22 # Distributed under the terms of the BSD License. The full license is in
23 23 # the file COPYING, distributed as part of this software.
24 24 #-----------------------------------------------------------------------------
25 25
26 26 #-----------------------------------------------------------------------------
27 27 # Imports
28 28 #-----------------------------------------------------------------------------
29 29
30 30 # Stdlib
31 31 import os
32 32 import os.path as path
33 33 import signal
34 34 import sys
35 35 import subprocess
36 36 import tempfile
37 37 import time
38 38 import warnings
39 39
40 40 # Note: monkeypatch!
41 41 # We need to monkeypatch a small problem in nose itself first, before importing
42 42 # it for actual use. This should get into nose upstream, but its release cycle
43 43 # is slow and we need it for our parametric tests to work correctly.
44 44 from IPython.testing import nosepatch
45 45 # Now, proceed to import nose itself
46 46 import nose.plugins.builtin
47 47 from nose.core import TestProgram
48 48
49 49 # Our own imports
50 50 from IPython.utils.path import get_ipython_module_path
51 51 from IPython.utils.process import find_cmd, pycmd2argv
52 52 from IPython.utils.sysinfo import sys_info
53 53
54 54 from IPython.testing import globalipapp
55 55 from IPython.testing.plugin.ipdoctest import IPythonDoctest
56 from IPython.external.decorators import KnownFailure
56 57
57 58 pjoin = path.join
58 59
59 60
60 61 #-----------------------------------------------------------------------------
61 62 # Globals
62 63 #-----------------------------------------------------------------------------
63 64
64 65
65 66 #-----------------------------------------------------------------------------
66 67 # Warnings control
67 68 #-----------------------------------------------------------------------------
68 69
69 70 # Twisted generates annoying warnings with Python 2.6, as will do other code
70 71 # that imports 'sets' as of today
71 72 warnings.filterwarnings('ignore', 'the sets module is deprecated',
72 73 DeprecationWarning )
73 74
74 75 # This one also comes from Twisted
75 76 warnings.filterwarnings('ignore', 'the sha module is deprecated',
76 77 DeprecationWarning)
77 78
78 79 # Wx on Fedora11 spits these out
79 80 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
80 81 UserWarning)
81 82
82 83 #-----------------------------------------------------------------------------
83 84 # Logic for skipping doctests
84 85 #-----------------------------------------------------------------------------
85 86
86 87 def test_for(mod):
87 88 """Test to see if mod is importable."""
88 89 try:
89 90 __import__(mod)
90 91 except (ImportError, RuntimeError):
91 92 # GTK reports Runtime error if it can't be initialized even if it's
92 93 # importable.
93 94 return False
94 95 else:
95 96 return True
96 97
97 98 # Global dict where we can store information on what we have and what we don't
98 99 # have available at test run time
99 100 have = {}
100 101
101 102 have['curses'] = test_for('_curses')
102 103 have['wx'] = test_for('wx')
103 104 have['wx.aui'] = test_for('wx.aui')
104 105 have['zope.interface'] = test_for('zope.interface')
105 106 have['twisted'] = test_for('twisted')
106 107 have['foolscap'] = test_for('foolscap')
107 108 have['pexpect'] = test_for('pexpect')
108 109
109 110 #-----------------------------------------------------------------------------
110 111 # Functions and classes
111 112 #-----------------------------------------------------------------------------
112 113
113 114 def report():
114 115 """Return a string with a summary report of test-related variables."""
115 116
116 117 out = [ sys_info(), '\n']
117 118
118 119 avail = []
119 120 not_avail = []
120 121
121 122 for k, is_avail in have.items():
122 123 if is_avail:
123 124 avail.append(k)
124 125 else:
125 126 not_avail.append(k)
126 127
127 128 if avail:
128 129 out.append('\nTools and libraries available at test time:\n')
129 130 avail.sort()
130 131 out.append(' ' + ' '.join(avail)+'\n')
131 132
132 133 if not_avail:
133 134 out.append('\nTools and libraries NOT available at test time:\n')
134 135 not_avail.sort()
135 136 out.append(' ' + ' '.join(not_avail)+'\n')
136 137
137 138 return ''.join(out)
138 139
139 140
140 141 def make_exclude():
141 142 """Make patterns of modules and packages to exclude from testing.
142 143
143 144 For the IPythonDoctest plugin, we need to exclude certain patterns that
144 145 cause testing problems. We should strive to minimize the number of
145 146 skipped modules, since this means untested code.
146 147
147 148 These modules and packages will NOT get scanned by nose at all for tests.
148 149 """
149 150 # Simple utility to make IPython paths more readably, we need a lot of
150 151 # these below
151 152 ipjoin = lambda *paths: pjoin('IPython', *paths)
152 153
153 154 exclusions = [ipjoin('external'),
154 155 pjoin('IPython_doctest_plugin'),
155 156 ipjoin('quarantine'),
156 157 ipjoin('deathrow'),
157 158 ipjoin('testing', 'attic'),
158 159 # This guy is probably attic material
159 160 ipjoin('testing', 'mkdoctests'),
160 161 # Testing inputhook will need a lot of thought, to figure out
161 162 # how to have tests that don't lock up with the gui event
162 163 # loops in the picture
163 164 ipjoin('lib', 'inputhook'),
164 165 # Config files aren't really importable stand-alone
165 166 ipjoin('config', 'default'),
166 167 ipjoin('config', 'profile'),
167 168 ]
168 169
169 170 if not have['wx']:
170 171 exclusions.append(ipjoin('lib', 'inputhookwx'))
171 172
172 173 # We do this unconditionally, so that the test suite doesn't import
173 174 # gtk, changing the default encoding and masking some unicode bugs.
174 175 exclusions.append(ipjoin('lib', 'inputhookgtk'))
175 176
176 177 # These have to be skipped on win32 because the use echo, rm, cd, etc.
177 178 # See ticket https://bugs.launchpad.net/bugs/366982
178 179 if sys.platform == 'win32':
179 180 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
180 181 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
181 182
182 183 if not have['pexpect']:
183 184 exclusions.extend([ipjoin('scripts', 'irunner'),
184 185 ipjoin('lib', 'irunner')])
185 186
186 187 # This is scary. We still have things in frontend and testing that
187 188 # are being tested by nose that use twisted. We need to rethink
188 189 # how we are isolating dependencies in testing.
189 190 if not (have['twisted'] and have['zope.interface'] and have['foolscap']):
190 191 exclusions.extend(
191 192 [ipjoin('testing', 'parametric'),
192 193 ipjoin('testing', 'util'),
193 194 ipjoin('testing', 'tests', 'test_decorators_trial'),
194 195 ] )
195 196
196 197 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
197 198 if sys.platform == 'win32':
198 199 exclusions = [s.replace('\\','\\\\') for s in exclusions]
199 200
200 201 return exclusions
201 202
202 203
203 204 class IPTester(object):
204 205 """Call that calls iptest or trial in a subprocess.
205 206 """
206 207 #: string, name of test runner that will be called
207 208 runner = None
208 209 #: list, parameters for test runner
209 210 params = None
210 211 #: list, arguments of system call to be made to call test runner
211 212 call_args = None
212 213 #: list, process ids of subprocesses we start (for cleanup)
213 214 pids = None
214 215
215 216 def __init__(self, runner='iptest', params=None):
216 217 """Create new test runner."""
217 218 p = os.path
218 219 if runner == 'iptest':
219 220 iptest_app = get_ipython_module_path('IPython.testing.iptest')
220 221 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
221 222 elif runner == 'trial':
222 223 # For trial, it needs to be installed system-wide
223 224 self.runner = pycmd2argv(p.abspath(find_cmd('trial')))
224 225 else:
225 226 raise Exception('Not a valid test runner: %s' % repr(runner))
226 227 if params is None:
227 228 params = []
228 229 if isinstance(params, str):
229 230 params = [params]
230 231 self.params = params
231 232
232 233 # Assemble call
233 234 self.call_args = self.runner+self.params
234 235
235 236 # Store pids of anything we start to clean up on deletion, if possible
236 237 # (on posix only, since win32 has no os.kill)
237 238 self.pids = []
238 239
239 240 if sys.platform == 'win32':
240 241 def _run_cmd(self):
241 242 # On Windows, use os.system instead of subprocess.call, because I
242 243 # was having problems with subprocess and I just don't know enough
243 244 # about win32 to debug this reliably. Os.system may be the 'old
244 245 # fashioned' way to do it, but it works just fine. If someone
245 246 # later can clean this up that's fine, as long as the tests run
246 247 # reliably in win32.
247 248 # What types of problems are you having. They may be related to
248 249 # running Python in unboffered mode. BG.
249 250 return os.system(' '.join(self.call_args))
250 251 else:
251 252 def _run_cmd(self):
252 253 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
253 254 subp = subprocess.Popen(self.call_args)
254 255 self.pids.append(subp.pid)
255 256 # If this fails, the pid will be left in self.pids and cleaned up
256 257 # later, but if the wait call succeeds, then we can clear the
257 258 # stored pid.
258 259 retcode = subp.wait()
259 260 self.pids.pop()
260 261 return retcode
261 262
262 263 def run(self):
263 264 """Run the stored commands"""
264 265 try:
265 266 return self._run_cmd()
266 267 except:
267 268 import traceback
268 269 traceback.print_exc()
269 270 return 1 # signal failure
270 271
271 272 def __del__(self):
272 273 """Cleanup on exit by killing any leftover processes."""
273 274
274 275 if not hasattr(os, 'kill'):
275 276 return
276 277
277 278 for pid in self.pids:
278 279 try:
279 280 print 'Cleaning stale PID:', pid
280 281 os.kill(pid, signal.SIGKILL)
281 282 except OSError:
282 283 # This is just a best effort, if we fail or the process was
283 284 # really gone, ignore it.
284 285 pass
285 286
286 287
287 288 def make_runners():
288 289 """Define the top-level packages that need to be tested.
289 290 """
290 291
291 292 # Packages to be tested via nose, that only depend on the stdlib
292 293 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
293 294 'scripts', 'testing', 'utils' ]
294 295 # The machinery in kernel needs twisted for real testing
295 296 trial_pkg_names = []
296 297
297 298 # And add twisted ones if conditions are met
298 299 if have['zope.interface'] and have['twisted'] and have['foolscap']:
299 300 # We only list IPython.kernel for testing using twisted.trial as
300 301 # nose and twisted.trial have conflicts that make the testing system
301 302 # unstable.
302 303 trial_pkg_names.append('kernel')
303 304
304 305 # For debugging this code, only load quick stuff
305 306 #nose_pkg_names = ['core', 'extensions'] # dbg
306 307 #trial_pkg_names = [] # dbg
307 308
308 309 # Make fully qualified package names prepending 'IPython.' to our name lists
309 310 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
310 311 trial_packages = ['IPython.%s' % m for m in trial_pkg_names ]
311 312
312 313 # Make runners
313 314 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
314 315 runners.extend([ (v, IPTester('trial', params=v)) for v in trial_packages ])
315 316
316 317 return runners
317 318
318 319
319 320 def run_iptest():
320 321 """Run the IPython test suite using nose.
321 322
322 323 This function is called when this script is **not** called with the form
323 324 `iptest all`. It simply calls nose with appropriate command line flags
324 325 and accepts all of the standard nose arguments.
325 326 """
326 327
327 328 warnings.filterwarnings('ignore',
328 329 'This will be removed soon. Use IPython.testing.util instead')
329 330
330 331 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
331 332
332 333 # Loading ipdoctest causes problems with Twisted, but
333 334 # our test suite runner now separates things and runs
334 335 # all Twisted tests with trial.
335 336 '--with-ipdoctest',
336 337 '--ipdoctest-tests','--ipdoctest-extension=txt',
337 338
338 339 # We add --exe because of setuptools' imbecility (it
339 340 # blindly does chmod +x on ALL files). Nose does the
340 341 # right thing and it tries to avoid executables,
341 342 # setuptools unfortunately forces our hand here. This
342 343 # has been discussed on the distutils list and the
343 344 # setuptools devs refuse to fix this problem!
344 345 '--exe',
345 346 ]
346 347
347 348 if nose.__version__ >= '0.11':
348 349 # I don't fully understand why we need this one, but depending on what
349 350 # directory the test suite is run from, if we don't give it, 0 tests
350 351 # get run. Specifically, if the test suite is run from the source dir
351 352 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
352 353 # even if the same call done in this directory works fine). It appears
353 354 # that if the requested package is in the current dir, nose bails early
354 355 # by default. Since it's otherwise harmless, leave it in by default
355 356 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
356 357 argv.append('--traverse-namespace')
357 358
358 359 # Construct list of plugins, omitting the existing doctest plugin, which
359 360 # ours replaces (and extends).
360 plugins = [IPythonDoctest(make_exclude())]
361 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
361 362 for p in nose.plugins.builtin.plugins:
362 363 plug = p()
363 364 if plug.name == 'doctest':
364 365 continue
365 366 plugins.append(plug)
366 367
367 368 # We need a global ipython running in this process
368 369 globalipapp.start_ipython()
369 370 # Now nose can run
370 371 TestProgram(argv=argv, plugins=plugins)
371 372
372 373
373 374 def run_iptestall():
374 375 """Run the entire IPython test suite by calling nose and trial.
375 376
376 377 This function constructs :class:`IPTester` instances for all IPython
377 378 modules and package and then runs each of them. This causes the modules
378 379 and packages of IPython to be tested each in their own subprocess using
379 380 nose or twisted.trial appropriately.
380 381 """
381 382
382 383 runners = make_runners()
383 384
384 385 # Run the test runners in a temporary dir so we can nuke it when finished
385 386 # to clean up any junk files left over by accident. This also makes it
386 387 # robust against being run in non-writeable directories by mistake, as the
387 388 # temp dir will always be user-writeable.
388 389 curdir = os.getcwd()
389 390 testdir = tempfile.gettempdir()
390 391 os.chdir(testdir)
391 392
392 393 # Run all test runners, tracking execution time
393 394 failed = []
394 395 t_start = time.time()
395 396 try:
396 397 for (name, runner) in runners:
397 398 print '*'*70
398 399 print 'IPython test group:',name
399 400 res = runner.run()
400 401 if res:
401 402 failed.append( (name, runner) )
402 403 finally:
403 404 os.chdir(curdir)
404 405 t_end = time.time()
405 406 t_tests = t_end - t_start
406 407 nrunners = len(runners)
407 408 nfail = len(failed)
408 409 # summarize results
409 410 print
410 411 print '*'*70
411 412 print 'Test suite completed for system with the following information:'
412 413 print report()
413 414 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
414 415 print
415 416 print 'Status:'
416 417 if not failed:
417 418 print 'OK'
418 419 else:
419 420 # If anything went wrong, point out what command to rerun manually to
420 421 # see the actual errors and individual summary
421 422 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
422 423 for name, failed_runner in failed:
423 424 print '-'*40
424 425 print 'Runner failed:',name
425 426 print 'You may wish to rerun this one individually, with:'
426 427 print ' '.join(failed_runner.call_args)
427 428 print
428 429
429 430
430 431 def main():
431 432 for arg in sys.argv[1:]:
432 433 if arg.startswith('IPython'):
433 434 # This is in-process
434 435 run_iptest()
435 436 else:
436 437 # This starts subprocesses
437 438 run_iptestall()
438 439
439 440
440 441 if __name__ == '__main__':
441 442 main()
General Comments 0
You need to be logged in to leave comments. Login now