##// END OF EJS Templates
Make separate function to monkeypatch Xunit.
Thomas Kluyver -
Show More
@@ -1,524 +1,523 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 # Stdlib
28 # Stdlib
29 import os
29 import os
30 import os.path as path
30 import os.path as path
31 import signal
31 import signal
32 import sys
32 import sys
33 import subprocess
33 import subprocess
34 import tempfile
34 import tempfile
35 import time
35 import time
36 import warnings
36 import warnings
37
37
38 # Note: monkeypatch!
38 # Note: monkeypatch!
39 # We need to monkeypatch a small problem in nose itself first, before importing
39 # We need to monkeypatch a small problem in nose itself first, before importing
40 # it for actual use. This should get into nose upstream, but its release cycle
40 # it for actual use. This should get into nose upstream, but its release cycle
41 # is slow and we need it for our parametric tests to work correctly.
41 # is slow and we need it for our parametric tests to work correctly.
42 from IPython.testing import nosepatch
42 from IPython.testing import nosepatch
43 # Now, proceed to import nose itself
43 # Now, proceed to import nose itself
44 import nose.plugins.builtin
44 import nose.plugins.builtin
45 from nose.plugins.xunit import Xunit
45 from nose.plugins.xunit import Xunit
46 from nose import SkipTest
46 from nose import SkipTest
47 from nose.core import TestProgram
47 from nose.core import TestProgram
48
48
49 # Our own imports
49 # Our own imports
50 from IPython.utils.importstring import import_item
50 from IPython.utils.importstring import import_item
51 from IPython.utils.path import get_ipython_module_path
51 from IPython.utils.path import get_ipython_module_path
52 from IPython.utils.process import find_cmd, pycmd2argv
52 from IPython.utils.process import find_cmd, pycmd2argv
53 from IPython.utils.sysinfo import sys_info
53 from IPython.utils.sysinfo import sys_info
54
54
55 from IPython.testing import globalipapp
55 from IPython.testing import globalipapp
56 from IPython.testing.plugin.ipdoctest import IPythonDoctest
56 from IPython.testing.plugin.ipdoctest import IPythonDoctest
57 from IPython.external.decorators import KnownFailure, knownfailureif
57 from IPython.external.decorators import KnownFailure, knownfailureif
58
58
59 pjoin = path.join
59 pjoin = path.join
60
60
61
61
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63 # Globals
63 # Globals
64 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
65
65
66
66
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68 # Warnings control
68 # Warnings control
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70
70
71 # Twisted generates annoying warnings with Python 2.6, as will do other code
71 # Twisted generates annoying warnings with Python 2.6, as will do other code
72 # that imports 'sets' as of today
72 # that imports 'sets' as of today
73 warnings.filterwarnings('ignore', 'the sets module is deprecated',
73 warnings.filterwarnings('ignore', 'the sets module is deprecated',
74 DeprecationWarning )
74 DeprecationWarning )
75
75
76 # This one also comes from Twisted
76 # This one also comes from Twisted
77 warnings.filterwarnings('ignore', 'the sha module is deprecated',
77 warnings.filterwarnings('ignore', 'the sha module is deprecated',
78 DeprecationWarning)
78 DeprecationWarning)
79
79
80 # Wx on Fedora11 spits these out
80 # Wx on Fedora11 spits these out
81 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
81 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
82 UserWarning)
82 UserWarning)
83
83
84 # ------------------------------------------------------------------------------
84 # ------------------------------------------------------------------------------
85 # Monkeypatch Xunit to count known failures as skipped.
85 # Monkeypatch Xunit to count known failures as skipped.
86 # ------------------------------------------------------------------------------
86 # ------------------------------------------------------------------------------
87 if not hasattr(Xunit, 'orig_addError'):
87 def monkeypatch_xunit():
88 try:
88 try:
89 knownfailureif(True)(lambda: None)()
89 knownfailureif(True)(lambda: None)()
90 except Exception as e:
90 except Exception as e:
91 KnownFailureTest = type(e)
91 KnownFailureTest = type(e)
92
92
93
94 def addError(self, test, err, capt=None):
93 def addError(self, test, err, capt=None):
95 if issubclass(err[0], KnownFailureTest):
94 if issubclass(err[0], KnownFailureTest):
96 err = (SkipTest,) + err[1:]
95 err = (SkipTest,) + err[1:]
97 return self.orig_addError(test, err, capt)
96 return self.orig_addError(test, err, capt)
98
97
99 #xunit.Xunit = Xunit
100
101
102 Xunit.orig_addError = Xunit.addError
98 Xunit.orig_addError = Xunit.addError
103 Xunit.addError = addError
99 Xunit.addError = addError
104
100
105 #-----------------------------------------------------------------------------
101 #-----------------------------------------------------------------------------
106 # Logic for skipping doctests
102 # Logic for skipping doctests
107 #-----------------------------------------------------------------------------
103 #-----------------------------------------------------------------------------
108 def extract_version(mod):
104 def extract_version(mod):
109 return mod.__version__
105 return mod.__version__
110
106
111 def test_for(item, min_version=None, callback=extract_version):
107 def test_for(item, min_version=None, callback=extract_version):
112 """Test to see if item is importable, and optionally check against a minimum
108 """Test to see if item is importable, and optionally check against a minimum
113 version.
109 version.
114
110
115 If min_version is given, the default behavior is to check against the
111 If min_version is given, the default behavior is to check against the
116 `__version__` attribute of the item, but specifying `callback` allows you to
112 `__version__` attribute of the item, but specifying `callback` allows you to
117 extract the value you are interested in. e.g::
113 extract the value you are interested in. e.g::
118
114
119 In [1]: import sys
115 In [1]: import sys
120
116
121 In [2]: from IPython.testing.iptest import test_for
117 In [2]: from IPython.testing.iptest import test_for
122
118
123 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
119 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
124 Out[3]: True
120 Out[3]: True
125
121
126 """
122 """
127 try:
123 try:
128 check = import_item(item)
124 check = import_item(item)
129 except (ImportError, RuntimeError):
125 except (ImportError, RuntimeError):
130 # GTK reports Runtime error if it can't be initialized even if it's
126 # GTK reports Runtime error if it can't be initialized even if it's
131 # importable.
127 # importable.
132 return False
128 return False
133 else:
129 else:
134 if min_version:
130 if min_version:
135 if callback:
131 if callback:
136 # extra processing step to get version to compare
132 # extra processing step to get version to compare
137 check = callback(check)
133 check = callback(check)
138
134
139 return check >= min_version
135 return check >= min_version
140 else:
136 else:
141 return True
137 return True
142
138
143 # Global dict where we can store information on what we have and what we don't
139 # Global dict where we can store information on what we have and what we don't
144 # have available at test run time
140 # have available at test run time
145 have = {}
141 have = {}
146
142
147 have['curses'] = test_for('_curses')
143 have['curses'] = test_for('_curses')
148 have['matplotlib'] = test_for('matplotlib')
144 have['matplotlib'] = test_for('matplotlib')
149 have['pexpect'] = test_for('IPython.external.pexpect')
145 have['pexpect'] = test_for('IPython.external.pexpect')
150 have['pymongo'] = test_for('pymongo')
146 have['pymongo'] = test_for('pymongo')
151 have['wx'] = test_for('wx')
147 have['wx'] = test_for('wx')
152 have['wx.aui'] = test_for('wx.aui')
148 have['wx.aui'] = test_for('wx.aui')
153 have['qt'] = test_for('IPython.external.qt')
149 have['qt'] = test_for('IPython.external.qt')
154 have['sqlite3'] = test_for('sqlite3')
150 have['sqlite3'] = test_for('sqlite3')
155
151
156 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
152 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
157
153
158 if os.name == 'nt':
154 if os.name == 'nt':
159 min_zmq = (2,1,7)
155 min_zmq = (2,1,7)
160 else:
156 else:
161 min_zmq = (2,1,4)
157 min_zmq = (2,1,4)
162
158
163 def version_tuple(mod):
159 def version_tuple(mod):
164 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
160 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
165 # turn 'dev' into 999, because Python3 rejects str-int comparisons
161 # turn 'dev' into 999, because Python3 rejects str-int comparisons
166 vs = mod.__version__.replace('dev', '.999')
162 vs = mod.__version__.replace('dev', '.999')
167 tup = tuple([int(v) for v in vs.split('.') ])
163 tup = tuple([int(v) for v in vs.split('.') ])
168 return tup
164 return tup
169
165
170 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
166 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
171
167
172 #-----------------------------------------------------------------------------
168 #-----------------------------------------------------------------------------
173 # Functions and classes
169 # Functions and classes
174 #-----------------------------------------------------------------------------
170 #-----------------------------------------------------------------------------
175
171
176 def report():
172 def report():
177 """Return a string with a summary report of test-related variables."""
173 """Return a string with a summary report of test-related variables."""
178
174
179 out = [ sys_info(), '\n']
175 out = [ sys_info(), '\n']
180
176
181 avail = []
177 avail = []
182 not_avail = []
178 not_avail = []
183
179
184 for k, is_avail in have.items():
180 for k, is_avail in have.items():
185 if is_avail:
181 if is_avail:
186 avail.append(k)
182 avail.append(k)
187 else:
183 else:
188 not_avail.append(k)
184 not_avail.append(k)
189
185
190 if avail:
186 if avail:
191 out.append('\nTools and libraries available at test time:\n')
187 out.append('\nTools and libraries available at test time:\n')
192 avail.sort()
188 avail.sort()
193 out.append(' ' + ' '.join(avail)+'\n')
189 out.append(' ' + ' '.join(avail)+'\n')
194
190
195 if not_avail:
191 if not_avail:
196 out.append('\nTools and libraries NOT available at test time:\n')
192 out.append('\nTools and libraries NOT available at test time:\n')
197 not_avail.sort()
193 not_avail.sort()
198 out.append(' ' + ' '.join(not_avail)+'\n')
194 out.append(' ' + ' '.join(not_avail)+'\n')
199
195
200 return ''.join(out)
196 return ''.join(out)
201
197
202
198
203 def make_exclude():
199 def make_exclude():
204 """Make patterns of modules and packages to exclude from testing.
200 """Make patterns of modules and packages to exclude from testing.
205
201
206 For the IPythonDoctest plugin, we need to exclude certain patterns that
202 For the IPythonDoctest plugin, we need to exclude certain patterns that
207 cause testing problems. We should strive to minimize the number of
203 cause testing problems. We should strive to minimize the number of
208 skipped modules, since this means untested code.
204 skipped modules, since this means untested code.
209
205
210 These modules and packages will NOT get scanned by nose at all for tests.
206 These modules and packages will NOT get scanned by nose at all for tests.
211 """
207 """
212 # Simple utility to make IPython paths more readably, we need a lot of
208 # Simple utility to make IPython paths more readably, we need a lot of
213 # these below
209 # these below
214 ipjoin = lambda *paths: pjoin('IPython', *paths)
210 ipjoin = lambda *paths: pjoin('IPython', *paths)
215
211
216 exclusions = [ipjoin('external'),
212 exclusions = [ipjoin('external'),
217 pjoin('IPython_doctest_plugin'),
213 pjoin('IPython_doctest_plugin'),
218 ipjoin('quarantine'),
214 ipjoin('quarantine'),
219 ipjoin('deathrow'),
215 ipjoin('deathrow'),
220 ipjoin('testing', 'attic'),
216 ipjoin('testing', 'attic'),
221 # This guy is probably attic material
217 # This guy is probably attic material
222 ipjoin('testing', 'mkdoctests'),
218 ipjoin('testing', 'mkdoctests'),
223 # Testing inputhook will need a lot of thought, to figure out
219 # Testing inputhook will need a lot of thought, to figure out
224 # how to have tests that don't lock up with the gui event
220 # how to have tests that don't lock up with the gui event
225 # loops in the picture
221 # loops in the picture
226 ipjoin('lib', 'inputhook'),
222 ipjoin('lib', 'inputhook'),
227 # Config files aren't really importable stand-alone
223 # Config files aren't really importable stand-alone
228 ipjoin('config', 'default'),
224 ipjoin('config', 'default'),
229 ipjoin('config', 'profile'),
225 ipjoin('config', 'profile'),
230 ]
226 ]
231 if not have['sqlite3']:
227 if not have['sqlite3']:
232 exclusions.append(ipjoin('core', 'tests', 'test_history'))
228 exclusions.append(ipjoin('core', 'tests', 'test_history'))
233 exclusions.append(ipjoin('core', 'history'))
229 exclusions.append(ipjoin('core', 'history'))
234 if not have['wx']:
230 if not have['wx']:
235 exclusions.append(ipjoin('lib', 'inputhookwx'))
231 exclusions.append(ipjoin('lib', 'inputhookwx'))
236
232
237 # We do this unconditionally, so that the test suite doesn't import
233 # We do this unconditionally, so that the test suite doesn't import
238 # gtk, changing the default encoding and masking some unicode bugs.
234 # gtk, changing the default encoding and masking some unicode bugs.
239 exclusions.append(ipjoin('lib', 'inputhookgtk'))
235 exclusions.append(ipjoin('lib', 'inputhookgtk'))
240
236
241 # These have to be skipped on win32 because the use echo, rm, cd, etc.
237 # These have to be skipped on win32 because the use echo, rm, cd, etc.
242 # See ticket https://github.com/ipython/ipython/issues/87
238 # See ticket https://github.com/ipython/ipython/issues/87
243 if sys.platform == 'win32':
239 if sys.platform == 'win32':
244 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
240 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
245 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
241 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
246
242
247 if not have['pexpect']:
243 if not have['pexpect']:
248 exclusions.extend([ipjoin('scripts', 'irunner'),
244 exclusions.extend([ipjoin('scripts', 'irunner'),
249 ipjoin('lib', 'irunner'),
245 ipjoin('lib', 'irunner'),
250 ipjoin('lib', 'tests', 'test_irunner'),
246 ipjoin('lib', 'tests', 'test_irunner'),
251 ipjoin('frontend', 'terminal', 'console'),
247 ipjoin('frontend', 'terminal', 'console'),
252 ])
248 ])
253
249
254 if not have['zmq']:
250 if not have['zmq']:
255 exclusions.append(ipjoin('zmq'))
251 exclusions.append(ipjoin('zmq'))
256 exclusions.append(ipjoin('frontend', 'qt'))
252 exclusions.append(ipjoin('frontend', 'qt'))
257 exclusions.append(ipjoin('frontend', 'html'))
253 exclusions.append(ipjoin('frontend', 'html'))
258 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
254 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
259 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
255 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
260 exclusions.append(ipjoin('parallel'))
256 exclusions.append(ipjoin('parallel'))
261 elif not have['qt']:
257 elif not have['qt']:
262 exclusions.append(ipjoin('frontend', 'qt'))
258 exclusions.append(ipjoin('frontend', 'qt'))
263
259
264 if not have['pymongo']:
260 if not have['pymongo']:
265 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
261 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
266 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
262 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
267
263
268 if not have['matplotlib']:
264 if not have['matplotlib']:
269 exclusions.extend([ipjoin('core', 'pylabtools'),
265 exclusions.extend([ipjoin('core', 'pylabtools'),
270 ipjoin('core', 'tests', 'test_pylabtools')])
266 ipjoin('core', 'tests', 'test_pylabtools')])
271
267
272 if not have['tornado']:
268 if not have['tornado']:
273 exclusions.append(ipjoin('frontend', 'html'))
269 exclusions.append(ipjoin('frontend', 'html'))
274
270
275 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
271 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
276 if sys.platform == 'win32':
272 if sys.platform == 'win32':
277 exclusions = [s.replace('\\','\\\\') for s in exclusions]
273 exclusions = [s.replace('\\','\\\\') for s in exclusions]
278
274
279 return exclusions
275 return exclusions
280
276
281
277
282 class IPTester(object):
278 class IPTester(object):
283 """Call that calls iptest or trial in a subprocess.
279 """Call that calls iptest or trial in a subprocess.
284 """
280 """
285 #: string, name of test runner that will be called
281 #: string, name of test runner that will be called
286 runner = None
282 runner = None
287 #: list, parameters for test runner
283 #: list, parameters for test runner
288 params = None
284 params = None
289 #: list, arguments of system call to be made to call test runner
285 #: list, arguments of system call to be made to call test runner
290 call_args = None
286 call_args = None
291 #: list, process ids of subprocesses we start (for cleanup)
287 #: list, process ids of subprocesses we start (for cleanup)
292 pids = None
288 pids = None
293 #: str, coverage xml output file
289 #: str, coverage xml output file
294 coverage_xml = None
290 coverage_xml = None
295
291
296 def __init__(self, runner='iptest', params=None):
292 def __init__(self, runner='iptest', params=None):
297 """Create new test runner."""
293 """Create new test runner."""
298 p = os.path
294 p = os.path
299 if runner == 'iptest':
295 if runner == 'iptest':
300 iptest_app = get_ipython_module_path('IPython.testing.iptest')
296 iptest_app = get_ipython_module_path('IPython.testing.iptest')
301 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
297 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
302 else:
298 else:
303 raise Exception('Not a valid test runner: %s' % repr(runner))
299 raise Exception('Not a valid test runner: %s' % repr(runner))
304 if params is None:
300 if params is None:
305 params = []
301 params = []
306 if isinstance(params, str):
302 if isinstance(params, str):
307 params = [params]
303 params = [params]
308 self.params = params
304 self.params = params
309
305
310 # Assemble call
306 # Assemble call
311 self.call_args = self.runner+self.params
307 self.call_args = self.runner+self.params
312
308
313 # Find the section we're testing (IPython.foo)
309 # Find the section we're testing (IPython.foo)
314 for sect in self.params:
310 for sect in self.params:
315 if sect.startswith('IPython'): break
311 if sect.startswith('IPython'): break
316 else:
312 else:
317 raise ValueError("Section not found", self.params)
313 raise ValueError("Section not found", self.params)
318
314
319 if '--with-xunit' in self.call_args:
315 if '--with-xunit' in self.call_args:
320 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
316 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
321
317
322 if '--with-xml-coverage' in self.call_args:
318 if '--with-xml-coverage' in self.call_args:
323 self.coverage_xml = path.abspath(sect+".coverage.xml")
319 self.coverage_xml = path.abspath(sect+".coverage.xml")
324 self.call_args.remove('--with-xml-coverage')
320 self.call_args.remove('--with-xml-coverage')
325 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
321 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
326
322
327 # Store pids of anything we start to clean up on deletion, if possible
323 # Store pids of anything we start to clean up on deletion, if possible
328 # (on posix only, since win32 has no os.kill)
324 # (on posix only, since win32 has no os.kill)
329 self.pids = []
325 self.pids = []
330
326
331 if sys.platform == 'win32':
327 if sys.platform == 'win32':
332 def _run_cmd(self):
328 def _run_cmd(self):
333 # On Windows, use os.system instead of subprocess.call, because I
329 # On Windows, use os.system instead of subprocess.call, because I
334 # was having problems with subprocess and I just don't know enough
330 # was having problems with subprocess and I just don't know enough
335 # about win32 to debug this reliably. Os.system may be the 'old
331 # about win32 to debug this reliably. Os.system may be the 'old
336 # fashioned' way to do it, but it works just fine. If someone
332 # fashioned' way to do it, but it works just fine. If someone
337 # later can clean this up that's fine, as long as the tests run
333 # later can clean this up that's fine, as long as the tests run
338 # reliably in win32.
334 # reliably in win32.
339 # What types of problems are you having. They may be related to
335 # What types of problems are you having. They may be related to
340 # running Python in unboffered mode. BG.
336 # running Python in unboffered mode. BG.
341 return os.system(' '.join(self.call_args))
337 return os.system(' '.join(self.call_args))
342 else:
338 else:
343 def _run_cmd(self):
339 def _run_cmd(self):
344 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
340 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
345 subp = subprocess.Popen(self.call_args)
341 subp = subprocess.Popen(self.call_args)
346 self.pids.append(subp.pid)
342 self.pids.append(subp.pid)
347 # If this fails, the pid will be left in self.pids and cleaned up
343 # If this fails, the pid will be left in self.pids and cleaned up
348 # later, but if the wait call succeeds, then we can clear the
344 # later, but if the wait call succeeds, then we can clear the
349 # stored pid.
345 # stored pid.
350 retcode = subp.wait()
346 retcode = subp.wait()
351 self.pids.pop()
347 self.pids.pop()
352 return retcode
348 return retcode
353
349
354 def run(self):
350 def run(self):
355 """Run the stored commands"""
351 """Run the stored commands"""
356 try:
352 try:
357 retcode = self._run_cmd()
353 retcode = self._run_cmd()
358 except:
354 except:
359 import traceback
355 import traceback
360 traceback.print_exc()
356 traceback.print_exc()
361 return 1 # signal failure
357 return 1 # signal failure
362
358
363 if self.coverage_xml:
359 if self.coverage_xml:
364 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
360 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
365 return retcode
361 return retcode
366
362
367 def __del__(self):
363 def __del__(self):
368 """Cleanup on exit by killing any leftover processes."""
364 """Cleanup on exit by killing any leftover processes."""
369
365
370 if not hasattr(os, 'kill'):
366 if not hasattr(os, 'kill'):
371 return
367 return
372
368
373 for pid in self.pids:
369 for pid in self.pids:
374 try:
370 try:
375 print 'Cleaning stale PID:', pid
371 print 'Cleaning stale PID:', pid
376 os.kill(pid, signal.SIGKILL)
372 os.kill(pid, signal.SIGKILL)
377 except OSError:
373 except OSError:
378 # This is just a best effort, if we fail or the process was
374 # This is just a best effort, if we fail or the process was
379 # really gone, ignore it.
375 # really gone, ignore it.
380 pass
376 pass
381
377
382
378
383 def make_runners():
379 def make_runners():
384 """Define the top-level packages that need to be tested.
380 """Define the top-level packages that need to be tested.
385 """
381 """
386
382
387 # Packages to be tested via nose, that only depend on the stdlib
383 # Packages to be tested via nose, that only depend on the stdlib
388 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
384 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
389 'scripts', 'testing', 'utils', 'nbformat' ]
385 'scripts', 'testing', 'utils', 'nbformat' ]
390
386
391 if have['zmq']:
387 if have['zmq']:
392 nose_pkg_names.append('parallel')
388 nose_pkg_names.append('parallel')
393
389
394 # For debugging this code, only load quick stuff
390 # For debugging this code, only load quick stuff
395 #nose_pkg_names = ['core', 'extensions'] # dbg
391 #nose_pkg_names = ['core', 'extensions'] # dbg
396
392
397 # Make fully qualified package names prepending 'IPython.' to our name lists
393 # Make fully qualified package names prepending 'IPython.' to our name lists
398 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
394 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
399
395
400 # Make runners
396 # Make runners
401 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
397 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
402
398
403 return runners
399 return runners
404
400
405
401
406 def run_iptest():
402 def run_iptest():
407 """Run the IPython test suite using nose.
403 """Run the IPython test suite using nose.
408
404
409 This function is called when this script is **not** called with the form
405 This function is called when this script is **not** called with the form
410 `iptest all`. It simply calls nose with appropriate command line flags
406 `iptest all`. It simply calls nose with appropriate command line flags
411 and accepts all of the standard nose arguments.
407 and accepts all of the standard nose arguments.
412 """
408 """
409 # Apply our monkeypatch to Xunit
410 if not hasattr(Xunit, 'orig_addError'):
411 monkeypatch_xunit()
413
412
414 warnings.filterwarnings('ignore',
413 warnings.filterwarnings('ignore',
415 'This will be removed soon. Use IPython.testing.util instead')
414 'This will be removed soon. Use IPython.testing.util instead')
416
415
417 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
416 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
418
417
419 # Loading ipdoctest causes problems with Twisted, but
418 # Loading ipdoctest causes problems with Twisted, but
420 # our test suite runner now separates things and runs
419 # our test suite runner now separates things and runs
421 # all Twisted tests with trial.
420 # all Twisted tests with trial.
422 '--with-ipdoctest',
421 '--with-ipdoctest',
423 '--ipdoctest-tests','--ipdoctest-extension=txt',
422 '--ipdoctest-tests','--ipdoctest-extension=txt',
424
423
425 # We add --exe because of setuptools' imbecility (it
424 # We add --exe because of setuptools' imbecility (it
426 # blindly does chmod +x on ALL files). Nose does the
425 # blindly does chmod +x on ALL files). Nose does the
427 # right thing and it tries to avoid executables,
426 # right thing and it tries to avoid executables,
428 # setuptools unfortunately forces our hand here. This
427 # setuptools unfortunately forces our hand here. This
429 # has been discussed on the distutils list and the
428 # has been discussed on the distutils list and the
430 # setuptools devs refuse to fix this problem!
429 # setuptools devs refuse to fix this problem!
431 '--exe',
430 '--exe',
432 ]
431 ]
433
432
434 if nose.__version__ >= '0.11':
433 if nose.__version__ >= '0.11':
435 # I don't fully understand why we need this one, but depending on what
434 # I don't fully understand why we need this one, but depending on what
436 # directory the test suite is run from, if we don't give it, 0 tests
435 # directory the test suite is run from, if we don't give it, 0 tests
437 # get run. Specifically, if the test suite is run from the source dir
436 # get run. Specifically, if the test suite is run from the source dir
438 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
437 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
439 # even if the same call done in this directory works fine). It appears
438 # even if the same call done in this directory works fine). It appears
440 # that if the requested package is in the current dir, nose bails early
439 # that if the requested package is in the current dir, nose bails early
441 # by default. Since it's otherwise harmless, leave it in by default
440 # by default. Since it's otherwise harmless, leave it in by default
442 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
441 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
443 argv.append('--traverse-namespace')
442 argv.append('--traverse-namespace')
444
443
445 # use our plugin for doctesting. It will remove the standard doctest plugin
444 # use our plugin for doctesting. It will remove the standard doctest plugin
446 # if it finds it enabled
445 # if it finds it enabled
447 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
446 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
448 # We need a global ipython running in this process
447 # We need a global ipython running in this process
449 globalipapp.start_ipython()
448 globalipapp.start_ipython()
450 # Now nose can run
449 # Now nose can run
451 TestProgram(argv=argv, addplugins=plugins)
450 TestProgram(argv=argv, addplugins=plugins)
452
451
453
452
454 def run_iptestall():
453 def run_iptestall():
455 """Run the entire IPython test suite by calling nose and trial.
454 """Run the entire IPython test suite by calling nose and trial.
456
455
457 This function constructs :class:`IPTester` instances for all IPython
456 This function constructs :class:`IPTester` instances for all IPython
458 modules and package and then runs each of them. This causes the modules
457 modules and package and then runs each of them. This causes the modules
459 and packages of IPython to be tested each in their own subprocess using
458 and packages of IPython to be tested each in their own subprocess using
460 nose or twisted.trial appropriately.
459 nose or twisted.trial appropriately.
461 """
460 """
462
461
463 runners = make_runners()
462 runners = make_runners()
464
463
465 # Run the test runners in a temporary dir so we can nuke it when finished
464 # Run the test runners in a temporary dir so we can nuke it when finished
466 # to clean up any junk files left over by accident. This also makes it
465 # to clean up any junk files left over by accident. This also makes it
467 # robust against being run in non-writeable directories by mistake, as the
466 # robust against being run in non-writeable directories by mistake, as the
468 # temp dir will always be user-writeable.
467 # temp dir will always be user-writeable.
469 curdir = os.getcwdu()
468 curdir = os.getcwdu()
470 testdir = tempfile.gettempdir()
469 testdir = tempfile.gettempdir()
471 os.chdir(testdir)
470 os.chdir(testdir)
472
471
473 # Run all test runners, tracking execution time
472 # Run all test runners, tracking execution time
474 failed = []
473 failed = []
475 t_start = time.time()
474 t_start = time.time()
476 try:
475 try:
477 for (name, runner) in runners:
476 for (name, runner) in runners:
478 print '*'*70
477 print '*'*70
479 print 'IPython test group:',name
478 print 'IPython test group:',name
480 res = runner.run()
479 res = runner.run()
481 if res:
480 if res:
482 failed.append( (name, runner) )
481 failed.append( (name, runner) )
483 finally:
482 finally:
484 os.chdir(curdir)
483 os.chdir(curdir)
485 t_end = time.time()
484 t_end = time.time()
486 t_tests = t_end - t_start
485 t_tests = t_end - t_start
487 nrunners = len(runners)
486 nrunners = len(runners)
488 nfail = len(failed)
487 nfail = len(failed)
489 # summarize results
488 # summarize results
490 print
489 print
491 print '*'*70
490 print '*'*70
492 print 'Test suite completed for system with the following information:'
491 print 'Test suite completed for system with the following information:'
493 print report()
492 print report()
494 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
493 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
495 print
494 print
496 print 'Status:'
495 print 'Status:'
497 if not failed:
496 if not failed:
498 print 'OK'
497 print 'OK'
499 else:
498 else:
500 # If anything went wrong, point out what command to rerun manually to
499 # If anything went wrong, point out what command to rerun manually to
501 # see the actual errors and individual summary
500 # see the actual errors and individual summary
502 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
501 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
503 for name, failed_runner in failed:
502 for name, failed_runner in failed:
504 print '-'*40
503 print '-'*40
505 print 'Runner failed:',name
504 print 'Runner failed:',name
506 print 'You may wish to rerun this one individually, with:'
505 print 'You may wish to rerun this one individually, with:'
507 print ' '.join(failed_runner.call_args)
506 print ' '.join(failed_runner.call_args)
508 print
507 print
509 # Ensure that our exit code indicates failure
508 # Ensure that our exit code indicates failure
510 sys.exit(1)
509 sys.exit(1)
511
510
512
511
513 def main():
512 def main():
514 for arg in sys.argv[1:]:
513 for arg in sys.argv[1:]:
515 if arg.startswith('IPython'):
514 if arg.startswith('IPython'):
516 # This is in-process
515 # This is in-process
517 run_iptest()
516 run_iptest()
518 else:
517 else:
519 # This starts subprocesses
518 # This starts subprocesses
520 run_iptestall()
519 run_iptestall()
521
520
522
521
523 if __name__ == '__main__':
522 if __name__ == '__main__':
524 main()
523 main()
General Comments 0
You need to be logged in to leave comments. Login now