##// END OF EJS Templates
Pass subprocess test runners a suitable location for xunit output.
Thomas Kluyver -
Show More
@@ -1,481 +1,486 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 # Stdlib
28 # Stdlib
29 import os
29 import os
30 import os.path as path
30 import os.path as path
31 import signal
31 import signal
32 import sys
32 import sys
33 import subprocess
33 import subprocess
34 import tempfile
34 import tempfile
35 import time
35 import time
36 import warnings
36 import warnings
37
37
38 # Note: monkeypatch!
38 # Note: monkeypatch!
39 # We need to monkeypatch a small problem in nose itself first, before importing
39 # We need to monkeypatch a small problem in nose itself first, before importing
40 # it for actual use. This should get into nose upstream, but its release cycle
40 # it for actual use. This should get into nose upstream, but its release cycle
41 # is slow and we need it for our parametric tests to work correctly.
41 # is slow and we need it for our parametric tests to work correctly.
42 from IPython.testing import nosepatch
42 from IPython.testing import nosepatch
43 # Now, proceed to import nose itself
43 # Now, proceed to import nose itself
44 import nose.plugins.builtin
44 import nose.plugins.builtin
45 from nose.core import TestProgram
45 from nose.core import TestProgram
46
46
47 # Our own imports
47 # Our own imports
48 from IPython.utils.importstring import import_item
48 from IPython.utils.importstring import import_item
49 from IPython.utils.path import get_ipython_module_path
49 from IPython.utils.path import get_ipython_module_path
50 from IPython.utils.process import find_cmd, pycmd2argv
50 from IPython.utils.process import find_cmd, pycmd2argv
51 from IPython.utils.sysinfo import sys_info
51 from IPython.utils.sysinfo import sys_info
52
52
53 from IPython.testing import globalipapp
53 from IPython.testing import globalipapp
54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
55 from IPython.external.decorators import KnownFailure
55 from IPython.external.decorators import KnownFailure
56
56
57 pjoin = path.join
57 pjoin = path.join
58
58
59
59
60 #-----------------------------------------------------------------------------
60 #-----------------------------------------------------------------------------
61 # Globals
61 # Globals
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63
63
64
64
65 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
66 # Warnings control
66 # Warnings control
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68
68
69 # Twisted generates annoying warnings with Python 2.6, as will do other code
69 # Twisted generates annoying warnings with Python 2.6, as will do other code
70 # that imports 'sets' as of today
70 # that imports 'sets' as of today
71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
72 DeprecationWarning )
72 DeprecationWarning )
73
73
74 # This one also comes from Twisted
74 # This one also comes from Twisted
75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
76 DeprecationWarning)
76 DeprecationWarning)
77
77
78 # Wx on Fedora11 spits these out
78 # Wx on Fedora11 spits these out
79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
80 UserWarning)
80 UserWarning)
81
81
82 #-----------------------------------------------------------------------------
82 #-----------------------------------------------------------------------------
83 # Logic for skipping doctests
83 # Logic for skipping doctests
84 #-----------------------------------------------------------------------------
84 #-----------------------------------------------------------------------------
85 def extract_version(mod):
85 def extract_version(mod):
86 return mod.__version__
86 return mod.__version__
87
87
88 def test_for(item, min_version=None, callback=extract_version):
88 def test_for(item, min_version=None, callback=extract_version):
89 """Test to see if item is importable, and optionally check against a minimum
89 """Test to see if item is importable, and optionally check against a minimum
90 version.
90 version.
91
91
92 If min_version is given, the default behavior is to check against the
92 If min_version is given, the default behavior is to check against the
93 `__version__` attribute of the item, but specifying `callback` allows you to
93 `__version__` attribute of the item, but specifying `callback` allows you to
94 extract the value you are interested in. e.g::
94 extract the value you are interested in. e.g::
95
95
96 In [1]: import sys
96 In [1]: import sys
97
97
98 In [2]: from IPython.testing.iptest import test_for
98 In [2]: from IPython.testing.iptest import test_for
99
99
100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
101 Out[3]: True
101 Out[3]: True
102
102
103 """
103 """
104 try:
104 try:
105 check = import_item(item)
105 check = import_item(item)
106 except (ImportError, RuntimeError):
106 except (ImportError, RuntimeError):
107 # GTK reports Runtime error if it can't be initialized even if it's
107 # GTK reports Runtime error if it can't be initialized even if it's
108 # importable.
108 # importable.
109 return False
109 return False
110 else:
110 else:
111 if min_version:
111 if min_version:
112 if callback:
112 if callback:
113 # extra processing step to get version to compare
113 # extra processing step to get version to compare
114 check = callback(check)
114 check = callback(check)
115
115
116 return check >= min_version
116 return check >= min_version
117 else:
117 else:
118 return True
118 return True
119
119
120 # Global dict where we can store information on what we have and what we don't
120 # Global dict where we can store information on what we have and what we don't
121 # have available at test run time
121 # have available at test run time
122 have = {}
122 have = {}
123
123
124 have['curses'] = test_for('_curses')
124 have['curses'] = test_for('_curses')
125 have['matplotlib'] = test_for('matplotlib')
125 have['matplotlib'] = test_for('matplotlib')
126 have['pexpect'] = test_for('IPython.external.pexpect')
126 have['pexpect'] = test_for('IPython.external.pexpect')
127 have['pymongo'] = test_for('pymongo')
127 have['pymongo'] = test_for('pymongo')
128 have['wx'] = test_for('wx')
128 have['wx'] = test_for('wx')
129 have['wx.aui'] = test_for('wx.aui')
129 have['wx.aui'] = test_for('wx.aui')
130 have['qt'] = test_for('IPython.external.qt')
130 have['qt'] = test_for('IPython.external.qt')
131 have['sqlite3'] = test_for('sqlite3')
131 have['sqlite3'] = test_for('sqlite3')
132
132
133 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
133 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
134
134
135 if os.name == 'nt':
135 if os.name == 'nt':
136 min_zmq = (2,1,7)
136 min_zmq = (2,1,7)
137 else:
137 else:
138 min_zmq = (2,1,4)
138 min_zmq = (2,1,4)
139
139
140 def version_tuple(mod):
140 def version_tuple(mod):
141 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
141 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
142 # turn 'dev' into 999, because Python3 rejects str-int comparisons
142 # turn 'dev' into 999, because Python3 rejects str-int comparisons
143 vs = mod.__version__.replace('dev', '.999')
143 vs = mod.__version__.replace('dev', '.999')
144 tup = tuple([int(v) for v in vs.split('.') ])
144 tup = tuple([int(v) for v in vs.split('.') ])
145 return tup
145 return tup
146
146
147 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
147 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
148
148
149 #-----------------------------------------------------------------------------
149 #-----------------------------------------------------------------------------
150 # Functions and classes
150 # Functions and classes
151 #-----------------------------------------------------------------------------
151 #-----------------------------------------------------------------------------
152
152
153 def report():
153 def report():
154 """Return a string with a summary report of test-related variables."""
154 """Return a string with a summary report of test-related variables."""
155
155
156 out = [ sys_info(), '\n']
156 out = [ sys_info(), '\n']
157
157
158 avail = []
158 avail = []
159 not_avail = []
159 not_avail = []
160
160
161 for k, is_avail in have.items():
161 for k, is_avail in have.items():
162 if is_avail:
162 if is_avail:
163 avail.append(k)
163 avail.append(k)
164 else:
164 else:
165 not_avail.append(k)
165 not_avail.append(k)
166
166
167 if avail:
167 if avail:
168 out.append('\nTools and libraries available at test time:\n')
168 out.append('\nTools and libraries available at test time:\n')
169 avail.sort()
169 avail.sort()
170 out.append(' ' + ' '.join(avail)+'\n')
170 out.append(' ' + ' '.join(avail)+'\n')
171
171
172 if not_avail:
172 if not_avail:
173 out.append('\nTools and libraries NOT available at test time:\n')
173 out.append('\nTools and libraries NOT available at test time:\n')
174 not_avail.sort()
174 not_avail.sort()
175 out.append(' ' + ' '.join(not_avail)+'\n')
175 out.append(' ' + ' '.join(not_avail)+'\n')
176
176
177 return ''.join(out)
177 return ''.join(out)
178
178
179
179
180 def make_exclude():
180 def make_exclude():
181 """Make patterns of modules and packages to exclude from testing.
181 """Make patterns of modules and packages to exclude from testing.
182
182
183 For the IPythonDoctest plugin, we need to exclude certain patterns that
183 For the IPythonDoctest plugin, we need to exclude certain patterns that
184 cause testing problems. We should strive to minimize the number of
184 cause testing problems. We should strive to minimize the number of
185 skipped modules, since this means untested code.
185 skipped modules, since this means untested code.
186
186
187 These modules and packages will NOT get scanned by nose at all for tests.
187 These modules and packages will NOT get scanned by nose at all for tests.
188 """
188 """
189 # Simple utility to make IPython paths more readably, we need a lot of
189 # Simple utility to make IPython paths more readably, we need a lot of
190 # these below
190 # these below
191 ipjoin = lambda *paths: pjoin('IPython', *paths)
191 ipjoin = lambda *paths: pjoin('IPython', *paths)
192
192
193 exclusions = [ipjoin('external'),
193 exclusions = [ipjoin('external'),
194 pjoin('IPython_doctest_plugin'),
194 pjoin('IPython_doctest_plugin'),
195 ipjoin('quarantine'),
195 ipjoin('quarantine'),
196 ipjoin('deathrow'),
196 ipjoin('deathrow'),
197 ipjoin('testing', 'attic'),
197 ipjoin('testing', 'attic'),
198 # This guy is probably attic material
198 # This guy is probably attic material
199 ipjoin('testing', 'mkdoctests'),
199 ipjoin('testing', 'mkdoctests'),
200 # Testing inputhook will need a lot of thought, to figure out
200 # Testing inputhook will need a lot of thought, to figure out
201 # how to have tests that don't lock up with the gui event
201 # how to have tests that don't lock up with the gui event
202 # loops in the picture
202 # loops in the picture
203 ipjoin('lib', 'inputhook'),
203 ipjoin('lib', 'inputhook'),
204 # Config files aren't really importable stand-alone
204 # Config files aren't really importable stand-alone
205 ipjoin('config', 'default'),
205 ipjoin('config', 'default'),
206 ipjoin('config', 'profile'),
206 ipjoin('config', 'profile'),
207 ]
207 ]
208 if not have['sqlite3']:
208 if not have['sqlite3']:
209 exclusions.append(ipjoin('core', 'tests', 'test_history'))
209 exclusions.append(ipjoin('core', 'tests', 'test_history'))
210 exclusions.append(ipjoin('core', 'history'))
210 exclusions.append(ipjoin('core', 'history'))
211 if not have['wx']:
211 if not have['wx']:
212 exclusions.append(ipjoin('lib', 'inputhookwx'))
212 exclusions.append(ipjoin('lib', 'inputhookwx'))
213
213
214 # We do this unconditionally, so that the test suite doesn't import
214 # We do this unconditionally, so that the test suite doesn't import
215 # gtk, changing the default encoding and masking some unicode bugs.
215 # gtk, changing the default encoding and masking some unicode bugs.
216 exclusions.append(ipjoin('lib', 'inputhookgtk'))
216 exclusions.append(ipjoin('lib', 'inputhookgtk'))
217
217
218 # These have to be skipped on win32 because the use echo, rm, cd, etc.
218 # These have to be skipped on win32 because the use echo, rm, cd, etc.
219 # See ticket https://github.com/ipython/ipython/issues/87
219 # See ticket https://github.com/ipython/ipython/issues/87
220 if sys.platform == 'win32':
220 if sys.platform == 'win32':
221 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
221 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
222 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
222 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
223
223
224 if not have['pexpect']:
224 if not have['pexpect']:
225 exclusions.extend([ipjoin('scripts', 'irunner'),
225 exclusions.extend([ipjoin('scripts', 'irunner'),
226 ipjoin('lib', 'irunner'),
226 ipjoin('lib', 'irunner'),
227 ipjoin('lib', 'tests', 'test_irunner'),
227 ipjoin('lib', 'tests', 'test_irunner'),
228 ipjoin('frontend', 'terminal', 'console'),
228 ipjoin('frontend', 'terminal', 'console'),
229 ])
229 ])
230
230
231 if not have['zmq']:
231 if not have['zmq']:
232 exclusions.append(ipjoin('zmq'))
232 exclusions.append(ipjoin('zmq'))
233 exclusions.append(ipjoin('frontend', 'qt'))
233 exclusions.append(ipjoin('frontend', 'qt'))
234 exclusions.append(ipjoin('frontend', 'html'))
234 exclusions.append(ipjoin('frontend', 'html'))
235 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
235 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
236 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
236 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
237 exclusions.append(ipjoin('parallel'))
237 exclusions.append(ipjoin('parallel'))
238 elif not have['qt']:
238 elif not have['qt']:
239 exclusions.append(ipjoin('frontend', 'qt'))
239 exclusions.append(ipjoin('frontend', 'qt'))
240
240
241 if not have['pymongo']:
241 if not have['pymongo']:
242 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
242 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
243 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
243 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
244
244
245 if not have['matplotlib']:
245 if not have['matplotlib']:
246 exclusions.extend([ipjoin('core', 'pylabtools'),
246 exclusions.extend([ipjoin('core', 'pylabtools'),
247 ipjoin('core', 'tests', 'test_pylabtools')])
247 ipjoin('core', 'tests', 'test_pylabtools')])
248
248
249 if not have['tornado']:
249 if not have['tornado']:
250 exclusions.append(ipjoin('frontend', 'html'))
250 exclusions.append(ipjoin('frontend', 'html'))
251
251
252 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
252 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
253 if sys.platform == 'win32':
253 if sys.platform == 'win32':
254 exclusions = [s.replace('\\','\\\\') for s in exclusions]
254 exclusions = [s.replace('\\','\\\\') for s in exclusions]
255
255
256 return exclusions
256 return exclusions
257
257
258
258
259 class IPTester(object):
259 class IPTester(object):
260 """Call that calls iptest or trial in a subprocess.
260 """Call that calls iptest or trial in a subprocess.
261 """
261 """
262 #: string, name of test runner that will be called
262 #: string, name of test runner that will be called
263 runner = None
263 runner = None
264 #: list, parameters for test runner
264 #: list, parameters for test runner
265 params = None
265 params = None
266 #: list, arguments of system call to be made to call test runner
266 #: list, arguments of system call to be made to call test runner
267 call_args = None
267 call_args = None
268 #: list, process ids of subprocesses we start (for cleanup)
268 #: list, process ids of subprocesses we start (for cleanup)
269 pids = None
269 pids = None
270
270
271 def __init__(self, runner='iptest', params=None):
271 def __init__(self, runner='iptest', params=None):
272 """Create new test runner."""
272 """Create new test runner."""
273 p = os.path
273 p = os.path
274 if runner == 'iptest':
274 if runner == 'iptest':
275 iptest_app = get_ipython_module_path('IPython.testing.iptest')
275 iptest_app = get_ipython_module_path('IPython.testing.iptest')
276 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
276 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
277 else:
277 else:
278 raise Exception('Not a valid test runner: %s' % repr(runner))
278 raise Exception('Not a valid test runner: %s' % repr(runner))
279 if params is None:
279 if params is None:
280 params = []
280 params = []
281 if isinstance(params, str):
281 if isinstance(params, str):
282 params = [params]
282 params = [params]
283 self.params = params
283 self.params = params
284
284
285 # Assemble call
285 # Assemble call
286 self.call_args = self.runner+self.params
286 self.call_args = self.runner+self.params
287
288 print self.call_args
289 if '--with-xunit' in self.call_args:
290 sect = [p for p in self.params if p.startswith('IPython')][0]
291 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
287
292
288 # Store pids of anything we start to clean up on deletion, if possible
293 # Store pids of anything we start to clean up on deletion, if possible
289 # (on posix only, since win32 has no os.kill)
294 # (on posix only, since win32 has no os.kill)
290 self.pids = []
295 self.pids = []
291
296
292 if sys.platform == 'win32':
297 if sys.platform == 'win32':
293 def _run_cmd(self):
298 def _run_cmd(self):
294 # On Windows, use os.system instead of subprocess.call, because I
299 # On Windows, use os.system instead of subprocess.call, because I
295 # was having problems with subprocess and I just don't know enough
300 # was having problems with subprocess and I just don't know enough
296 # about win32 to debug this reliably. Os.system may be the 'old
301 # about win32 to debug this reliably. Os.system may be the 'old
297 # fashioned' way to do it, but it works just fine. If someone
302 # fashioned' way to do it, but it works just fine. If someone
298 # later can clean this up that's fine, as long as the tests run
303 # later can clean this up that's fine, as long as the tests run
299 # reliably in win32.
304 # reliably in win32.
300 # What types of problems are you having. They may be related to
305 # What types of problems are you having. They may be related to
301 # running Python in unboffered mode. BG.
306 # running Python in unboffered mode. BG.
302 return os.system(' '.join(self.call_args))
307 return os.system(' '.join(self.call_args))
303 else:
308 else:
304 def _run_cmd(self):
309 def _run_cmd(self):
305 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
310 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
306 subp = subprocess.Popen(self.call_args)
311 subp = subprocess.Popen(self.call_args)
307 self.pids.append(subp.pid)
312 self.pids.append(subp.pid)
308 # If this fails, the pid will be left in self.pids and cleaned up
313 # If this fails, the pid will be left in self.pids and cleaned up
309 # later, but if the wait call succeeds, then we can clear the
314 # later, but if the wait call succeeds, then we can clear the
310 # stored pid.
315 # stored pid.
311 retcode = subp.wait()
316 retcode = subp.wait()
312 self.pids.pop()
317 self.pids.pop()
313 return retcode
318 return retcode
314
319
315 def run(self):
320 def run(self):
316 """Run the stored commands"""
321 """Run the stored commands"""
317 try:
322 try:
318 return self._run_cmd()
323 return self._run_cmd()
319 except:
324 except:
320 import traceback
325 import traceback
321 traceback.print_exc()
326 traceback.print_exc()
322 return 1 # signal failure
327 return 1 # signal failure
323
328
324 def __del__(self):
329 def __del__(self):
325 """Cleanup on exit by killing any leftover processes."""
330 """Cleanup on exit by killing any leftover processes."""
326
331
327 if not hasattr(os, 'kill'):
332 if not hasattr(os, 'kill'):
328 return
333 return
329
334
330 for pid in self.pids:
335 for pid in self.pids:
331 try:
336 try:
332 print 'Cleaning stale PID:', pid
337 print 'Cleaning stale PID:', pid
333 os.kill(pid, signal.SIGKILL)
338 os.kill(pid, signal.SIGKILL)
334 except OSError:
339 except OSError:
335 # This is just a best effort, if we fail or the process was
340 # This is just a best effort, if we fail or the process was
336 # really gone, ignore it.
341 # really gone, ignore it.
337 pass
342 pass
338
343
339
344
340 def make_runners():
345 def make_runners():
341 """Define the top-level packages that need to be tested.
346 """Define the top-level packages that need to be tested.
342 """
347 """
343
348
344 # Packages to be tested via nose, that only depend on the stdlib
349 # Packages to be tested via nose, that only depend on the stdlib
345 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
350 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
346 'scripts', 'testing', 'utils', 'nbformat' ]
351 'scripts', 'testing', 'utils', 'nbformat' ]
347
352
348 if have['zmq']:
353 if have['zmq']:
349 nose_pkg_names.append('parallel')
354 nose_pkg_names.append('parallel')
350
355
351 # For debugging this code, only load quick stuff
356 # For debugging this code, only load quick stuff
352 #nose_pkg_names = ['core', 'extensions'] # dbg
357 #nose_pkg_names = ['core', 'extensions'] # dbg
353
358
354 # Make fully qualified package names prepending 'IPython.' to our name lists
359 # Make fully qualified package names prepending 'IPython.' to our name lists
355 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
360 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
356
361
357 # Make runners
362 # Make runners
358 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
363 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
359
364
360 return runners
365 return runners
361
366
362
367
363 def run_iptest():
368 def run_iptest():
364 """Run the IPython test suite using nose.
369 """Run the IPython test suite using nose.
365
370
366 This function is called when this script is **not** called with the form
371 This function is called when this script is **not** called with the form
367 `iptest all`. It simply calls nose with appropriate command line flags
372 `iptest all`. It simply calls nose with appropriate command line flags
368 and accepts all of the standard nose arguments.
373 and accepts all of the standard nose arguments.
369 """
374 """
370
375
371 warnings.filterwarnings('ignore',
376 warnings.filterwarnings('ignore',
372 'This will be removed soon. Use IPython.testing.util instead')
377 'This will be removed soon. Use IPython.testing.util instead')
373
378
374 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
379 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
375
380
376 # Loading ipdoctest causes problems with Twisted, but
381 # Loading ipdoctest causes problems with Twisted, but
377 # our test suite runner now separates things and runs
382 # our test suite runner now separates things and runs
378 # all Twisted tests with trial.
383 # all Twisted tests with trial.
379 '--with-ipdoctest',
384 '--with-ipdoctest',
380 '--ipdoctest-tests','--ipdoctest-extension=txt',
385 '--ipdoctest-tests','--ipdoctest-extension=txt',
381
386
382 # We add --exe because of setuptools' imbecility (it
387 # We add --exe because of setuptools' imbecility (it
383 # blindly does chmod +x on ALL files). Nose does the
388 # blindly does chmod +x on ALL files). Nose does the
384 # right thing and it tries to avoid executables,
389 # right thing and it tries to avoid executables,
385 # setuptools unfortunately forces our hand here. This
390 # setuptools unfortunately forces our hand here. This
386 # has been discussed on the distutils list and the
391 # has been discussed on the distutils list and the
387 # setuptools devs refuse to fix this problem!
392 # setuptools devs refuse to fix this problem!
388 '--exe',
393 '--exe',
389 ]
394 ]
390
395
391 if nose.__version__ >= '0.11':
396 if nose.__version__ >= '0.11':
392 # I don't fully understand why we need this one, but depending on what
397 # I don't fully understand why we need this one, but depending on what
393 # directory the test suite is run from, if we don't give it, 0 tests
398 # directory the test suite is run from, if we don't give it, 0 tests
394 # get run. Specifically, if the test suite is run from the source dir
399 # get run. Specifically, if the test suite is run from the source dir
395 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
400 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
396 # even if the same call done in this directory works fine). It appears
401 # even if the same call done in this directory works fine). It appears
397 # that if the requested package is in the current dir, nose bails early
402 # that if the requested package is in the current dir, nose bails early
398 # by default. Since it's otherwise harmless, leave it in by default
403 # by default. Since it's otherwise harmless, leave it in by default
399 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
404 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
400 argv.append('--traverse-namespace')
405 argv.append('--traverse-namespace')
401
406
402 # use our plugin for doctesting. It will remove the standard doctest plugin
407 # use our plugin for doctesting. It will remove the standard doctest plugin
403 # if it finds it enabled
408 # if it finds it enabled
404 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
409 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
405 # We need a global ipython running in this process
410 # We need a global ipython running in this process
406 globalipapp.start_ipython()
411 globalipapp.start_ipython()
407 # Now nose can run
412 # Now nose can run
408 TestProgram(argv=argv, addplugins=plugins)
413 TestProgram(argv=argv, addplugins=plugins)
409
414
410
415
411 def run_iptestall():
416 def run_iptestall():
412 """Run the entire IPython test suite by calling nose and trial.
417 """Run the entire IPython test suite by calling nose and trial.
413
418
414 This function constructs :class:`IPTester` instances for all IPython
419 This function constructs :class:`IPTester` instances for all IPython
415 modules and package and then runs each of them. This causes the modules
420 modules and package and then runs each of them. This causes the modules
416 and packages of IPython to be tested each in their own subprocess using
421 and packages of IPython to be tested each in their own subprocess using
417 nose or twisted.trial appropriately.
422 nose or twisted.trial appropriately.
418 """
423 """
419
424
420 runners = make_runners()
425 runners = make_runners()
421
426
422 # Run the test runners in a temporary dir so we can nuke it when finished
427 # Run the test runners in a temporary dir so we can nuke it when finished
423 # to clean up any junk files left over by accident. This also makes it
428 # to clean up any junk files left over by accident. This also makes it
424 # robust against being run in non-writeable directories by mistake, as the
429 # robust against being run in non-writeable directories by mistake, as the
425 # temp dir will always be user-writeable.
430 # temp dir will always be user-writeable.
426 curdir = os.getcwdu()
431 curdir = os.getcwdu()
427 testdir = tempfile.gettempdir()
432 testdir = tempfile.gettempdir()
428 os.chdir(testdir)
433 os.chdir(testdir)
429
434
430 # Run all test runners, tracking execution time
435 # Run all test runners, tracking execution time
431 failed = []
436 failed = []
432 t_start = time.time()
437 t_start = time.time()
433 try:
438 try:
434 for (name, runner) in runners:
439 for (name, runner) in runners:
435 print '*'*70
440 print '*'*70
436 print 'IPython test group:',name
441 print 'IPython test group:',name
437 res = runner.run()
442 res = runner.run()
438 if res:
443 if res:
439 failed.append( (name, runner) )
444 failed.append( (name, runner) )
440 finally:
445 finally:
441 os.chdir(curdir)
446 os.chdir(curdir)
442 t_end = time.time()
447 t_end = time.time()
443 t_tests = t_end - t_start
448 t_tests = t_end - t_start
444 nrunners = len(runners)
449 nrunners = len(runners)
445 nfail = len(failed)
450 nfail = len(failed)
446 # summarize results
451 # summarize results
447 print
452 print
448 print '*'*70
453 print '*'*70
449 print 'Test suite completed for system with the following information:'
454 print 'Test suite completed for system with the following information:'
450 print report()
455 print report()
451 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
456 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
452 print
457 print
453 print 'Status:'
458 print 'Status:'
454 if not failed:
459 if not failed:
455 print 'OK'
460 print 'OK'
456 else:
461 else:
457 # If anything went wrong, point out what command to rerun manually to
462 # If anything went wrong, point out what command to rerun manually to
458 # see the actual errors and individual summary
463 # see the actual errors and individual summary
459 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
464 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
460 for name, failed_runner in failed:
465 for name, failed_runner in failed:
461 print '-'*40
466 print '-'*40
462 print 'Runner failed:',name
467 print 'Runner failed:',name
463 print 'You may wish to rerun this one individually, with:'
468 print 'You may wish to rerun this one individually, with:'
464 print ' '.join(failed_runner.call_args)
469 print ' '.join(failed_runner.call_args)
465 print
470 print
466 # Ensure that our exit code indicates failure
471 # Ensure that our exit code indicates failure
467 sys.exit(1)
472 sys.exit(1)
468
473
469
474
470 def main():
475 def main():
471 for arg in sys.argv[1:]:
476 for arg in sys.argv[1:]:
472 if arg.startswith('IPython'):
477 if arg.startswith('IPython'):
473 # This is in-process
478 # This is in-process
474 run_iptest()
479 run_iptest()
475 else:
480 else:
476 # This starts subprocesses
481 # This starts subprocesses
477 run_iptestall()
482 run_iptestall()
478
483
479
484
480 if __name__ == '__main__':
485 if __name__ == '__main__':
481 main()
486 main()
General Comments 0
You need to be logged in to leave comments. Login now