##// END OF EJS Templates
Pass subprocess test runners a suitable location for xunit output.
Thomas Kluyver -
Show More
@@ -1,481 +1,486 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Copyright (C) 2009-2011 The IPython Development Team
19 19 #
20 20 # Distributed under the terms of the BSD License. The full license is in
21 21 # the file COPYING, distributed as part of this software.
22 22 #-----------------------------------------------------------------------------
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Imports
26 26 #-----------------------------------------------------------------------------
27 27
28 28 # Stdlib
29 29 import os
30 30 import os.path as path
31 31 import signal
32 32 import sys
33 33 import subprocess
34 34 import tempfile
35 35 import time
36 36 import warnings
37 37
38 38 # Note: monkeypatch!
39 39 # We need to monkeypatch a small problem in nose itself first, before importing
40 40 # it for actual use. This should get into nose upstream, but its release cycle
41 41 # is slow and we need it for our parametric tests to work correctly.
42 42 from IPython.testing import nosepatch
43 43 # Now, proceed to import nose itself
44 44 import nose.plugins.builtin
45 45 from nose.core import TestProgram
46 46
47 47 # Our own imports
48 48 from IPython.utils.importstring import import_item
49 49 from IPython.utils.path import get_ipython_module_path
50 50 from IPython.utils.process import find_cmd, pycmd2argv
51 51 from IPython.utils.sysinfo import sys_info
52 52
53 53 from IPython.testing import globalipapp
54 54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
55 55 from IPython.external.decorators import KnownFailure
56 56
57 57 pjoin = path.join
58 58
59 59
60 60 #-----------------------------------------------------------------------------
61 61 # Globals
62 62 #-----------------------------------------------------------------------------
63 63
64 64
65 65 #-----------------------------------------------------------------------------
66 66 # Warnings control
67 67 #-----------------------------------------------------------------------------
68 68
69 69 # Twisted generates annoying warnings with Python 2.6, as will do other code
70 70 # that imports 'sets' as of today
71 71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
72 72 DeprecationWarning )
73 73
74 74 # This one also comes from Twisted
75 75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
76 76 DeprecationWarning)
77 77
78 78 # Wx on Fedora11 spits these out
79 79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
80 80 UserWarning)
81 81
82 82 #-----------------------------------------------------------------------------
83 83 # Logic for skipping doctests
84 84 #-----------------------------------------------------------------------------
85 85 def extract_version(mod):
86 86 return mod.__version__
87 87
88 88 def test_for(item, min_version=None, callback=extract_version):
89 89 """Test to see if item is importable, and optionally check against a minimum
90 90 version.
91 91
92 92 If min_version is given, the default behavior is to check against the
93 93 `__version__` attribute of the item, but specifying `callback` allows you to
94 94 extract the value you are interested in. e.g::
95 95
96 96 In [1]: import sys
97 97
98 98 In [2]: from IPython.testing.iptest import test_for
99 99
100 100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
101 101 Out[3]: True
102 102
103 103 """
104 104 try:
105 105 check = import_item(item)
106 106 except (ImportError, RuntimeError):
107 107 # GTK reports Runtime error if it can't be initialized even if it's
108 108 # importable.
109 109 return False
110 110 else:
111 111 if min_version:
112 112 if callback:
113 113 # extra processing step to get version to compare
114 114 check = callback(check)
115 115
116 116 return check >= min_version
117 117 else:
118 118 return True
119 119
120 120 # Global dict where we can store information on what we have and what we don't
121 121 # have available at test run time
122 122 have = {}
123 123
124 124 have['curses'] = test_for('_curses')
125 125 have['matplotlib'] = test_for('matplotlib')
126 126 have['pexpect'] = test_for('IPython.external.pexpect')
127 127 have['pymongo'] = test_for('pymongo')
128 128 have['wx'] = test_for('wx')
129 129 have['wx.aui'] = test_for('wx.aui')
130 130 have['qt'] = test_for('IPython.external.qt')
131 131 have['sqlite3'] = test_for('sqlite3')
132 132
133 133 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
134 134
135 135 if os.name == 'nt':
136 136 min_zmq = (2,1,7)
137 137 else:
138 138 min_zmq = (2,1,4)
139 139
140 140 def version_tuple(mod):
141 141 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
142 142 # turn 'dev' into 999, because Python3 rejects str-int comparisons
143 143 vs = mod.__version__.replace('dev', '.999')
144 144 tup = tuple([int(v) for v in vs.split('.') ])
145 145 return tup
146 146
147 147 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
148 148
149 149 #-----------------------------------------------------------------------------
150 150 # Functions and classes
151 151 #-----------------------------------------------------------------------------
152 152
153 153 def report():
154 154 """Return a string with a summary report of test-related variables."""
155 155
156 156 out = [ sys_info(), '\n']
157 157
158 158 avail = []
159 159 not_avail = []
160 160
161 161 for k, is_avail in have.items():
162 162 if is_avail:
163 163 avail.append(k)
164 164 else:
165 165 not_avail.append(k)
166 166
167 167 if avail:
168 168 out.append('\nTools and libraries available at test time:\n')
169 169 avail.sort()
170 170 out.append(' ' + ' '.join(avail)+'\n')
171 171
172 172 if not_avail:
173 173 out.append('\nTools and libraries NOT available at test time:\n')
174 174 not_avail.sort()
175 175 out.append(' ' + ' '.join(not_avail)+'\n')
176 176
177 177 return ''.join(out)
178 178
179 179
180 180 def make_exclude():
181 181 """Make patterns of modules and packages to exclude from testing.
182 182
183 183 For the IPythonDoctest plugin, we need to exclude certain patterns that
184 184 cause testing problems. We should strive to minimize the number of
185 185 skipped modules, since this means untested code.
186 186
187 187 These modules and packages will NOT get scanned by nose at all for tests.
188 188 """
189 189 # Simple utility to make IPython paths more readably, we need a lot of
190 190 # these below
191 191 ipjoin = lambda *paths: pjoin('IPython', *paths)
192 192
193 193 exclusions = [ipjoin('external'),
194 194 pjoin('IPython_doctest_plugin'),
195 195 ipjoin('quarantine'),
196 196 ipjoin('deathrow'),
197 197 ipjoin('testing', 'attic'),
198 198 # This guy is probably attic material
199 199 ipjoin('testing', 'mkdoctests'),
200 200 # Testing inputhook will need a lot of thought, to figure out
201 201 # how to have tests that don't lock up with the gui event
202 202 # loops in the picture
203 203 ipjoin('lib', 'inputhook'),
204 204 # Config files aren't really importable stand-alone
205 205 ipjoin('config', 'default'),
206 206 ipjoin('config', 'profile'),
207 207 ]
208 208 if not have['sqlite3']:
209 209 exclusions.append(ipjoin('core', 'tests', 'test_history'))
210 210 exclusions.append(ipjoin('core', 'history'))
211 211 if not have['wx']:
212 212 exclusions.append(ipjoin('lib', 'inputhookwx'))
213 213
214 214 # We do this unconditionally, so that the test suite doesn't import
215 215 # gtk, changing the default encoding and masking some unicode bugs.
216 216 exclusions.append(ipjoin('lib', 'inputhookgtk'))
217 217
218 218 # These have to be skipped on win32 because the use echo, rm, cd, etc.
219 219 # See ticket https://github.com/ipython/ipython/issues/87
220 220 if sys.platform == 'win32':
221 221 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
222 222 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
223 223
224 224 if not have['pexpect']:
225 225 exclusions.extend([ipjoin('scripts', 'irunner'),
226 226 ipjoin('lib', 'irunner'),
227 227 ipjoin('lib', 'tests', 'test_irunner'),
228 228 ipjoin('frontend', 'terminal', 'console'),
229 229 ])
230 230
231 231 if not have['zmq']:
232 232 exclusions.append(ipjoin('zmq'))
233 233 exclusions.append(ipjoin('frontend', 'qt'))
234 234 exclusions.append(ipjoin('frontend', 'html'))
235 235 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
236 236 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
237 237 exclusions.append(ipjoin('parallel'))
238 238 elif not have['qt']:
239 239 exclusions.append(ipjoin('frontend', 'qt'))
240 240
241 241 if not have['pymongo']:
242 242 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
243 243 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
244 244
245 245 if not have['matplotlib']:
246 246 exclusions.extend([ipjoin('core', 'pylabtools'),
247 247 ipjoin('core', 'tests', 'test_pylabtools')])
248 248
249 249 if not have['tornado']:
250 250 exclusions.append(ipjoin('frontend', 'html'))
251 251
252 252 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
253 253 if sys.platform == 'win32':
254 254 exclusions = [s.replace('\\','\\\\') for s in exclusions]
255 255
256 256 return exclusions
257 257
258 258
259 259 class IPTester(object):
260 260 """Call that calls iptest or trial in a subprocess.
261 261 """
262 262 #: string, name of test runner that will be called
263 263 runner = None
264 264 #: list, parameters for test runner
265 265 params = None
266 266 #: list, arguments of system call to be made to call test runner
267 267 call_args = None
268 268 #: list, process ids of subprocesses we start (for cleanup)
269 269 pids = None
270 270
271 271 def __init__(self, runner='iptest', params=None):
272 272 """Create new test runner."""
273 273 p = os.path
274 274 if runner == 'iptest':
275 275 iptest_app = get_ipython_module_path('IPython.testing.iptest')
276 276 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
277 277 else:
278 278 raise Exception('Not a valid test runner: %s' % repr(runner))
279 279 if params is None:
280 280 params = []
281 281 if isinstance(params, str):
282 282 params = [params]
283 283 self.params = params
284 284
285 285 # Assemble call
286 286 self.call_args = self.runner+self.params
287
288 print self.call_args
289 if '--with-xunit' in self.call_args:
290 sect = [p for p in self.params if p.startswith('IPython')][0]
291 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
287 292
288 293 # Store pids of anything we start to clean up on deletion, if possible
289 294 # (on posix only, since win32 has no os.kill)
290 295 self.pids = []
291 296
292 297 if sys.platform == 'win32':
293 298 def _run_cmd(self):
294 299 # On Windows, use os.system instead of subprocess.call, because I
295 300 # was having problems with subprocess and I just don't know enough
296 301 # about win32 to debug this reliably. Os.system may be the 'old
297 302 # fashioned' way to do it, but it works just fine. If someone
298 303 # later can clean this up that's fine, as long as the tests run
299 304 # reliably in win32.
300 305 # What types of problems are you having. They may be related to
301 306 # running Python in unboffered mode. BG.
302 307 return os.system(' '.join(self.call_args))
303 308 else:
304 309 def _run_cmd(self):
305 310 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
306 311 subp = subprocess.Popen(self.call_args)
307 312 self.pids.append(subp.pid)
308 313 # If this fails, the pid will be left in self.pids and cleaned up
309 314 # later, but if the wait call succeeds, then we can clear the
310 315 # stored pid.
311 316 retcode = subp.wait()
312 317 self.pids.pop()
313 318 return retcode
314 319
315 320 def run(self):
316 321 """Run the stored commands"""
317 322 try:
318 323 return self._run_cmd()
319 324 except:
320 325 import traceback
321 326 traceback.print_exc()
322 327 return 1 # signal failure
323 328
324 329 def __del__(self):
325 330 """Cleanup on exit by killing any leftover processes."""
326 331
327 332 if not hasattr(os, 'kill'):
328 333 return
329 334
330 335 for pid in self.pids:
331 336 try:
332 337 print 'Cleaning stale PID:', pid
333 338 os.kill(pid, signal.SIGKILL)
334 339 except OSError:
335 340 # This is just a best effort, if we fail or the process was
336 341 # really gone, ignore it.
337 342 pass
338 343
339 344
340 345 def make_runners():
341 346 """Define the top-level packages that need to be tested.
342 347 """
343 348
344 349 # Packages to be tested via nose, that only depend on the stdlib
345 350 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
346 351 'scripts', 'testing', 'utils', 'nbformat' ]
347 352
348 353 if have['zmq']:
349 354 nose_pkg_names.append('parallel')
350 355
351 356 # For debugging this code, only load quick stuff
352 357 #nose_pkg_names = ['core', 'extensions'] # dbg
353 358
354 359 # Make fully qualified package names prepending 'IPython.' to our name lists
355 360 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
356 361
357 362 # Make runners
358 363 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
359 364
360 365 return runners
361 366
362 367
363 368 def run_iptest():
364 369 """Run the IPython test suite using nose.
365 370
366 371 This function is called when this script is **not** called with the form
367 372 `iptest all`. It simply calls nose with appropriate command line flags
368 373 and accepts all of the standard nose arguments.
369 374 """
370 375
371 376 warnings.filterwarnings('ignore',
372 377 'This will be removed soon. Use IPython.testing.util instead')
373 378
374 379 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
375 380
376 381 # Loading ipdoctest causes problems with Twisted, but
377 382 # our test suite runner now separates things and runs
378 383 # all Twisted tests with trial.
379 384 '--with-ipdoctest',
380 385 '--ipdoctest-tests','--ipdoctest-extension=txt',
381 386
382 387 # We add --exe because of setuptools' imbecility (it
383 388 # blindly does chmod +x on ALL files). Nose does the
384 389 # right thing and it tries to avoid executables,
385 390 # setuptools unfortunately forces our hand here. This
386 391 # has been discussed on the distutils list and the
387 392 # setuptools devs refuse to fix this problem!
388 393 '--exe',
389 394 ]
390 395
391 396 if nose.__version__ >= '0.11':
392 397 # I don't fully understand why we need this one, but depending on what
393 398 # directory the test suite is run from, if we don't give it, 0 tests
394 399 # get run. Specifically, if the test suite is run from the source dir
395 400 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
396 401 # even if the same call done in this directory works fine). It appears
397 402 # that if the requested package is in the current dir, nose bails early
398 403 # by default. Since it's otherwise harmless, leave it in by default
399 404 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
400 405 argv.append('--traverse-namespace')
401 406
402 407 # use our plugin for doctesting. It will remove the standard doctest plugin
403 408 # if it finds it enabled
404 409 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
405 410 # We need a global ipython running in this process
406 411 globalipapp.start_ipython()
407 412 # Now nose can run
408 413 TestProgram(argv=argv, addplugins=plugins)
409 414
410 415
411 416 def run_iptestall():
412 417 """Run the entire IPython test suite by calling nose and trial.
413 418
414 419 This function constructs :class:`IPTester` instances for all IPython
415 420 modules and package and then runs each of them. This causes the modules
416 421 and packages of IPython to be tested each in their own subprocess using
417 422 nose or twisted.trial appropriately.
418 423 """
419 424
420 425 runners = make_runners()
421 426
422 427 # Run the test runners in a temporary dir so we can nuke it when finished
423 428 # to clean up any junk files left over by accident. This also makes it
424 429 # robust against being run in non-writeable directories by mistake, as the
425 430 # temp dir will always be user-writeable.
426 431 curdir = os.getcwdu()
427 432 testdir = tempfile.gettempdir()
428 433 os.chdir(testdir)
429 434
430 435 # Run all test runners, tracking execution time
431 436 failed = []
432 437 t_start = time.time()
433 438 try:
434 439 for (name, runner) in runners:
435 440 print '*'*70
436 441 print 'IPython test group:',name
437 442 res = runner.run()
438 443 if res:
439 444 failed.append( (name, runner) )
440 445 finally:
441 446 os.chdir(curdir)
442 447 t_end = time.time()
443 448 t_tests = t_end - t_start
444 449 nrunners = len(runners)
445 450 nfail = len(failed)
446 451 # summarize results
447 452 print
448 453 print '*'*70
449 454 print 'Test suite completed for system with the following information:'
450 455 print report()
451 456 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
452 457 print
453 458 print 'Status:'
454 459 if not failed:
455 460 print 'OK'
456 461 else:
457 462 # If anything went wrong, point out what command to rerun manually to
458 463 # see the actual errors and individual summary
459 464 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
460 465 for name, failed_runner in failed:
461 466 print '-'*40
462 467 print 'Runner failed:',name
463 468 print 'You may wish to rerun this one individually, with:'
464 469 print ' '.join(failed_runner.call_args)
465 470 print
466 471 # Ensure that our exit code indicates failure
467 472 sys.exit(1)
468 473
469 474
470 475 def main():
471 476 for arg in sys.argv[1:]:
472 477 if arg.startswith('IPython'):
473 478 # This is in-process
474 479 run_iptest()
475 480 else:
476 481 # This starts subprocesses
477 482 run_iptestall()
478 483
479 484
480 485 if __name__ == '__main__':
481 486 main()
General Comments 0
You need to be logged in to leave comments. Login now