##// END OF EJS Templates
Use separate option --with-xml-coverage to ask for coverage xml output.
Thomas Kluyver -
Show More
@@ -1,496 +1,496 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Copyright (C) 2009-2011 The IPython Development Team
19 19 #
20 20 # Distributed under the terms of the BSD License. The full license is in
21 21 # the file COPYING, distributed as part of this software.
22 22 #-----------------------------------------------------------------------------
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Imports
26 26 #-----------------------------------------------------------------------------
27 27
28 28 # Stdlib
29 29 import os
30 30 import os.path as path
31 31 import signal
32 32 import sys
33 33 import subprocess
34 34 import tempfile
35 35 import time
36 36 import warnings
37 37
38 38 # Note: monkeypatch!
39 39 # We need to monkeypatch a small problem in nose itself first, before importing
40 40 # it for actual use. This should get into nose upstream, but its release cycle
41 41 # is slow and we need it for our parametric tests to work correctly.
42 42 from IPython.testing import nosepatch
43 43 # Now, proceed to import nose itself
44 44 import nose.plugins.builtin
45 45 from nose.core import TestProgram
46 46
47 47 # Our own imports
48 48 from IPython.utils.importstring import import_item
49 49 from IPython.utils.path import get_ipython_module_path
50 50 from IPython.utils.process import find_cmd, pycmd2argv
51 51 from IPython.utils.sysinfo import sys_info
52 52
53 53 from IPython.testing import globalipapp
54 54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
55 55 from IPython.external.decorators import KnownFailure
56 56
57 57 pjoin = path.join
58 58
59 59
60 60 #-----------------------------------------------------------------------------
61 61 # Globals
62 62 #-----------------------------------------------------------------------------
63 63
64 64
65 65 #-----------------------------------------------------------------------------
66 66 # Warnings control
67 67 #-----------------------------------------------------------------------------
68 68
69 69 # Twisted generates annoying warnings with Python 2.6, as will do other code
70 70 # that imports 'sets' as of today
71 71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
72 72 DeprecationWarning )
73 73
74 74 # This one also comes from Twisted
75 75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
76 76 DeprecationWarning)
77 77
78 78 # Wx on Fedora11 spits these out
79 79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
80 80 UserWarning)
81 81
82 82 #-----------------------------------------------------------------------------
83 83 # Logic for skipping doctests
84 84 #-----------------------------------------------------------------------------
85 85 def extract_version(mod):
86 86 return mod.__version__
87 87
88 88 def test_for(item, min_version=None, callback=extract_version):
89 89 """Test to see if item is importable, and optionally check against a minimum
90 90 version.
91 91
92 92 If min_version is given, the default behavior is to check against the
93 93 `__version__` attribute of the item, but specifying `callback` allows you to
94 94 extract the value you are interested in. e.g::
95 95
96 96 In [1]: import sys
97 97
98 98 In [2]: from IPython.testing.iptest import test_for
99 99
100 100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
101 101 Out[3]: True
102 102
103 103 """
104 104 try:
105 105 check = import_item(item)
106 106 except (ImportError, RuntimeError):
107 107 # GTK reports Runtime error if it can't be initialized even if it's
108 108 # importable.
109 109 return False
110 110 else:
111 111 if min_version:
112 112 if callback:
113 113 # extra processing step to get version to compare
114 114 check = callback(check)
115 115
116 116 return check >= min_version
117 117 else:
118 118 return True
119 119
120 120 # Global dict where we can store information on what we have and what we don't
121 121 # have available at test run time
122 122 have = {}
123 123
124 124 have['curses'] = test_for('_curses')
125 125 have['matplotlib'] = test_for('matplotlib')
126 126 have['pexpect'] = test_for('IPython.external.pexpect')
127 127 have['pymongo'] = test_for('pymongo')
128 128 have['wx'] = test_for('wx')
129 129 have['wx.aui'] = test_for('wx.aui')
130 130 have['qt'] = test_for('IPython.external.qt')
131 131 have['sqlite3'] = test_for('sqlite3')
132 132
133 133 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
134 134
135 135 if os.name == 'nt':
136 136 min_zmq = (2,1,7)
137 137 else:
138 138 min_zmq = (2,1,4)
139 139
140 140 def version_tuple(mod):
141 141 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
142 142 # turn 'dev' into 999, because Python3 rejects str-int comparisons
143 143 vs = mod.__version__.replace('dev', '.999')
144 144 tup = tuple([int(v) for v in vs.split('.') ])
145 145 return tup
146 146
147 147 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
148 148
149 149 #-----------------------------------------------------------------------------
150 150 # Functions and classes
151 151 #-----------------------------------------------------------------------------
152 152
153 153 def report():
154 154 """Return a string with a summary report of test-related variables."""
155 155
156 156 out = [ sys_info(), '\n']
157 157
158 158 avail = []
159 159 not_avail = []
160 160
161 161 for k, is_avail in have.items():
162 162 if is_avail:
163 163 avail.append(k)
164 164 else:
165 165 not_avail.append(k)
166 166
167 167 if avail:
168 168 out.append('\nTools and libraries available at test time:\n')
169 169 avail.sort()
170 170 out.append(' ' + ' '.join(avail)+'\n')
171 171
172 172 if not_avail:
173 173 out.append('\nTools and libraries NOT available at test time:\n')
174 174 not_avail.sort()
175 175 out.append(' ' + ' '.join(not_avail)+'\n')
176 176
177 177 return ''.join(out)
178 178
179 179
180 180 def make_exclude():
181 181 """Make patterns of modules and packages to exclude from testing.
182 182
183 183 For the IPythonDoctest plugin, we need to exclude certain patterns that
184 184 cause testing problems. We should strive to minimize the number of
185 185 skipped modules, since this means untested code.
186 186
187 187 These modules and packages will NOT get scanned by nose at all for tests.
188 188 """
189 189 # Simple utility to make IPython paths more readably, we need a lot of
190 190 # these below
191 191 ipjoin = lambda *paths: pjoin('IPython', *paths)
192 192
193 193 exclusions = [ipjoin('external'),
194 194 pjoin('IPython_doctest_plugin'),
195 195 ipjoin('quarantine'),
196 196 ipjoin('deathrow'),
197 197 ipjoin('testing', 'attic'),
198 198 # This guy is probably attic material
199 199 ipjoin('testing', 'mkdoctests'),
200 200 # Testing inputhook will need a lot of thought, to figure out
201 201 # how to have tests that don't lock up with the gui event
202 202 # loops in the picture
203 203 ipjoin('lib', 'inputhook'),
204 204 # Config files aren't really importable stand-alone
205 205 ipjoin('config', 'default'),
206 206 ipjoin('config', 'profile'),
207 207 ]
208 208 if not have['sqlite3']:
209 209 exclusions.append(ipjoin('core', 'tests', 'test_history'))
210 210 exclusions.append(ipjoin('core', 'history'))
211 211 if not have['wx']:
212 212 exclusions.append(ipjoin('lib', 'inputhookwx'))
213 213
214 214 # We do this unconditionally, so that the test suite doesn't import
215 215 # gtk, changing the default encoding and masking some unicode bugs.
216 216 exclusions.append(ipjoin('lib', 'inputhookgtk'))
217 217
218 218 # These have to be skipped on win32 because the use echo, rm, cd, etc.
219 219 # See ticket https://github.com/ipython/ipython/issues/87
220 220 if sys.platform == 'win32':
221 221 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
222 222 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
223 223
224 224 if not have['pexpect']:
225 225 exclusions.extend([ipjoin('scripts', 'irunner'),
226 226 ipjoin('lib', 'irunner'),
227 227 ipjoin('lib', 'tests', 'test_irunner'),
228 228 ipjoin('frontend', 'terminal', 'console'),
229 229 ])
230 230
231 231 if not have['zmq']:
232 232 exclusions.append(ipjoin('zmq'))
233 233 exclusions.append(ipjoin('frontend', 'qt'))
234 234 exclusions.append(ipjoin('frontend', 'html'))
235 235 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
236 236 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
237 237 exclusions.append(ipjoin('parallel'))
238 238 elif not have['qt']:
239 239 exclusions.append(ipjoin('frontend', 'qt'))
240 240
241 241 if not have['pymongo']:
242 242 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
243 243 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
244 244
245 245 if not have['matplotlib']:
246 246 exclusions.extend([ipjoin('core', 'pylabtools'),
247 247 ipjoin('core', 'tests', 'test_pylabtools')])
248 248
249 249 if not have['tornado']:
250 250 exclusions.append(ipjoin('frontend', 'html'))
251 251
252 252 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
253 253 if sys.platform == 'win32':
254 254 exclusions = [s.replace('\\','\\\\') for s in exclusions]
255 255
256 256 return exclusions
257 257
258 258
259 259 class IPTester(object):
260 260 """Call that calls iptest or trial in a subprocess.
261 261 """
262 262 #: string, name of test runner that will be called
263 263 runner = None
264 264 #: list, parameters for test runner
265 265 params = None
266 266 #: list, arguments of system call to be made to call test runner
267 267 call_args = None
268 268 #: list, process ids of subprocesses we start (for cleanup)
269 269 pids = None
270 270 #: str, coverage xml output file
271 271 coverage_xml = None
272 272
273 273 def __init__(self, runner='iptest', params=None):
274 274 """Create new test runner."""
275 275 p = os.path
276 276 if runner == 'iptest':
277 277 iptest_app = get_ipython_module_path('IPython.testing.iptest')
278 278 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
279 279 else:
280 280 raise Exception('Not a valid test runner: %s' % repr(runner))
281 281 if params is None:
282 282 params = []
283 283 if isinstance(params, str):
284 284 params = [params]
285 285 self.params = params
286 286
287 287 # Assemble call
288 288 self.call_args = self.runner+self.params
289 289
290 290 sect = [p for p in self.params if p.startswith('IPython')][0]
291 291 if '--with-xunit' in self.call_args:
292 292 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
293 293
294 if '--with-coverage' in self.call_args:
294 if '--with-xml-coverage' in self.call_args:
295 295 self.coverage_xml = path.abspath(sect+".coverage.xml")
296 self.call_args.remove('--with-coverage')
296 self.call_args.remove('--with-xml-coverage')
297 297 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
298 298
299 299 # Store pids of anything we start to clean up on deletion, if possible
300 300 # (on posix only, since win32 has no os.kill)
301 301 self.pids = []
302 302
303 303 if sys.platform == 'win32':
304 304 def _run_cmd(self):
305 305 # On Windows, use os.system instead of subprocess.call, because I
306 306 # was having problems with subprocess and I just don't know enough
307 307 # about win32 to debug this reliably. Os.system may be the 'old
308 308 # fashioned' way to do it, but it works just fine. If someone
309 309 # later can clean this up that's fine, as long as the tests run
310 310 # reliably in win32.
311 311 # What types of problems are you having. They may be related to
312 312 # running Python in unboffered mode. BG.
313 313 return os.system(' '.join(self.call_args))
314 314 else:
315 315 def _run_cmd(self):
316 316 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
317 317 subp = subprocess.Popen(self.call_args)
318 318 self.pids.append(subp.pid)
319 319 # If this fails, the pid will be left in self.pids and cleaned up
320 320 # later, but if the wait call succeeds, then we can clear the
321 321 # stored pid.
322 322 retcode = subp.wait()
323 323 self.pids.pop()
324 324 return retcode
325 325
326 326 def run(self):
327 327 """Run the stored commands"""
328 328 try:
329 329 retcode = self._run_cmd()
330 330 except:
331 331 import traceback
332 332 traceback.print_exc()
333 333 return 1 # signal failure
334 334
335 335 if self.coverage_xml:
336 336 subprocess.check_call(["coverage", "xml", "-o", self.coverage_xml])
337 337 return retcode
338 338
339 339 def __del__(self):
340 340 """Cleanup on exit by killing any leftover processes."""
341 341
342 342 if not hasattr(os, 'kill'):
343 343 return
344 344
345 345 for pid in self.pids:
346 346 try:
347 347 print 'Cleaning stale PID:', pid
348 348 os.kill(pid, signal.SIGKILL)
349 349 except OSError:
350 350 # This is just a best effort, if we fail or the process was
351 351 # really gone, ignore it.
352 352 pass
353 353
354 354
355 355 def make_runners():
356 356 """Define the top-level packages that need to be tested.
357 357 """
358 358
359 359 # Packages to be tested via nose, that only depend on the stdlib
360 360 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
361 361 'scripts', 'testing', 'utils', 'nbformat' ]
362 362
363 363 if have['zmq']:
364 364 nose_pkg_names.append('parallel')
365 365
366 366 # For debugging this code, only load quick stuff
367 367 #nose_pkg_names = ['core', 'extensions'] # dbg
368 368
369 369 # Make fully qualified package names prepending 'IPython.' to our name lists
370 370 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
371 371
372 372 # Make runners
373 373 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
374 374
375 375 return runners
376 376
377 377
378 378 def run_iptest():
379 379 """Run the IPython test suite using nose.
380 380
381 381 This function is called when this script is **not** called with the form
382 382 `iptest all`. It simply calls nose with appropriate command line flags
383 383 and accepts all of the standard nose arguments.
384 384 """
385 385
386 386 warnings.filterwarnings('ignore',
387 387 'This will be removed soon. Use IPython.testing.util instead')
388 388
389 389 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
390 390
391 391 # Loading ipdoctest causes problems with Twisted, but
392 392 # our test suite runner now separates things and runs
393 393 # all Twisted tests with trial.
394 394 '--with-ipdoctest',
395 395 '--ipdoctest-tests','--ipdoctest-extension=txt',
396 396
397 397 # We add --exe because of setuptools' imbecility (it
398 398 # blindly does chmod +x on ALL files). Nose does the
399 399 # right thing and it tries to avoid executables,
400 400 # setuptools unfortunately forces our hand here. This
401 401 # has been discussed on the distutils list and the
402 402 # setuptools devs refuse to fix this problem!
403 403 '--exe',
404 404 ]
405 405
406 406 if nose.__version__ >= '0.11':
407 407 # I don't fully understand why we need this one, but depending on what
408 408 # directory the test suite is run from, if we don't give it, 0 tests
409 409 # get run. Specifically, if the test suite is run from the source dir
410 410 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
411 411 # even if the same call done in this directory works fine). It appears
412 412 # that if the requested package is in the current dir, nose bails early
413 413 # by default. Since it's otherwise harmless, leave it in by default
414 414 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
415 415 argv.append('--traverse-namespace')
416 416
417 417 # use our plugin for doctesting. It will remove the standard doctest plugin
418 418 # if it finds it enabled
419 419 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
420 420 # We need a global ipython running in this process
421 421 globalipapp.start_ipython()
422 422 # Now nose can run
423 423 TestProgram(argv=argv, addplugins=plugins)
424 424
425 425
426 426 def run_iptestall():
427 427 """Run the entire IPython test suite by calling nose and trial.
428 428
429 429 This function constructs :class:`IPTester` instances for all IPython
430 430 modules and package and then runs each of them. This causes the modules
431 431 and packages of IPython to be tested each in their own subprocess using
432 432 nose or twisted.trial appropriately.
433 433 """
434 434
435 435 runners = make_runners()
436 436
437 437 # Run the test runners in a temporary dir so we can nuke it when finished
438 438 # to clean up any junk files left over by accident. This also makes it
439 439 # robust against being run in non-writeable directories by mistake, as the
440 440 # temp dir will always be user-writeable.
441 441 curdir = os.getcwdu()
442 442 testdir = tempfile.gettempdir()
443 443 os.chdir(testdir)
444 444
445 445 # Run all test runners, tracking execution time
446 446 failed = []
447 447 t_start = time.time()
448 448 try:
449 449 for (name, runner) in runners:
450 450 print '*'*70
451 451 print 'IPython test group:',name
452 452 res = runner.run()
453 453 if res:
454 454 failed.append( (name, runner) )
455 455 finally:
456 456 os.chdir(curdir)
457 457 t_end = time.time()
458 458 t_tests = t_end - t_start
459 459 nrunners = len(runners)
460 460 nfail = len(failed)
461 461 # summarize results
462 462 print
463 463 print '*'*70
464 464 print 'Test suite completed for system with the following information:'
465 465 print report()
466 466 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
467 467 print
468 468 print 'Status:'
469 469 if not failed:
470 470 print 'OK'
471 471 else:
472 472 # If anything went wrong, point out what command to rerun manually to
473 473 # see the actual errors and individual summary
474 474 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
475 475 for name, failed_runner in failed:
476 476 print '-'*40
477 477 print 'Runner failed:',name
478 478 print 'You may wish to rerun this one individually, with:'
479 479 print ' '.join(failed_runner.call_args)
480 480 print
481 481 # Ensure that our exit code indicates failure
482 482 sys.exit(1)
483 483
484 484
485 485 def main():
486 486 for arg in sys.argv[1:]:
487 487 if arg.startswith('IPython'):
488 488 # This is in-process
489 489 run_iptest()
490 490 else:
491 491 # This starts subprocesses
492 492 run_iptestall()
493 493
494 494
495 495 if __name__ == '__main__':
496 496 main()
General Comments 0
You need to be logged in to leave comments. Login now