##// END OF EJS Templates
Skip octavemagic tests if oct2py is unavailable.
Stefan van der Walt -
Show More
@@ -1,545 +1,550 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Copyright (C) 2009-2011 The IPython Development Team
19 19 #
20 20 # Distributed under the terms of the BSD License. The full license is in
21 21 # the file COPYING, distributed as part of this software.
22 22 #-----------------------------------------------------------------------------
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Imports
26 26 #-----------------------------------------------------------------------------
27 27
28 28 # Stdlib
29 29 import os
30 30 import os.path as path
31 31 import signal
32 32 import sys
33 33 import subprocess
34 34 import tempfile
35 35 import time
36 36 import warnings
37 37
38 38 # Note: monkeypatch!
39 39 # We need to monkeypatch a small problem in nose itself first, before importing
40 40 # it for actual use. This should get into nose upstream, but its release cycle
41 41 # is slow and we need it for our parametric tests to work correctly.
42 42 from IPython.testing import nosepatch
43 43
44 44 # Monkeypatch extra assert methods into nose.tools if they're not already there.
45 45 # This can be dropped once we no longer test on Python 2.6
46 46 from IPython.testing import nose_assert_methods
47 47
48 48 # Now, proceed to import nose itself
49 49 import nose.plugins.builtin
50 50 from nose.plugins.xunit import Xunit
51 51 from nose import SkipTest
52 52 from nose.core import TestProgram
53 53
54 54 # Our own imports
55 55 from IPython.utils.importstring import import_item
56 56 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
57 57 from IPython.utils.process import find_cmd, pycmd2argv
58 58 from IPython.utils.sysinfo import sys_info
59 59 from IPython.utils.warn import warn
60 60
61 61 from IPython.testing import globalipapp
62 62 from IPython.testing.plugin.ipdoctest import IPythonDoctest
63 63 from IPython.external.decorators import KnownFailure, knownfailureif
64 64
65 65 pjoin = path.join
66 66
67 67
68 68 #-----------------------------------------------------------------------------
69 69 # Globals
70 70 #-----------------------------------------------------------------------------
71 71
72 72
73 73 #-----------------------------------------------------------------------------
74 74 # Warnings control
75 75 #-----------------------------------------------------------------------------
76 76
77 77 # Twisted generates annoying warnings with Python 2.6, as will do other code
78 78 # that imports 'sets' as of today
79 79 warnings.filterwarnings('ignore', 'the sets module is deprecated',
80 80 DeprecationWarning )
81 81
82 82 # This one also comes from Twisted
83 83 warnings.filterwarnings('ignore', 'the sha module is deprecated',
84 84 DeprecationWarning)
85 85
86 86 # Wx on Fedora11 spits these out
87 87 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
88 88 UserWarning)
89 89
90 90 # ------------------------------------------------------------------------------
91 91 # Monkeypatch Xunit to count known failures as skipped.
92 92 # ------------------------------------------------------------------------------
93 93 def monkeypatch_xunit():
94 94 try:
95 95 knownfailureif(True)(lambda: None)()
96 96 except Exception as e:
97 97 KnownFailureTest = type(e)
98 98
99 99 def addError(self, test, err, capt=None):
100 100 if issubclass(err[0], KnownFailureTest):
101 101 err = (SkipTest,) + err[1:]
102 102 return self.orig_addError(test, err, capt)
103 103
104 104 Xunit.orig_addError = Xunit.addError
105 105 Xunit.addError = addError
106 106
107 107 #-----------------------------------------------------------------------------
108 108 # Logic for skipping doctests
109 109 #-----------------------------------------------------------------------------
110 110 def extract_version(mod):
111 111 return mod.__version__
112 112
113 113 def test_for(item, min_version=None, callback=extract_version):
114 114 """Test to see if item is importable, and optionally check against a minimum
115 115 version.
116 116
117 117 If min_version is given, the default behavior is to check against the
118 118 `__version__` attribute of the item, but specifying `callback` allows you to
119 119 extract the value you are interested in. e.g::
120 120
121 121 In [1]: import sys
122 122
123 123 In [2]: from IPython.testing.iptest import test_for
124 124
125 125 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
126 126 Out[3]: True
127 127
128 128 """
129 129 try:
130 130 check = import_item(item)
131 131 except (ImportError, RuntimeError):
132 132 # GTK reports Runtime error if it can't be initialized even if it's
133 133 # importable.
134 134 return False
135 135 else:
136 136 if min_version:
137 137 if callback:
138 138 # extra processing step to get version to compare
139 139 check = callback(check)
140 140
141 141 return check >= min_version
142 142 else:
143 143 return True
144 144
145 145 # Global dict where we can store information on what we have and what we don't
146 146 # have available at test run time
147 147 have = {}
148 148
149 149 have['curses'] = test_for('_curses')
150 150 have['matplotlib'] = test_for('matplotlib')
151 151 have['numpy'] = test_for('numpy')
152 152 have['pexpect'] = test_for('IPython.external.pexpect')
153 153 have['pymongo'] = test_for('pymongo')
154 154 have['pygments'] = test_for('pygments')
155 155 have['qt'] = test_for('IPython.external.qt')
156 156 have['rpy2'] = test_for('rpy2')
157 157 have['sqlite3'] = test_for('sqlite3')
158 158 have['cython'] = test_for('Cython')
159 have['oct2py'] = test_for('oct2py')
159 160 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
160 161 have['wx'] = test_for('wx')
161 162 have['wx.aui'] = test_for('wx.aui')
162 163
163 164 if os.name == 'nt':
164 165 min_zmq = (2,1,7)
165 166 else:
166 167 min_zmq = (2,1,4)
167 168
168 169 def version_tuple(mod):
169 170 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
170 171 # turn 'dev' into 999, because Python3 rejects str-int comparisons
171 172 vs = mod.__version__.replace('dev', '.999')
172 173 tup = tuple([int(v) for v in vs.split('.') ])
173 174 return tup
174 175
175 176 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
176 177
177 178 #-----------------------------------------------------------------------------
178 179 # Functions and classes
179 180 #-----------------------------------------------------------------------------
180 181
181 182 def report():
182 183 """Return a string with a summary report of test-related variables."""
183 184
184 185 out = [ sys_info(), '\n']
185 186
186 187 avail = []
187 188 not_avail = []
188 189
189 190 for k, is_avail in have.items():
190 191 if is_avail:
191 192 avail.append(k)
192 193 else:
193 194 not_avail.append(k)
194 195
195 196 if avail:
196 197 out.append('\nTools and libraries available at test time:\n')
197 198 avail.sort()
198 199 out.append(' ' + ' '.join(avail)+'\n')
199 200
200 201 if not_avail:
201 202 out.append('\nTools and libraries NOT available at test time:\n')
202 203 not_avail.sort()
203 204 out.append(' ' + ' '.join(not_avail)+'\n')
204 205
205 206 return ''.join(out)
206 207
207 208
208 209 def make_exclude():
209 210 """Make patterns of modules and packages to exclude from testing.
210 211
211 212 For the IPythonDoctest plugin, we need to exclude certain patterns that
212 213 cause testing problems. We should strive to minimize the number of
213 214 skipped modules, since this means untested code.
214 215
215 216 These modules and packages will NOT get scanned by nose at all for tests.
216 217 """
217 218 # Simple utility to make IPython paths more readably, we need a lot of
218 219 # these below
219 220 ipjoin = lambda *paths: pjoin('IPython', *paths)
220 221
221 222 exclusions = [ipjoin('external'),
222 223 ipjoin('quarantine'),
223 224 ipjoin('deathrow'),
224 225 # This guy is probably attic material
225 226 ipjoin('testing', 'mkdoctests'),
226 227 # Testing inputhook will need a lot of thought, to figure out
227 228 # how to have tests that don't lock up with the gui event
228 229 # loops in the picture
229 230 ipjoin('lib', 'inputhook'),
230 231 # Config files aren't really importable stand-alone
231 232 ipjoin('config', 'profile'),
232 233 ]
233 234 if not have['sqlite3']:
234 235 exclusions.append(ipjoin('core', 'tests', 'test_history'))
235 236 exclusions.append(ipjoin('core', 'history'))
236 237 if not have['wx']:
237 238 exclusions.append(ipjoin('lib', 'inputhookwx'))
238 239
239 240 # We do this unconditionally, so that the test suite doesn't import
240 241 # gtk, changing the default encoding and masking some unicode bugs.
241 242 exclusions.append(ipjoin('lib', 'inputhookgtk'))
242 243 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
243 244
244 245 # These have to be skipped on win32 because the use echo, rm, cd, etc.
245 246 # See ticket https://github.com/ipython/ipython/issues/87
246 247 if sys.platform == 'win32':
247 248 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
248 249 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
249 250
250 251 if not have['pexpect']:
251 252 exclusions.extend([ipjoin('scripts', 'irunner'),
252 253 ipjoin('lib', 'irunner'),
253 254 ipjoin('lib', 'tests', 'test_irunner'),
254 255 ipjoin('frontend', 'terminal', 'console'),
255 256 ])
256 257
257 258 if not have['zmq']:
258 259 exclusions.append(ipjoin('zmq'))
259 260 exclusions.append(ipjoin('frontend', 'qt'))
260 261 exclusions.append(ipjoin('frontend', 'html'))
261 262 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
262 263 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
263 264 exclusions.append(ipjoin('parallel'))
264 265 elif not have['qt'] or not have['pygments']:
265 266 exclusions.append(ipjoin('frontend', 'qt'))
266 267
267 268 if not have['pymongo']:
268 269 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
269 270 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
270 271
271 272 if not have['matplotlib']:
272 273 exclusions.extend([ipjoin('core', 'pylabtools'),
273 274 ipjoin('core', 'tests', 'test_pylabtools'),
274 275 ipjoin('zmq', 'pylab'),
275 276 ])
276 277
277 278 if not have['cython']:
278 279 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
279 280 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
280 281
282 if not have['oct2py']:
283 exclusions.extend([ipjoin('extensions', 'octavemagic')])
284 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
285
281 286 if not have['tornado']:
282 287 exclusions.append(ipjoin('frontend', 'html'))
283 288
284 289 if not have['rpy2'] or not have['numpy']:
285 290 exclusions.append(ipjoin('extensions', 'rmagic'))
286 291 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
287 292
288 293 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
289 294 if sys.platform == 'win32':
290 295 exclusions = [s.replace('\\','\\\\') for s in exclusions]
291 296
292 297 # check for any exclusions that don't seem to exist:
293 298 parent, _ = os.path.split(get_ipython_package_dir())
294 299 for exclusion in exclusions:
295 300 fullpath = pjoin(parent, exclusion)
296 301 if not os.path.exists(fullpath) and not os.path.exists(fullpath + '.py'):
297 302 warn("Excluding nonexistent file: %r\n" % exclusion)
298 303
299 304 return exclusions
300 305
301 306
302 307 class IPTester(object):
303 308 """Call that calls iptest or trial in a subprocess.
304 309 """
305 310 #: string, name of test runner that will be called
306 311 runner = None
307 312 #: list, parameters for test runner
308 313 params = None
309 314 #: list, arguments of system call to be made to call test runner
310 315 call_args = None
311 316 #: list, process ids of subprocesses we start (for cleanup)
312 317 pids = None
313 318 #: str, coverage xml output file
314 319 coverage_xml = None
315 320
316 321 def __init__(self, runner='iptest', params=None):
317 322 """Create new test runner."""
318 323 p = os.path
319 324 if runner == 'iptest':
320 325 iptest_app = get_ipython_module_path('IPython.testing.iptest')
321 326 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
322 327 else:
323 328 raise Exception('Not a valid test runner: %s' % repr(runner))
324 329 if params is None:
325 330 params = []
326 331 if isinstance(params, str):
327 332 params = [params]
328 333 self.params = params
329 334
330 335 # Assemble call
331 336 self.call_args = self.runner+self.params
332 337
333 338 # Find the section we're testing (IPython.foo)
334 339 for sect in self.params:
335 340 if sect.startswith('IPython'): break
336 341 else:
337 342 raise ValueError("Section not found", self.params)
338 343
339 344 if '--with-xunit' in self.call_args:
340 345 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
341 346
342 347 if '--with-xml-coverage' in self.call_args:
343 348 self.coverage_xml = path.abspath(sect+".coverage.xml")
344 349 self.call_args.remove('--with-xml-coverage')
345 350 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
346 351
347 352 # Store pids of anything we start to clean up on deletion, if possible
348 353 # (on posix only, since win32 has no os.kill)
349 354 self.pids = []
350 355
351 356 if sys.platform == 'win32':
352 357 def _run_cmd(self):
353 358 # On Windows, use os.system instead of subprocess.call, because I
354 359 # was having problems with subprocess and I just don't know enough
355 360 # about win32 to debug this reliably. Os.system may be the 'old
356 361 # fashioned' way to do it, but it works just fine. If someone
357 362 # later can clean this up that's fine, as long as the tests run
358 363 # reliably in win32.
359 364 # What types of problems are you having. They may be related to
360 365 # running Python in unboffered mode. BG.
361 366 return os.system(' '.join(self.call_args))
362 367 else:
363 368 def _run_cmd(self):
364 369 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
365 370 subp = subprocess.Popen(self.call_args)
366 371 self.pids.append(subp.pid)
367 372 # If this fails, the pid will be left in self.pids and cleaned up
368 373 # later, but if the wait call succeeds, then we can clear the
369 374 # stored pid.
370 375 retcode = subp.wait()
371 376 self.pids.pop()
372 377 return retcode
373 378
374 379 def run(self):
375 380 """Run the stored commands"""
376 381 try:
377 382 retcode = self._run_cmd()
378 383 except:
379 384 import traceback
380 385 traceback.print_exc()
381 386 return 1 # signal failure
382 387
383 388 if self.coverage_xml:
384 389 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
385 390 return retcode
386 391
387 392 def __del__(self):
388 393 """Cleanup on exit by killing any leftover processes."""
389 394
390 395 if not hasattr(os, 'kill'):
391 396 return
392 397
393 398 for pid in self.pids:
394 399 try:
395 400 print 'Cleaning stale PID:', pid
396 401 os.kill(pid, signal.SIGKILL)
397 402 except OSError:
398 403 # This is just a best effort, if we fail or the process was
399 404 # really gone, ignore it.
400 405 pass
401 406
402 407
403 408 def make_runners():
404 409 """Define the top-level packages that need to be tested.
405 410 """
406 411
407 412 # Packages to be tested via nose, that only depend on the stdlib
408 413 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
409 414 'scripts', 'testing', 'utils', 'nbformat' ]
410 415
411 416 if have['zmq']:
412 417 nose_pkg_names.append('zmq')
413 418 nose_pkg_names.append('parallel')
414 419
415 420 # For debugging this code, only load quick stuff
416 421 #nose_pkg_names = ['core', 'extensions'] # dbg
417 422
418 423 # Make fully qualified package names prepending 'IPython.' to our name lists
419 424 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
420 425
421 426 # Make runners
422 427 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
423 428
424 429 return runners
425 430
426 431
427 432 def run_iptest():
428 433 """Run the IPython test suite using nose.
429 434
430 435 This function is called when this script is **not** called with the form
431 436 `iptest all`. It simply calls nose with appropriate command line flags
432 437 and accepts all of the standard nose arguments.
433 438 """
434 439 # Apply our monkeypatch to Xunit
435 440 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
436 441 monkeypatch_xunit()
437 442
438 443 warnings.filterwarnings('ignore',
439 444 'This will be removed soon. Use IPython.testing.util instead')
440 445
441 446 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
442 447
443 448 '--with-ipdoctest',
444 449 '--ipdoctest-tests','--ipdoctest-extension=txt',
445 450
446 451 # We add --exe because of setuptools' imbecility (it
447 452 # blindly does chmod +x on ALL files). Nose does the
448 453 # right thing and it tries to avoid executables,
449 454 # setuptools unfortunately forces our hand here. This
450 455 # has been discussed on the distutils list and the
451 456 # setuptools devs refuse to fix this problem!
452 457 '--exe',
453 458 ]
454 459
455 460 if nose.__version__ >= '0.11':
456 461 # I don't fully understand why we need this one, but depending on what
457 462 # directory the test suite is run from, if we don't give it, 0 tests
458 463 # get run. Specifically, if the test suite is run from the source dir
459 464 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
460 465 # even if the same call done in this directory works fine). It appears
461 466 # that if the requested package is in the current dir, nose bails early
462 467 # by default. Since it's otherwise harmless, leave it in by default
463 468 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
464 469 argv.append('--traverse-namespace')
465 470
466 471 # use our plugin for doctesting. It will remove the standard doctest plugin
467 472 # if it finds it enabled
468 473 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
469 474 # We need a global ipython running in this process
470 475 globalipapp.start_ipython()
471 476 # Now nose can run
472 477 TestProgram(argv=argv, addplugins=plugins)
473 478
474 479
475 480 def run_iptestall():
476 481 """Run the entire IPython test suite by calling nose and trial.
477 482
478 483 This function constructs :class:`IPTester` instances for all IPython
479 484 modules and package and then runs each of them. This causes the modules
480 485 and packages of IPython to be tested each in their own subprocess using
481 486 nose.
482 487 """
483 488
484 489 runners = make_runners()
485 490
486 491 # Run the test runners in a temporary dir so we can nuke it when finished
487 492 # to clean up any junk files left over by accident. This also makes it
488 493 # robust against being run in non-writeable directories by mistake, as the
489 494 # temp dir will always be user-writeable.
490 495 curdir = os.getcwdu()
491 496 testdir = tempfile.gettempdir()
492 497 os.chdir(testdir)
493 498
494 499 # Run all test runners, tracking execution time
495 500 failed = []
496 501 t_start = time.time()
497 502 try:
498 503 for (name, runner) in runners:
499 504 print '*'*70
500 505 print 'IPython test group:',name
501 506 res = runner.run()
502 507 if res:
503 508 failed.append( (name, runner) )
504 509 finally:
505 510 os.chdir(curdir)
506 511 t_end = time.time()
507 512 t_tests = t_end - t_start
508 513 nrunners = len(runners)
509 514 nfail = len(failed)
510 515 # summarize results
511 516 print
512 517 print '*'*70
513 518 print 'Test suite completed for system with the following information:'
514 519 print report()
515 520 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
516 521 print
517 522 print 'Status:'
518 523 if not failed:
519 524 print 'OK'
520 525 else:
521 526 # If anything went wrong, point out what command to rerun manually to
522 527 # see the actual errors and individual summary
523 528 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
524 529 for name, failed_runner in failed:
525 530 print '-'*40
526 531 print 'Runner failed:',name
527 532 print 'You may wish to rerun this one individually, with:'
528 533 print ' '.join(failed_runner.call_args)
529 534 print
530 535 # Ensure that our exit code indicates failure
531 536 sys.exit(1)
532 537
533 538
534 539 def main():
535 540 for arg in sys.argv[1:]:
536 541 if arg.startswith('IPython'):
537 542 # This is in-process
538 543 run_iptest()
539 544 else:
540 545 # This starts subprocesses
541 546 run_iptestall()
542 547
543 548
544 549 if __name__ == '__main__':
545 550 main()
General Comments 0
You need to be logged in to leave comments. Login now