##// END OF EJS Templates
Split js tests in N subgroups....
Matthias Bussonnier -
Show More
@@ -1,35 +1,44 b''
1 1 # http://travis-ci.org/#!/ipython/ipython
2 2 language: python
3 3 python:
4 4 - 3.4
5 5 - 2.7
6 6 - 3.3
7 7 env:
8 - GROUP=js
8 - GROUP=js/base
9 - GROUP=js/notebook
10 - GROUP=js/tree
11 - GROUP=js/widgets
9 12 - GROUP=
10 13 before_install:
11 14 # workaround for https://github.com/travis-ci/travis-cookbooks/issues/155
12 15 - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
13 16 # Pierre Carrier's PPA for PhantomJS and CasperJS
14 17 - sudo add-apt-repository -y ppa:pcarrier/ppa
15 18 # Needed to get recent version of pandoc in ubntu 12.04
16 19 - sudo add-apt-repository -y ppa:marutter/c2d4u
17 20 - sudo apt-get update
18 21 - sudo apt-get install pandoc casperjs libzmq3-dev
19 22 - git clone --quiet --depth 1 https://github.com/minrk/travis-wheels travis-wheels
20 23 - 'if [[ $GROUP == js* ]]; then python -m IPython.external.mathjax; fi'
21 24 install:
22 25 - pip install coveralls
23 26 - pip install -f travis-wheels/wheelhouse file://$PWD#egg=ipython[all]
24 27 script:
25 28 - cd /tmp && iptest $GROUP --coverage xml && cd -
26 29
27 30 matrix:
28 31 exclude:
29 32 - python: 3.3
30 env: GROUP=js
33 env: GROUP=js/base
34 - python: 3.3
35 env: GROUP=js/notebook
36 - python: 3.3
37 env: GROUP=js/tree
38 - python: 3.3
39 env: GROUP=js/widgets
31 40
32 41 after_success:
33 42 - cp /tmp/ipy_coverage.xml ./
34 43 - cp /tmp/.coverage ./
35 44 - coveralls
@@ -1,742 +1,739 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Process Controller
3 3
4 4 This module runs one or more subprocesses which will actually run the IPython
5 5 test suite.
6 6
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 from __future__ import print_function
13 13
14 14 import argparse
15 15 import json
16 16 import multiprocessing.pool
17 17 import os
18 18 import stat
19 19 import re
20 20 import requests
21 21 import shutil
22 22 import signal
23 23 import sys
24 24 import subprocess
25 25 import time
26 26
27 27 from .iptest import (
28 28 have, test_group_names as py_test_group_names, test_sections, StreamCapturer,
29 29 test_for,
30 30 )
31 31 from IPython.utils.path import compress_user
32 32 from IPython.utils.py3compat import bytes_to_str
33 33 from IPython.utils.sysinfo import get_sys_info
34 34 from IPython.utils.tempdir import TemporaryDirectory
35 35 from IPython.utils.text import strip_ansi
36 36
37 37 try:
38 38 # Python >= 3.3
39 39 from subprocess import TimeoutExpired
40 40 def popen_wait(p, timeout):
41 41 return p.wait(timeout)
42 42 except ImportError:
43 43 class TimeoutExpired(Exception):
44 44 pass
45 45 def popen_wait(p, timeout):
46 46 """backport of Popen.wait from Python 3"""
47 47 for i in range(int(10 * timeout)):
48 48 if p.poll() is not None:
49 49 return
50 50 time.sleep(0.1)
51 51 if p.poll() is None:
52 52 raise TimeoutExpired
53 53
54 54 NOTEBOOK_SHUTDOWN_TIMEOUT = 10
55 55
56 56 class TestController(object):
57 57 """Run tests in a subprocess
58 58 """
59 59 #: str, IPython test suite to be executed.
60 60 section = None
61 61 #: list, command line arguments to be executed
62 62 cmd = None
63 63 #: dict, extra environment variables to set for the subprocess
64 64 env = None
65 65 #: list, TemporaryDirectory instances to clear up when the process finishes
66 66 dirs = None
67 67 #: subprocess.Popen instance
68 68 process = None
69 69 #: str, process stdout+stderr
70 70 stdout = None
71 71
72 72 def __init__(self):
73 73 self.cmd = []
74 74 self.env = {}
75 75 self.dirs = []
76 76
77 77 def setup(self):
78 78 """Create temporary directories etc.
79 79
80 80 This is only called when we know the test group will be run. Things
81 81 created here may be cleaned up by self.cleanup().
82 82 """
83 83 pass
84 84
85 85 def launch(self, buffer_output=False, capture_output=False):
86 86 # print('*** ENV:', self.env) # dbg
87 87 # print('*** CMD:', self.cmd) # dbg
88 88 env = os.environ.copy()
89 89 env.update(self.env)
90 90 if buffer_output:
91 91 capture_output = True
92 92 self.stdout_capturer = c = StreamCapturer(echo=not buffer_output)
93 93 c.start()
94 94 stdout = c.writefd if capture_output else None
95 95 stderr = subprocess.STDOUT if capture_output else None
96 96 self.process = subprocess.Popen(self.cmd, stdout=stdout,
97 97 stderr=stderr, env=env)
98 98
99 99 def wait(self):
100 100 self.process.wait()
101 101 self.stdout_capturer.halt()
102 102 self.stdout = self.stdout_capturer.get_buffer()
103 103 return self.process.returncode
104 104
105 105 def print_extra_info(self):
106 106 """Print extra information about this test run.
107 107
108 108 If we're running in parallel and showing the concise view, this is only
109 109 called if the test group fails. Otherwise, it's called before the test
110 110 group is started.
111 111
112 112 The base implementation does nothing, but it can be overridden by
113 113 subclasses.
114 114 """
115 115 return
116 116
117 117 def cleanup_process(self):
118 118 """Cleanup on exit by killing any leftover processes."""
119 119 subp = self.process
120 120 if subp is None or (subp.poll() is not None):
121 121 return # Process doesn't exist, or is already dead.
122 122
123 123 try:
124 124 print('Cleaning up stale PID: %d' % subp.pid)
125 125 subp.kill()
126 126 except: # (OSError, WindowsError) ?
127 127 # This is just a best effort, if we fail or the process was
128 128 # really gone, ignore it.
129 129 pass
130 130 else:
131 131 for i in range(10):
132 132 if subp.poll() is None:
133 133 time.sleep(0.1)
134 134 else:
135 135 break
136 136
137 137 if subp.poll() is None:
138 138 # The process did not die...
139 139 print('... failed. Manual cleanup may be required.')
140 140
141 141 def cleanup(self):
142 142 "Kill process if it's still alive, and clean up temporary directories"
143 143 self.cleanup_process()
144 144 for td in self.dirs:
145 145 td.cleanup()
146 146
147 147 __del__ = cleanup
148 148
149 149
150 150 class PyTestController(TestController):
151 151 """Run Python tests using IPython.testing.iptest"""
152 152 #: str, Python command to execute in subprocess
153 153 pycmd = None
154 154
155 155 def __init__(self, section, options):
156 156 """Create new test runner."""
157 157 TestController.__init__(self)
158 158 self.section = section
159 159 # pycmd is put into cmd[2] in PyTestController.launch()
160 160 self.cmd = [sys.executable, '-c', None, section]
161 161 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
162 162 self.options = options
163 163
164 164 def setup(self):
165 165 ipydir = TemporaryDirectory()
166 166 self.dirs.append(ipydir)
167 167 self.env['IPYTHONDIR'] = ipydir.name
168 168 self.workingdir = workingdir = TemporaryDirectory()
169 169 self.dirs.append(workingdir)
170 170 self.env['IPTEST_WORKING_DIR'] = workingdir.name
171 171 # This means we won't get odd effects from our own matplotlib config
172 172 self.env['MPLCONFIGDIR'] = workingdir.name
173 173 # For security reasons (http://bugs.python.org/issue16202), use
174 174 # a temporary directory to which other users have no access.
175 175 self.env['TMPDIR'] = workingdir.name
176 176
177 177 # Add a non-accessible directory to PATH (see gh-7053)
178 178 noaccess = os.path.join(self.workingdir.name, "_no_access_")
179 179 self.noaccess = noaccess
180 180 os.mkdir(noaccess, 0)
181 181
182 182 PATH = os.environ.get('PATH', '')
183 183 if PATH:
184 184 PATH = noaccess + os.pathsep + PATH
185 185 else:
186 186 PATH = noaccess
187 187 self.env['PATH'] = PATH
188 188
189 189 # From options:
190 190 if self.options.xunit:
191 191 self.add_xunit()
192 192 if self.options.coverage:
193 193 self.add_coverage()
194 194 self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
195 195 self.cmd.extend(self.options.extra_args)
196 196
197 197 def cleanup(self):
198 198 """
199 199 Make the non-accessible directory created in setup() accessible
200 200 again, otherwise deleting the workingdir will fail.
201 201 """
202 202 os.chmod(self.noaccess, stat.S_IRWXU)
203 203 TestController.cleanup(self)
204 204
205 205 @property
206 206 def will_run(self):
207 207 try:
208 208 return test_sections[self.section].will_run
209 209 except KeyError:
210 210 return True
211 211
212 212 def add_xunit(self):
213 213 xunit_file = os.path.abspath(self.section + '.xunit.xml')
214 214 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
215 215
216 216 def add_coverage(self):
217 217 try:
218 218 sources = test_sections[self.section].includes
219 219 except KeyError:
220 220 sources = ['IPython']
221 221
222 222 coverage_rc = ("[run]\n"
223 223 "data_file = {data_file}\n"
224 224 "source =\n"
225 225 " {source}\n"
226 226 ).format(data_file=os.path.abspath('.coverage.'+self.section),
227 227 source="\n ".join(sources))
228 228 config_file = os.path.join(self.workingdir.name, '.coveragerc')
229 229 with open(config_file, 'w') as f:
230 230 f.write(coverage_rc)
231 231
232 232 self.env['COVERAGE_PROCESS_START'] = config_file
233 233 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
234 234
235 235 def launch(self, buffer_output=False):
236 236 self.cmd[2] = self.pycmd
237 237 super(PyTestController, self).launch(buffer_output=buffer_output)
238 238
239 239
240 240 js_prefix = 'js/'
241 241
242 242 def get_js_test_dir():
243 243 import IPython.html.tests as t
244 244 return os.path.join(os.path.dirname(t.__file__), '')
245 245
246 246 def all_js_groups():
247 247 import glob
248 248 test_dir = get_js_test_dir()
249 249 all_subdirs = glob.glob(test_dir + '[!_]*/')
250 250 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs]
251 251
252 252 class JSController(TestController):
253 253 """Run CasperJS tests """
254 254
255 255 requirements = ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3',
256 256 'jsonschema']
257 257
258 258 def __init__(self, section, xunit=True, engine='phantomjs', url=None):
259 259 """Create new test runner."""
260 260 TestController.__init__(self)
261 261 self.engine = engine
262 262 self.section = section
263 263 self.xunit = xunit
264 264 self.url = url
265 265 self.slimer_failure = re.compile('^FAIL.*', flags=re.MULTILINE)
266 266 js_test_dir = get_js_test_dir()
267 267 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
268 268 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
269 269 self.cmd = ['casperjs', 'test', includes, test_cases, '--engine=%s' % self.engine]
270 270
271 271 def setup(self):
272 272 self.ipydir = TemporaryDirectory()
273 273 self.nbdir = TemporaryDirectory()
274 274 self.dirs.append(self.ipydir)
275 275 self.dirs.append(self.nbdir)
276 276 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
277 277 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
278 278
279 279 if self.xunit:
280 280 self.add_xunit()
281 281
282 282 # If a url was specified, use that for the testing.
283 283 if self.url:
284 284 try:
285 285 alive = requests.get(self.url).status_code == 200
286 286 except:
287 287 alive = False
288 288
289 289 if alive:
290 290 self.cmd.append("--url=%s" % self.url)
291 291 else:
292 292 raise Exception('Could not reach "%s".' % self.url)
293 293 else:
294 294 # start the ipython notebook, so we get the port number
295 295 self.server_port = 0
296 296 self._init_server()
297 297 if self.server_port:
298 298 self.cmd.append("--port=%i" % self.server_port)
299 299 else:
300 300 # don't launch tests if the server didn't start
301 301 self.cmd = [sys.executable, '-c', 'raise SystemExit(1)']
302 302
303 303 def add_xunit(self):
304 304 xunit_file = os.path.abspath(self.section.replace('/','.') + '.xunit.xml')
305 305 self.cmd.append('--xunit=%s' % xunit_file)
306 306
307 307 def launch(self, buffer_output):
308 308 # If the engine is SlimerJS, we need to buffer the output because
309 309 # SlimerJS does not support exit codes, so CasperJS always returns 0.
310 310 if self.engine == 'slimerjs' and not buffer_output:
311 311 return super(JSController, self).launch(capture_output=True)
312 312
313 313 else:
314 314 return super(JSController, self).launch(buffer_output=buffer_output)
315 315
316 316 def wait(self, *pargs, **kwargs):
317 317 """Wait for the JSController to finish"""
318 318 ret = super(JSController, self).wait(*pargs, **kwargs)
319 319 # If this is a SlimerJS controller, check the captured stdout for
320 320 # errors. Otherwise, just return the return code.
321 321 if self.engine == 'slimerjs':
322 322 stdout = bytes_to_str(self.stdout)
323 323 if ret != 0:
324 324 # This could still happen e.g. if it's stopped by SIGINT
325 325 return ret
326 326 return bool(self.slimer_failure.search(strip_ansi(stdout)))
327 327 else:
328 328 return ret
329 329
330 330 def print_extra_info(self):
331 331 print("Running tests with notebook directory %r" % self.nbdir.name)
332 332
333 333 @property
334 334 def will_run(self):
335 335 should_run = all(have[a] for a in self.requirements + [self.engine])
336 336 return should_run
337 337
338 338 def _init_server(self):
339 339 "Start the notebook server in a separate process"
340 340 self.server_command = command = [sys.executable,
341 341 '-m', 'IPython.html',
342 342 '--no-browser',
343 343 '--ipython-dir', self.ipydir.name,
344 344 '--notebook-dir', self.nbdir.name,
345 345 ]
346 346 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
347 347 # which run afoul of ipc's maximum path length.
348 348 if sys.platform.startswith('linux'):
349 349 command.append('--KernelManager.transport=ipc')
350 350 self.stream_capturer = c = StreamCapturer()
351 351 c.start()
352 352 env = os.environ.copy()
353 353 if self.engine == 'phantomjs':
354 354 env['IPYTHON_ALLOW_DRAFT_WEBSOCKETS_FOR_PHANTOMJS'] = '1'
355 355 self.server = subprocess.Popen(command,
356 356 stdout=c.writefd,
357 357 stderr=subprocess.STDOUT,
358 358 cwd=self.nbdir.name,
359 359 env=env,
360 360 )
361 361 self.server_info_file = os.path.join(self.ipydir.name,
362 362 'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
363 363 )
364 364 self._wait_for_server()
365 365
366 366 def _wait_for_server(self):
367 367 """Wait 30 seconds for the notebook server to start"""
368 368 for i in range(300):
369 369 if self.server.poll() is not None:
370 370 return self._failed_to_start()
371 371 if os.path.exists(self.server_info_file):
372 372 try:
373 373 self._load_server_info()
374 374 except ValueError:
375 375 # If the server is halfway through writing the file, we may
376 376 # get invalid JSON; it should be ready next iteration.
377 377 pass
378 378 else:
379 379 return
380 380 time.sleep(0.1)
381 381 print("Notebook server-info file never arrived: %s" % self.server_info_file,
382 382 file=sys.stderr
383 383 )
384 384
385 385 def _failed_to_start(self):
386 386 """Notebook server exited prematurely"""
387 387 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
388 388 print("Notebook failed to start: ", file=sys.stderr)
389 389 print(self.server_command)
390 390 print(captured, file=sys.stderr)
391 391
392 392 def _load_server_info(self):
393 393 """Notebook server started, load connection info from JSON"""
394 394 with open(self.server_info_file) as f:
395 395 info = json.load(f)
396 396 self.server_port = info['port']
397 397
398 398 def cleanup(self):
399 399 try:
400 400 self.server.terminate()
401 401 except OSError:
402 402 # already dead
403 403 pass
404 404 # wait 10s for the server to shutdown
405 405 try:
406 406 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT)
407 407 except TimeoutExpired:
408 408 # server didn't terminate, kill it
409 409 try:
410 410 print("Failed to terminate notebook server, killing it.",
411 411 file=sys.stderr
412 412 )
413 413 self.server.kill()
414 414 except OSError:
415 415 # already dead
416 416 pass
417 417 # wait another 10s
418 418 try:
419 419 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT)
420 420 except TimeoutExpired:
421 421 print("Notebook server still running (%s)" % self.server_info_file,
422 422 file=sys.stderr
423 423 )
424 424
425 425 self.stream_capturer.halt()
426 426 TestController.cleanup(self)
427 427
428 428
429 429 def prepare_controllers(options):
430 430 """Returns two lists of TestController instances, those to run, and those
431 431 not to run."""
432 432 testgroups = options.testgroups
433 433 if testgroups:
434 if 'js' in testgroups:
435 js_testgroups = all_js_groups()
436 else:
437 js_testgroups = [g for g in testgroups if g.startswith(js_prefix)]
438
439 py_testgroups = [g for g in testgroups if g not in ['js'] + js_testgroups]
434 alljs = all_js_groups()
435 js_testgroups = [g for g in testgroups if g.startswith(js_prefix)]
436 py_testgroups = [g for g in testgroups if g not in alljs]
440 437 else:
441 438 py_testgroups = py_test_group_names
442 439 if not options.all:
443 440 js_testgroups = []
444 441 test_sections['parallel'].enabled = False
445 442 else:
446 443 js_testgroups = all_js_groups()
447 444
448 445 engine = 'slimerjs' if options.slimerjs else 'phantomjs'
449 446 c_js = [JSController(name, xunit=options.xunit, engine=engine, url=options.url) for name in js_testgroups]
450 447 c_py = [PyTestController(name, options) for name in py_testgroups]
451 448
452 449 controllers = c_py + c_js
453 450 to_run = [c for c in controllers if c.will_run]
454 451 not_run = [c for c in controllers if not c.will_run]
455 452 return to_run, not_run
456 453
457 454 def do_run(controller, buffer_output=True):
458 455 """Setup and run a test controller.
459 456
460 457 If buffer_output is True, no output is displayed, to avoid it appearing
461 458 interleaved. In this case, the caller is responsible for displaying test
462 459 output on failure.
463 460
464 461 Returns
465 462 -------
466 463 controller : TestController
467 464 The same controller as passed in, as a convenience for using map() type
468 465 APIs.
469 466 exitcode : int
470 467 The exit code of the test subprocess. Non-zero indicates failure.
471 468 """
472 469 try:
473 470 try:
474 471 controller.setup()
475 472 if not buffer_output:
476 473 controller.print_extra_info()
477 474 controller.launch(buffer_output=buffer_output)
478 475 except Exception:
479 476 import traceback
480 477 traceback.print_exc()
481 478 return controller, 1 # signal failure
482 479
483 480 exitcode = controller.wait()
484 481 return controller, exitcode
485 482
486 483 except KeyboardInterrupt:
487 484 return controller, -signal.SIGINT
488 485 finally:
489 486 controller.cleanup()
490 487
491 488 def report():
492 489 """Return a string with a summary report of test-related variables."""
493 490 inf = get_sys_info()
494 491 out = []
495 492 def _add(name, value):
496 493 out.append((name, value))
497 494
498 495 _add('IPython version', inf['ipython_version'])
499 496 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
500 497 _add('IPython package', compress_user(inf['ipython_path']))
501 498 _add('Python version', inf['sys_version'].replace('\n',''))
502 499 _add('sys.executable', compress_user(inf['sys_executable']))
503 500 _add('Platform', inf['platform'])
504 501
505 502 width = max(len(n) for (n,v) in out)
506 503 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
507 504
508 505 avail = []
509 506 not_avail = []
510 507
511 508 for k, is_avail in have.items():
512 509 if is_avail:
513 510 avail.append(k)
514 511 else:
515 512 not_avail.append(k)
516 513
517 514 if avail:
518 515 out.append('\nTools and libraries available at test time:\n')
519 516 avail.sort()
520 517 out.append(' ' + ' '.join(avail)+'\n')
521 518
522 519 if not_avail:
523 520 out.append('\nTools and libraries NOT available at test time:\n')
524 521 not_avail.sort()
525 522 out.append(' ' + ' '.join(not_avail)+'\n')
526 523
527 524 return ''.join(out)
528 525
529 526 def run_iptestall(options):
530 527 """Run the entire IPython test suite by calling nose and trial.
531 528
532 529 This function constructs :class:`IPTester` instances for all IPython
533 530 modules and package and then runs each of them. This causes the modules
534 531 and packages of IPython to be tested each in their own subprocess using
535 532 nose.
536 533
537 534 Parameters
538 535 ----------
539 536
540 537 All parameters are passed as attributes of the options object.
541 538
542 539 testgroups : list of str
543 540 Run only these sections of the test suite. If empty, run all the available
544 541 sections.
545 542
546 543 fast : int or None
547 544 Run the test suite in parallel, using n simultaneous processes. If None
548 545 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
549 546
550 547 inc_slow : bool
551 548 Include slow tests, like IPython.parallel. By default, these tests aren't
552 549 run.
553 550
554 551 slimerjs : bool
555 552 Use slimerjs if it's installed instead of phantomjs for casperjs tests.
556 553
557 554 url : unicode
558 555 Address:port to use when running the JS tests.
559 556
560 557 xunit : bool
561 558 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
562 559
563 560 coverage : bool or str
564 561 Measure code coverage from tests. True will store the raw coverage data,
565 562 or pass 'html' or 'xml' to get reports.
566 563
567 564 extra_args : list
568 565 Extra arguments to pass to the test subprocesses, e.g. '-v'
569 566 """
570 567 to_run, not_run = prepare_controllers(options)
571 568
572 569 def justify(ltext, rtext, width=70, fill='-'):
573 570 ltext += ' '
574 571 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
575 572 return ltext + rtext
576 573
577 574 # Run all test runners, tracking execution time
578 575 failed = []
579 576 t_start = time.time()
580 577
581 578 print()
582 579 if options.fast == 1:
583 580 # This actually means sequential, i.e. with 1 job
584 581 for controller in to_run:
585 582 print('Test group:', controller.section)
586 583 sys.stdout.flush() # Show in correct order when output is piped
587 584 controller, res = do_run(controller, buffer_output=False)
588 585 if res:
589 586 failed.append(controller)
590 587 if res == -signal.SIGINT:
591 588 print("Interrupted")
592 589 break
593 590 print()
594 591
595 592 else:
596 593 # Run tests concurrently
597 594 try:
598 595 pool = multiprocessing.pool.ThreadPool(options.fast)
599 596 for (controller, res) in pool.imap_unordered(do_run, to_run):
600 597 res_string = 'OK' if res == 0 else 'FAILED'
601 598 print(justify('Test group: ' + controller.section, res_string))
602 599 if res:
603 600 controller.print_extra_info()
604 601 print(bytes_to_str(controller.stdout))
605 602 failed.append(controller)
606 603 if res == -signal.SIGINT:
607 604 print("Interrupted")
608 605 break
609 606 except KeyboardInterrupt:
610 607 return
611 608
612 609 for controller in not_run:
613 610 print(justify('Test group: ' + controller.section, 'NOT RUN'))
614 611
615 612 t_end = time.time()
616 613 t_tests = t_end - t_start
617 614 nrunners = len(to_run)
618 615 nfail = len(failed)
619 616 # summarize results
620 617 print('_'*70)
621 618 print('Test suite completed for system with the following information:')
622 619 print(report())
623 620 took = "Took %.3fs." % t_tests
624 621 print('Status: ', end='')
625 622 if not failed:
626 623 print('OK (%d test groups).' % nrunners, took)
627 624 else:
628 625 # If anything went wrong, point out what command to rerun manually to
629 626 # see the actual errors and individual summary
630 627 failed_sections = [c.section for c in failed]
631 628 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
632 629 nrunners, ', '.join(failed_sections)), took)
633 630 print()
634 631 print('You may wish to rerun these, with:')
635 632 print(' iptest', *failed_sections)
636 633 print()
637 634
638 635 if options.coverage:
639 636 from coverage import coverage, CoverageException
640 637 cov = coverage(data_file='.coverage')
641 638 cov.combine()
642 639 cov.save()
643 640
644 641 # Coverage HTML report
645 642 if options.coverage == 'html':
646 643 html_dir = 'ipy_htmlcov'
647 644 shutil.rmtree(html_dir, ignore_errors=True)
648 645 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
649 646 sys.stdout.flush()
650 647
651 648 # Custom HTML reporter to clean up module names.
652 649 from coverage.html import HtmlReporter
653 650 class CustomHtmlReporter(HtmlReporter):
654 651 def find_code_units(self, morfs):
655 652 super(CustomHtmlReporter, self).find_code_units(morfs)
656 653 for cu in self.code_units:
657 654 nameparts = cu.name.split(os.sep)
658 655 if 'IPython' not in nameparts:
659 656 continue
660 657 ix = nameparts.index('IPython')
661 658 cu.name = '.'.join(nameparts[ix:])
662 659
663 660 # Reimplement the html_report method with our custom reporter
664 661 cov._harvest_data()
665 662 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
666 663 html_title='IPython test coverage',
667 664 )
668 665 reporter = CustomHtmlReporter(cov, cov.config)
669 666 reporter.report(None)
670 667 print('done.')
671 668
672 669 # Coverage XML report
673 670 elif options.coverage == 'xml':
674 671 try:
675 672 cov.xml_report(outfile='ipy_coverage.xml')
676 673 except CoverageException as e:
677 674 print('Generating coverage report failed. Are you running javascript tests only?')
678 675 import traceback
679 676 traceback.print_exc()
680 677
681 678 if failed:
682 679 # Ensure that our exit code indicates failure
683 680 sys.exit(1)
684 681
685 682 argparser = argparse.ArgumentParser(description='Run IPython test suite')
686 683 argparser.add_argument('testgroups', nargs='*',
687 684 help='Run specified groups of tests. If omitted, run '
688 685 'all tests.')
689 686 argparser.add_argument('--all', action='store_true',
690 687 help='Include slow tests not run by default.')
691 688 argparser.add_argument('--slimerjs', action='store_true',
692 689 help="Use slimerjs if it's installed instead of phantomjs for casperjs tests.")
693 690 argparser.add_argument('--url', help="URL to use for the JS tests.")
694 691 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
695 692 help='Run test sections in parallel. This starts as many '
696 693 'processes as you have cores, or you can specify a number.')
697 694 argparser.add_argument('--xunit', action='store_true',
698 695 help='Produce Xunit XML results')
699 696 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
700 697 help="Measure test coverage. Specify 'html' or "
701 698 "'xml' to get reports.")
702 699 argparser.add_argument('--subproc-streams', default='capture',
703 700 help="What to do with stdout/stderr from subprocesses. "
704 701 "'capture' (default), 'show' and 'discard' are the options.")
705 702
706 703 def default_options():
707 704 """Get an argparse Namespace object with the default arguments, to pass to
708 705 :func:`run_iptestall`.
709 706 """
710 707 options = argparser.parse_args([])
711 708 options.extra_args = []
712 709 return options
713 710
714 711 def main():
715 712 # iptest doesn't work correctly if the working directory is the
716 713 # root of the IPython source tree. Tell the user to avoid
717 714 # frustration.
718 715 if os.path.exists(os.path.join(os.getcwd(),
719 716 'IPython', 'testing', '__main__.py')):
720 717 print("Don't run iptest from the IPython source directory",
721 718 file=sys.stderr)
722 719 sys.exit(1)
723 720 # Arguments after -- should be passed through to nose. Argparse treats
724 721 # everything after -- as regular positional arguments, so we separate them
725 722 # first.
726 723 try:
727 724 ix = sys.argv.index('--')
728 725 except ValueError:
729 726 to_parse = sys.argv[1:]
730 727 extra_args = []
731 728 else:
732 729 to_parse = sys.argv[1:ix]
733 730 extra_args = sys.argv[ix+1:]
734 731
735 732 options = argparser.parse_args(to_parse)
736 733 options.extra_args = extra_args
737 734
738 735 run_iptestall(options)
739 736
740 737
741 738 if __name__ == '__main__':
742 739 main()
General Comments 0
You need to be logged in to leave comments. Login now