##// END OF EJS Templates
Merge pull request #11784 from Carreau/clean-iptest...
Matthias Bussonnier -
r25088:5cb4865e merge
parent child Browse files
Show More
@@ -1,507 +1,492
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Process Controller
3 3
4 4 This module runs one or more subprocesses which will actually run the IPython
5 5 test suite.
6 6
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12
13 13 import argparse
14 14 import multiprocessing.pool
15 15 import os
16 16 import stat
17 17 import shutil
18 18 import signal
19 19 import sys
20 20 import subprocess
21 21 import time
22 22
23 23 from .iptest import (
24 24 have, test_group_names as py_test_group_names, test_sections, StreamCapturer,
25 25 )
26 26 from IPython.utils.path import compress_user
27 27 from IPython.utils.py3compat import decode
28 28 from IPython.utils.sysinfo import get_sys_info
29 29 from IPython.utils.tempdir import TemporaryDirectory
30 30
31 class TestController(object):
31 class TestController:
32 32 """Run tests in a subprocess
33 33 """
34 34 #: str, IPython test suite to be executed.
35 35 section = None
36 36 #: list, command line arguments to be executed
37 37 cmd = None
38 38 #: dict, extra environment variables to set for the subprocess
39 39 env = None
40 40 #: list, TemporaryDirectory instances to clear up when the process finishes
41 41 dirs = None
42 42 #: subprocess.Popen instance
43 43 process = None
44 44 #: str, process stdout+stderr
45 45 stdout = None
46 46
47 47 def __init__(self):
48 48 self.cmd = []
49 49 self.env = {}
50 50 self.dirs = []
51 51
52 52 def setup(self):
53 53 """Create temporary directories etc.
54 54
55 55 This is only called when we know the test group will be run. Things
56 56 created here may be cleaned up by self.cleanup().
57 57 """
58 58 pass
59 59
60 60 def launch(self, buffer_output=False, capture_output=False):
61 61 # print('*** ENV:', self.env) # dbg
62 62 # print('*** CMD:', self.cmd) # dbg
63 63 env = os.environ.copy()
64 64 env.update(self.env)
65 65 if buffer_output:
66 66 capture_output = True
67 67 self.stdout_capturer = c = StreamCapturer(echo=not buffer_output)
68 68 c.start()
69 69 stdout = c.writefd if capture_output else None
70 70 stderr = subprocess.STDOUT if capture_output else None
71 71 self.process = subprocess.Popen(self.cmd, stdout=stdout,
72 72 stderr=stderr, env=env)
73 73
74 74 def wait(self):
75 75 self.process.wait()
76 76 self.stdout_capturer.halt()
77 77 self.stdout = self.stdout_capturer.get_buffer()
78 78 return self.process.returncode
79 79
80 def print_extra_info(self):
81 """Print extra information about this test run.
82
83 If we're running in parallel and showing the concise view, this is only
84 called if the test group fails. Otherwise, it's called before the test
85 group is started.
86
87 The base implementation does nothing, but it can be overridden by
88 subclasses.
89 """
90 return
91
92 80 def cleanup_process(self):
93 81 """Cleanup on exit by killing any leftover processes."""
94 82 subp = self.process
95 83 if subp is None or (subp.poll() is not None):
96 84 return # Process doesn't exist, or is already dead.
97 85
98 86 try:
99 87 print('Cleaning up stale PID: %d' % subp.pid)
100 88 subp.kill()
101 89 except: # (OSError, WindowsError) ?
102 90 # This is just a best effort, if we fail or the process was
103 91 # really gone, ignore it.
104 92 pass
105 93 else:
106 94 for i in range(10):
107 95 if subp.poll() is None:
108 96 time.sleep(0.1)
109 97 else:
110 98 break
111 99
112 100 if subp.poll() is None:
113 101 # The process did not die...
114 102 print('... failed. Manual cleanup may be required.')
115 103
116 104 def cleanup(self):
117 105 "Kill process if it's still alive, and clean up temporary directories"
118 106 self.cleanup_process()
119 107 for td in self.dirs:
120 108 td.cleanup()
121 109
122 110 __del__ = cleanup
123 111
124 112
125 113 class PyTestController(TestController):
126 114 """Run Python tests using IPython.testing.iptest"""
127 115 #: str, Python command to execute in subprocess
128 116 pycmd = None
129 117
130 118 def __init__(self, section, options):
131 119 """Create new test runner."""
132 120 TestController.__init__(self)
133 121 self.section = section
134 122 # pycmd is put into cmd[2] in PyTestController.launch()
135 123 self.cmd = [sys.executable, '-c', None, section]
136 124 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
137 125 self.options = options
138 126
139 127 def setup(self):
140 128 ipydir = TemporaryDirectory()
141 129 self.dirs.append(ipydir)
142 130 self.env['IPYTHONDIR'] = ipydir.name
143 131 self.workingdir = workingdir = TemporaryDirectory()
144 132 self.dirs.append(workingdir)
145 133 self.env['IPTEST_WORKING_DIR'] = workingdir.name
146 134 # This means we won't get odd effects from our own matplotlib config
147 135 self.env['MPLCONFIGDIR'] = workingdir.name
148 136 # For security reasons (http://bugs.python.org/issue16202), use
149 137 # a temporary directory to which other users have no access.
150 138 self.env['TMPDIR'] = workingdir.name
151 139
152 140 # Add a non-accessible directory to PATH (see gh-7053)
153 141 noaccess = os.path.join(self.workingdir.name, "_no_access_")
154 142 self.noaccess = noaccess
155 143 os.mkdir(noaccess, 0)
156 144
157 145 PATH = os.environ.get('PATH', '')
158 146 if PATH:
159 147 PATH = noaccess + os.pathsep + PATH
160 148 else:
161 149 PATH = noaccess
162 150 self.env['PATH'] = PATH
163 151
164 152 # From options:
165 153 if self.options.xunit:
166 154 self.add_xunit()
167 155 if self.options.coverage:
168 156 self.add_coverage()
169 157 self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
170 158 self.cmd.extend(self.options.extra_args)
171 159
172 160 def cleanup(self):
173 161 """
174 162 Make the non-accessible directory created in setup() accessible
175 163 again, otherwise deleting the workingdir will fail.
176 164 """
177 165 os.chmod(self.noaccess, stat.S_IRWXU)
178 166 TestController.cleanup(self)
179 167
180 168 @property
181 169 def will_run(self):
182 170 try:
183 171 return test_sections[self.section].will_run
184 172 except KeyError:
185 173 return True
186 174
187 175 def add_xunit(self):
188 176 xunit_file = os.path.abspath(self.section + '.xunit.xml')
189 177 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
190 178
191 179 def add_coverage(self):
192 180 try:
193 181 sources = test_sections[self.section].includes
194 182 except KeyError:
195 183 sources = ['IPython']
196 184
197 185 coverage_rc = ("[run]\n"
198 186 "data_file = {data_file}\n"
199 187 "source =\n"
200 188 " {source}\n"
201 189 ).format(data_file=os.path.abspath('.coverage.'+self.section),
202 190 source="\n ".join(sources))
203 191 config_file = os.path.join(self.workingdir.name, '.coveragerc')
204 192 with open(config_file, 'w') as f:
205 193 f.write(coverage_rc)
206 194
207 195 self.env['COVERAGE_PROCESS_START'] = config_file
208 196 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
209 197
210 198 def launch(self, buffer_output=False):
211 199 self.cmd[2] = self.pycmd
212 200 super(PyTestController, self).launch(buffer_output=buffer_output)
213 201
214 202
215 203 def prepare_controllers(options):
216 204 """Returns two lists of TestController instances, those to run, and those
217 205 not to run."""
218 206 testgroups = options.testgroups
219 207 if not testgroups:
220 208 testgroups = py_test_group_names
221 209
222 210 controllers = [PyTestController(name, options) for name in testgroups]
223 211
224 212 to_run = [c for c in controllers if c.will_run]
225 213 not_run = [c for c in controllers if not c.will_run]
226 214 return to_run, not_run
227 215
228 216 def do_run(controller, buffer_output=True):
229 217 """Setup and run a test controller.
230 218
231 219 If buffer_output is True, no output is displayed, to avoid it appearing
232 220 interleaved. In this case, the caller is responsible for displaying test
233 221 output on failure.
234 222
235 223 Returns
236 224 -------
237 225 controller : TestController
238 226 The same controller as passed in, as a convenience for using map() type
239 227 APIs.
240 228 exitcode : int
241 229 The exit code of the test subprocess. Non-zero indicates failure.
242 230 """
243 231 try:
244 232 try:
245 233 controller.setup()
246 if not buffer_output:
247 controller.print_extra_info()
248 234 controller.launch(buffer_output=buffer_output)
249 235 except Exception:
250 236 import traceback
251 237 traceback.print_exc()
252 238 return controller, 1 # signal failure
253 239
254 240 exitcode = controller.wait()
255 241 return controller, exitcode
256 242
257 243 except KeyboardInterrupt:
258 244 return controller, -signal.SIGINT
259 245 finally:
260 246 controller.cleanup()
261 247
262 248 def report():
263 249 """Return a string with a summary report of test-related variables."""
264 250 inf = get_sys_info()
265 251 out = []
266 252 def _add(name, value):
267 253 out.append((name, value))
268 254
269 255 _add('IPython version', inf['ipython_version'])
270 256 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
271 257 _add('IPython package', compress_user(inf['ipython_path']))
272 258 _add('Python version', inf['sys_version'].replace('\n',''))
273 259 _add('sys.executable', compress_user(inf['sys_executable']))
274 260 _add('Platform', inf['platform'])
275 261
276 262 width = max(len(n) for (n,v) in out)
277 263 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
278 264
279 265 avail = []
280 266 not_avail = []
281 267
282 268 for k, is_avail in have.items():
283 269 if is_avail:
284 270 avail.append(k)
285 271 else:
286 272 not_avail.append(k)
287 273
288 274 if avail:
289 275 out.append('\nTools and libraries available at test time:\n')
290 276 avail.sort()
291 277 out.append(' ' + ' '.join(avail)+'\n')
292 278
293 279 if not_avail:
294 280 out.append('\nTools and libraries NOT available at test time:\n')
295 281 not_avail.sort()
296 282 out.append(' ' + ' '.join(not_avail)+'\n')
297 283
298 284 return ''.join(out)
299 285
300 286 def run_iptestall(options):
301 287 """Run the entire IPython test suite by calling nose and trial.
302 288
303 289 This function constructs :class:`IPTester` instances for all IPython
304 290 modules and package and then runs each of them. This causes the modules
305 291 and packages of IPython to be tested each in their own subprocess using
306 292 nose.
307 293
308 294 Parameters
309 295 ----------
310 296
311 297 All parameters are passed as attributes of the options object.
312 298
313 299 testgroups : list of str
314 300 Run only these sections of the test suite. If empty, run all the available
315 301 sections.
316 302
317 303 fast : int or None
318 304 Run the test suite in parallel, using n simultaneous processes. If None
319 305 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
320 306
321 307 inc_slow : bool
322 308 Include slow tests. By default, these tests aren't run.
323 309
324 310 url : unicode
325 311 Address:port to use when running the JS tests.
326 312
327 313 xunit : bool
328 314 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
329 315
330 316 coverage : bool or str
331 317 Measure code coverage from tests. True will store the raw coverage data,
332 318 or pass 'html' or 'xml' to get reports.
333 319
334 320 extra_args : list
335 321 Extra arguments to pass to the test subprocesses, e.g. '-v'
336 322 """
337 323 to_run, not_run = prepare_controllers(options)
338 324
339 325 def justify(ltext, rtext, width=70, fill='-'):
340 326 ltext += ' '
341 327 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
342 328 return ltext + rtext
343 329
344 330 # Run all test runners, tracking execution time
345 331 failed = []
346 332 t_start = time.time()
347 333
348 334 print()
349 335 if options.fast == 1:
350 336 # This actually means sequential, i.e. with 1 job
351 337 for controller in to_run:
352 338 print('Test group:', controller.section)
353 339 sys.stdout.flush() # Show in correct order when output is piped
354 340 controller, res = do_run(controller, buffer_output=False)
355 341 if res:
356 342 failed.append(controller)
357 343 if res == -signal.SIGINT:
358 344 print("Interrupted")
359 345 break
360 346 print()
361 347
362 348 else:
363 349 # Run tests concurrently
364 350 try:
365 351 pool = multiprocessing.pool.ThreadPool(options.fast)
366 352 for (controller, res) in pool.imap_unordered(do_run, to_run):
367 353 res_string = 'OK' if res == 0 else 'FAILED'
368 354 print(justify('Test group: ' + controller.section, res_string))
369 355 if res:
370 controller.print_extra_info()
371 356 print(decode(controller.stdout))
372 357 failed.append(controller)
373 358 if res == -signal.SIGINT:
374 359 print("Interrupted")
375 360 break
376 361 except KeyboardInterrupt:
377 362 return
378 363
379 364 for controller in not_run:
380 365 print(justify('Test group: ' + controller.section, 'NOT RUN'))
381 366
382 367 t_end = time.time()
383 368 t_tests = t_end - t_start
384 369 nrunners = len(to_run)
385 370 nfail = len(failed)
386 371 # summarize results
387 372 print('_'*70)
388 373 print('Test suite completed for system with the following information:')
389 374 print(report())
390 375 took = "Took %.3fs." % t_tests
391 376 print('Status: ', end='')
392 377 if not failed:
393 378 print('OK (%d test groups).' % nrunners, took)
394 379 else:
395 380 # If anything went wrong, point out what command to rerun manually to
396 381 # see the actual errors and individual summary
397 382 failed_sections = [c.section for c in failed]
398 383 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
399 384 nrunners, ', '.join(failed_sections)), took)
400 385 print()
401 386 print('You may wish to rerun these, with:')
402 387 print(' iptest', *failed_sections)
403 388 print()
404 389
405 390 if options.coverage:
406 391 from coverage import coverage, CoverageException
407 392 cov = coverage(data_file='.coverage')
408 393 cov.combine()
409 394 cov.save()
410 395
411 396 # Coverage HTML report
412 397 if options.coverage == 'html':
413 398 html_dir = 'ipy_htmlcov'
414 399 shutil.rmtree(html_dir, ignore_errors=True)
415 400 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
416 401 sys.stdout.flush()
417 402
418 403 # Custom HTML reporter to clean up module names.
419 404 from coverage.html import HtmlReporter
420 405 class CustomHtmlReporter(HtmlReporter):
421 406 def find_code_units(self, morfs):
422 407 super(CustomHtmlReporter, self).find_code_units(morfs)
423 408 for cu in self.code_units:
424 409 nameparts = cu.name.split(os.sep)
425 410 if 'IPython' not in nameparts:
426 411 continue
427 412 ix = nameparts.index('IPython')
428 413 cu.name = '.'.join(nameparts[ix:])
429 414
430 415 # Reimplement the html_report method with our custom reporter
431 416 cov.get_data()
432 417 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
433 418 html_title='IPython test coverage',
434 419 )
435 420 reporter = CustomHtmlReporter(cov, cov.config)
436 421 reporter.report(None)
437 422 print('done.')
438 423
439 424 # Coverage XML report
440 425 elif options.coverage == 'xml':
441 426 try:
442 427 cov.xml_report(outfile='ipy_coverage.xml')
443 428 except CoverageException as e:
444 429 print('Generating coverage report failed. Are you running javascript tests only?')
445 430 import traceback
446 431 traceback.print_exc()
447 432
448 433 if failed:
449 434 # Ensure that our exit code indicates failure
450 435 sys.exit(1)
451 436
452 437 argparser = argparse.ArgumentParser(description='Run IPython test suite')
453 438 argparser.add_argument('testgroups', nargs='*',
454 439 help='Run specified groups of tests. If omitted, run '
455 440 'all tests.')
456 441 argparser.add_argument('--all', action='store_true',
457 442 help='Include slow tests not run by default.')
458 443 argparser.add_argument('--url', help="URL to use for the JS tests.")
459 444 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
460 445 help='Run test sections in parallel. This starts as many '
461 446 'processes as you have cores, or you can specify a number.')
462 447 argparser.add_argument('--xunit', action='store_true',
463 448 help='Produce Xunit XML results')
464 449 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
465 450 help="Measure test coverage. Specify 'html' or "
466 451 "'xml' to get reports.")
467 452 argparser.add_argument('--subproc-streams', default='capture',
468 453 help="What to do with stdout/stderr from subprocesses. "
469 454 "'capture' (default), 'show' and 'discard' are the options.")
470 455
471 456 def default_options():
472 457 """Get an argparse Namespace object with the default arguments, to pass to
473 458 :func:`run_iptestall`.
474 459 """
475 460 options = argparser.parse_args([])
476 461 options.extra_args = []
477 462 return options
478 463
479 464 def main():
480 465 # iptest doesn't work correctly if the working directory is the
481 466 # root of the IPython source tree. Tell the user to avoid
482 467 # frustration.
483 468 if os.path.exists(os.path.join(os.getcwd(),
484 469 'IPython', 'testing', '__main__.py')):
485 470 print("Don't run iptest from the IPython source directory",
486 471 file=sys.stderr)
487 472 sys.exit(1)
488 473 # Arguments after -- should be passed through to nose. Argparse treats
489 474 # everything after -- as regular positional arguments, so we separate them
490 475 # first.
491 476 try:
492 477 ix = sys.argv.index('--')
493 478 except ValueError:
494 479 to_parse = sys.argv[1:]
495 480 extra_args = []
496 481 else:
497 482 to_parse = sys.argv[1:ix]
498 483 extra_args = sys.argv[ix+1:]
499 484
500 485 options = argparser.parse_args(to_parse)
501 486 options.extra_args = extra_args
502 487
503 488 run_iptestall(options)
504 489
505 490
506 491 if __name__ == '__main__':
507 492 main()
General Comments 0
You need to be logged in to leave comments. Login now