iptestcontroller.py
501 lines
| 17.0 KiB
| text/x-python
|
PythonLexer
|
r12607 | # -*- coding: utf-8 -*- | ||
"""IPython Test Process Controller | ||||
This module runs one or more subprocesses which will actually run the IPython | ||||
test suite. | ||||
""" | ||||
#----------------------------------------------------------------------------- | ||||
# Copyright (C) 2009-2011 The IPython Development Team | ||||
# | ||||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
#----------------------------------------------------------------------------- | ||||
from __future__ import print_function | ||||
|
r12608 | import argparse | ||
|
r12607 | import multiprocessing.pool | ||
|
r13265 | from multiprocessing import Process, Queue | ||
|
r12607 | import os | ||
|
r12612 | import shutil | ||
|
r12607 | import signal | ||
import sys | ||||
import subprocess | ||||
import time | ||||
|
r13279 | from .iptest import have, test_group_names as py_test_group_names, test_sections | ||
|
r13173 | from IPython.utils.path import compress_user | ||
|
r12619 | from IPython.utils.py3compat import bytes_to_str | ||
|
r13173 | from IPython.utils.sysinfo import get_sys_info | ||
|
r12607 | from IPython.utils.tempdir import TemporaryDirectory | ||
|
r12617 | class TestController(object): | ||
"""Run tests in a subprocess | ||||
|
r12607 | """ | ||
|
r12608 | #: str, IPython test suite to be executed. | ||
section = None | ||||
#: list, command line arguments to be executed | ||||
cmd = None | ||||
#: dict, extra environment variables to set for the subprocess | ||||
env = None | ||||
#: list, TemporaryDirectory instances to clear up when the process finishes | ||||
dirs = None | ||||
#: subprocess.Popen instance | ||||
process = None | ||||
|
r12617 | #: str, process stdout+stderr | ||
stdout = None | ||||
#: bool, whether to capture process stdout & stderr | ||||
|
r12607 | buffer_output = False | ||
|
r12617 | def __init__(self): | ||
self.cmd = [] | ||||
|
r12608 | self.env = {} | ||
self.dirs = [] | ||||
def launch(self): | ||||
# print('*** ENV:', self.env) # dbg | ||||
# print('*** CMD:', self.cmd) # dbg | ||||
env = os.environ.copy() | ||||
env.update(self.env) | ||||
output = subprocess.PIPE if self.buffer_output else None | ||||
|
r12610 | stdout = subprocess.STDOUT if self.buffer_output else None | ||
|
r12608 | self.process = subprocess.Popen(self.cmd, stdout=output, | ||
|
r12610 | stderr=stdout, env=env) | ||
|
r12607 | |||
|
r12610 | def wait(self): | ||
self.stdout, _ = self.process.communicate() | ||||
return self.process.returncode | ||||
|
r12607 | |||
|
r12609 | def cleanup_process(self): | ||
|
r12607 | """Cleanup on exit by killing any leftover processes.""" | ||
|
r12608 | subp = self.process | ||
if subp is None or (subp.poll() is not None): | ||||
return # Process doesn't exist, or is already dead. | ||||
|
r12607 | |||
|
r12608 | try: | ||
print('Cleaning up stale PID: %d' % subp.pid) | ||||
subp.kill() | ||||
except: # (OSError, WindowsError) ? | ||||
# This is just a best effort, if we fail or the process was | ||||
# really gone, ignore it. | ||||
pass | ||||
else: | ||||
for i in range(10): | ||||
if subp.poll() is None: | ||||
time.sleep(0.1) | ||||
else: | ||||
break | ||||
|
r12607 | |||
|
r12608 | if subp.poll() is None: | ||
# The process did not die... | ||||
print('... failed. Manual cleanup may be required.') | ||||
|
r13279 | |||
|
r12609 | def cleanup(self): | ||
"Kill process if it's still alive, and clean up temporary directories" | ||||
self.cleanup_process() | ||||
|
r12608 | for td in self.dirs: | ||
td.cleanup() | ||||
|
r13279 | |||
|
r12608 | __del__ = cleanup | ||
|
r12617 | class PyTestController(TestController): | ||
"""Run Python tests using IPython.testing.iptest""" | ||||
#: str, Python command to execute in subprocess | ||||
pycmd = None | ||||
def __init__(self, section): | ||||
"""Create new test runner.""" | ||||
TestController.__init__(self) | ||||
self.section = section | ||||
# pycmd is put into cmd[2] in PyTestController.launch() | ||||
self.cmd = [sys.executable, '-c', None, section] | ||||
self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" | ||||
ipydir = TemporaryDirectory() | ||||
self.dirs.append(ipydir) | ||||
self.env['IPYTHONDIR'] = ipydir.name | ||||
self.workingdir = workingdir = TemporaryDirectory() | ||||
self.dirs.append(workingdir) | ||||
self.env['IPTEST_WORKING_DIR'] = workingdir.name | ||||
# This means we won't get odd effects from our own matplotlib config | ||||
self.env['MPLCONFIGDIR'] = workingdir.name | ||||
|
r13279 | @property | ||
def will_run(self): | ||||
try: | ||||
return test_sections[self.section].will_run | ||||
except KeyError: | ||||
return True | ||||
|
r12617 | def add_xunit(self): | ||
xunit_file = os.path.abspath(self.section + '.xunit.xml') | ||||
self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) | ||||
def add_coverage(self): | ||||
|
r12623 | try: | ||
sources = test_sections[self.section].includes | ||||
except KeyError: | ||||
sources = ['IPython'] | ||||
|
r12617 | coverage_rc = ("[run]\n" | ||
"data_file = {data_file}\n" | ||||
"source =\n" | ||||
" {source}\n" | ||||
).format(data_file=os.path.abspath('.coverage.'+self.section), | ||||
|
r12623 | source="\n ".join(sources)) | ||
|
r12617 | config_file = os.path.join(self.workingdir.name, '.coveragerc') | ||
with open(config_file, 'w') as f: | ||||
f.write(coverage_rc) | ||||
self.env['COVERAGE_PROCESS_START'] = config_file | ||||
self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd | ||||
def launch(self): | ||||
self.cmd[2] = self.pycmd | ||||
super(PyTestController, self).launch() | ||||
|
r13264 | class JSController(TestController): | ||
"""Run CasperJS tests """ | ||||
def __init__(self, section): | ||||
"""Create new test runner.""" | ||||
TestController.__init__(self) | ||||
self.section = section | ||||
|
r13265 | |||
|
r13269 | self.ipydir = TemporaryDirectory() | ||
self.dirs.append(self.ipydir) | ||||
self.env['IPYTHONDIR'] = self.ipydir.name | ||||
|
r13279 | def launch(self): | ||
|
r13265 | # start the ipython notebook, so we get the port number | ||
self._init_server() | ||||
|
r13264 | import IPython.html.tests as t | ||
test_dir = os.path.join(os.path.dirname(t.__file__), 'casperjs') | ||||
includes = '--includes=' + os.path.join(test_dir,'util.js') | ||||
test_cases = os.path.join(test_dir, 'test_cases') | ||||
|
r13265 | port = '--port=' + str(self.server_port) | ||
self.cmd = ['casperjs', 'test', port, includes, test_cases] | ||||
|
r13279 | super(JSController, self).launch() | ||
@property | ||||
def will_run(self): | ||||
return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs']) | ||||
|
r13264 | |||
|
r13265 | def _init_server(self): | ||
"Start the notebook server in a separate process" | ||||
self.queue = q = Queue() | ||||
|
r13269 | self.server = Process(target=run_webapp, args=(q, self.ipydir.name)) | ||
self.server.start() | ||||
|
r13265 | self.server_port = q.get() | ||
def cleanup(self): | ||||
self.server.terminate() | ||||
|
r13269 | self.server.join() | ||
|
r13265 | TestController.cleanup(self) | ||
|
r13279 | js_test_group_names = {'js'} | ||
|
r13265 | |||
|
r13270 | def run_webapp(q, nbdir, loglevel=0): | ||
|
r13265 | """start the IPython Notebook, and pass port back to the queue""" | ||
|
r13287 | import os | ||
|
r13265 | import IPython.html.notebookapp as nbapp | ||
|
r13305 | import sys | ||
sys.stderr = open(os.devnull, 'w') | ||||
|
r13287 | os.environ["IPYTHONDIR"] = nbdir | ||
|
r13265 | server = nbapp.NotebookApp() | ||
|
r13270 | args = ['--no-browser'] | ||
args.append('--notebook-dir='+nbdir) | ||||
args.append('--profile-dir='+nbdir) | ||||
args.append('--log-level='+str(loglevel)) | ||||
server.initialize(args) | ||||
|
r13265 | # communicate the port number to the parent process | ||
q.put(server.port) | ||||
server.start() | ||||
|
r13264 | |||
def prepare_controllers(options): | ||||
|
r13262 | """Returns two lists of TestController instances, those to run, and those | ||
not to run.""" | ||||
|
r13264 | testgroups = options.testgroups | ||
|
r13279 | if testgroups: | ||
|
r13380 | py_testgroups = [g for g in testgroups if (g in py_test_group_names) \ | ||
or g.startswith('IPython')] | ||||
|
r13279 | js_testgroups = [g for g in testgroups if g in js_test_group_names] | ||
else: | ||||
py_testgroups = py_test_group_names | ||||
js_testgroups = js_test_group_names | ||||
|
r13264 | if not options.all: | ||
|
r13262 | test_sections['parallel'].enabled = False | ||
|
r13279 | c_js = [JSController(name) for name in js_testgroups] | ||
c_py = [PyTestController(name) for name in py_testgroups] | ||||
|
r13262 | |||
|
r13264 | configure_py_controllers(c_py, xunit=options.xunit, | ||
|
r13381 | coverage=options.coverage, extra_args=options.extra_args) | ||
|
r13279 | |||
|
r13264 | controllers = c_py + c_js | ||
|
r13262 | to_run = [c for c in controllers if c.will_run] | ||
not_run = [c for c in controllers if not c.will_run] | ||||
|
r12616 | return to_run, not_run | ||
|
r12608 | |||
|
r13264 | def configure_py_controllers(controllers, xunit=False, coverage=False, extra_args=()): | ||
|
r12623 | """Apply options for a collection of TestController objects.""" | ||
for controller in controllers: | ||||
if xunit: | ||||
controller.add_xunit() | ||||
if coverage: | ||||
controller.add_coverage() | ||||
|
r12638 | controller.cmd.extend(extra_args) | ||
|
r12623 | |||
|
r12608 | def do_run(controller): | ||
try: | ||||
try: | ||||
controller.launch() | ||||
except Exception: | ||||
import traceback | ||||
traceback.print_exc() | ||||
return controller, 1 # signal failure | ||||
|
r13279 | |||
|
r12610 | exitcode = controller.wait() | ||
|
r12608 | return controller, exitcode | ||
|
r13279 | |||
|
r12608 | except KeyboardInterrupt: | ||
return controller, -signal.SIGINT | ||||
|
r12610 | finally: | ||
controller.cleanup() | ||||
|
r12607 | |||
def report(): | ||||
"""Return a string with a summary report of test-related variables.""" | ||||
|
r13173 | inf = get_sys_info() | ||
out = [] | ||||
def _add(name, value): | ||||
out.append((name, value)) | ||||
|
r12607 | |||
|
r13173 | _add('IPython version', inf['ipython_version']) | ||
_add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source'])) | ||||
_add('IPython package', compress_user(inf['ipython_path'])) | ||||
_add('Python version', inf['sys_version'].replace('\n','')) | ||||
_add('sys.executable', compress_user(inf['sys_executable'])) | ||||
_add('Platform', inf['platform']) | ||||
width = max(len(n) for (n,v) in out) | ||||
out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] | ||||
|
r12607 | |||
avail = [] | ||||
not_avail = [] | ||||
for k, is_avail in have.items(): | ||||
if is_avail: | ||||
avail.append(k) | ||||
else: | ||||
not_avail.append(k) | ||||
if avail: | ||||
out.append('\nTools and libraries available at test time:\n') | ||||
avail.sort() | ||||
out.append(' ' + ' '.join(avail)+'\n') | ||||
if not_avail: | ||||
out.append('\nTools and libraries NOT available at test time:\n') | ||||
not_avail.sort() | ||||
out.append(' ' + ' '.join(not_avail)+'\n') | ||||
return ''.join(out) | ||||
|
r12623 | def run_iptestall(options): | ||
|
r12607 | """Run the entire IPython test suite by calling nose and trial. | ||
This function constructs :class:`IPTester` instances for all IPython | ||||
modules and package and then runs each of them. This causes the modules | ||||
and packages of IPython to be tested each in their own subprocess using | ||||
nose. | ||||
|
r13279 | |||
|
r12607 | Parameters | ||
---------- | ||||
|
r12623 | |||
All parameters are passed as attributes of the options object. | ||||
testgroups : list of str | ||||
Run only these sections of the test suite. If empty, run all the available | ||||
sections. | ||||
fast : int or None | ||||
Run the test suite in parallel, using n simultaneous processes. If None | ||||
is passed, one process is used per CPU core. Default 1 (i.e. sequential) | ||||
inc_slow : bool | ||||
|
r12607 | Include slow tests, like IPython.parallel. By default, these tests aren't | ||
run. | ||||
|
r12623 | xunit : bool | ||
Produce Xunit XML output. This is written to multiple foo.xunit.xml files. | ||||
coverage : bool or str | ||||
Measure code coverage from tests. True will store the raw coverage data, | ||||
or pass 'html' or 'xml' to get reports. | ||||
|
r12638 | |||
extra_args : list | ||||
Extra arguments to pass to the test subprocesses, e.g. '-v' | ||||
|
r12607 | """ | ||
|
r12623 | if options.fast != 1: | ||
# If running in parallel, capture output so it doesn't get interleaved | ||||
|
r12617 | TestController.buffer_output = True | ||
|
r12607 | |||
|
r13264 | to_run, not_run = prepare_controllers(options) | ||
|
r12607 | |||
|
r12616 | def justify(ltext, rtext, width=70, fill='-'): | ||
ltext += ' ' | ||||
rtext = (' ' + rtext).rjust(width - len(ltext), fill) | ||||
return ltext + rtext | ||||
|
r12607 | # Run all test runners, tracking execution time | ||
failed = [] | ||||
t_start = time.time() | ||||
|
r12623 | print() | ||
if options.fast == 1: | ||||
# This actually means sequential, i.e. with 1 job | ||||
|
r12616 | for controller in to_run: | ||
|
r12610 | print('IPython test group:', controller.section) | ||
|
r12783 | sys.stdout.flush() # Show in correct order when output is piped | ||
|
r12610 | controller, res = do_run(controller) | ||
if res: | ||||
failed.append(controller) | ||||
if res == -signal.SIGINT: | ||||
print("Interrupted") | ||||
break | ||||
print() | ||||
else: | ||||
|
r12623 | # Run tests concurrently | ||
|
r12610 | try: | ||
|
r12623 | pool = multiprocessing.pool.ThreadPool(options.fast) | ||
|
r12616 | for (controller, res) in pool.imap_unordered(do_run, to_run): | ||
res_string = 'OK' if res == 0 else 'FAILED' | ||||
print(justify('IPython test group: ' + controller.section, res_string)) | ||||
|
r12610 | if res: | ||
|
r12619 | print(bytes_to_str(controller.stdout)) | ||
|
r12610 | failed.append(controller) | ||
if res == -signal.SIGINT: | ||||
print("Interrupted") | ||||
break | ||||
except KeyboardInterrupt: | ||||
return | ||||
|
r13279 | |||
|
r12616 | for controller in not_run: | ||
print(justify('IPython test group: ' + controller.section, 'NOT RUN')) | ||||
|
r12607 | t_end = time.time() | ||
t_tests = t_end - t_start | ||||
|
r12616 | nrunners = len(to_run) | ||
|
r12607 | nfail = len(failed) | ||
# summarize results | ||||
|
r12623 | print('_'*70) | ||
|
r12607 | print('Test suite completed for system with the following information:') | ||
print(report()) | ||||
|
r12971 | took = "Took %.3fs." % t_tests | ||
|
r12623 | print('Status: ', end='') | ||
|
r12607 | if not failed: | ||
|
r12972 | print('OK (%d test groups).' % nrunners, took) | ||
|
r12607 | else: | ||
# If anything went wrong, point out what command to rerun manually to | ||||
# see the actual errors and individual summary | ||||
|
r12623 | failed_sections = [c.section for c in failed] | ||
print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, | ||||
|
r12971 | nrunners, ', '.join(failed_sections)), took) | ||
|
r12623 | print() | ||
print('You may wish to rerun these, with:') | ||||
print(' iptest', *failed_sections) | ||||
print() | ||||
if options.coverage: | ||||
|
r12612 | from coverage import coverage | ||
cov = coverage(data_file='.coverage') | ||||
cov.combine() | ||||
cov.save() | ||||
# Coverage HTML report | ||||
|
r12623 | if options.coverage == 'html': | ||
|
r12612 | html_dir = 'ipy_htmlcov' | ||
shutil.rmtree(html_dir, ignore_errors=True) | ||||
print("Writing HTML coverage report to %s/ ... " % html_dir, end="") | ||||
sys.stdout.flush() | ||||
# Custom HTML reporter to clean up module names. | ||||
from coverage.html import HtmlReporter | ||||
class CustomHtmlReporter(HtmlReporter): | ||||
def find_code_units(self, morfs): | ||||
super(CustomHtmlReporter, self).find_code_units(morfs) | ||||
for cu in self.code_units: | ||||
nameparts = cu.name.split(os.sep) | ||||
if 'IPython' not in nameparts: | ||||
continue | ||||
ix = nameparts.index('IPython') | ||||
cu.name = '.'.join(nameparts[ix:]) | ||||
# Reimplement the html_report method with our custom reporter | ||||
cov._harvest_data() | ||||
cov.config.from_args(omit='*%stests' % os.sep, html_dir=html_dir, | ||||
html_title='IPython test coverage', | ||||
) | ||||
reporter = CustomHtmlReporter(cov, cov.config) | ||||
reporter.report(None) | ||||
print('done.') | ||||
# Coverage XML report | ||||
|
r12623 | elif options.coverage == 'xml': | ||
|
r12612 | cov.xml_report(outfile='ipy_coverage.xml') | ||
if failed: | ||||
|
r12607 | # Ensure that our exit code indicates failure | ||
sys.exit(1) | ||||
|
r13740 | argparser = argparse.ArgumentParser(description='Run IPython test suite') | ||
argparser.add_argument('testgroups', nargs='*', | ||||
help='Run specified groups of tests. If omitted, run ' | ||||
'all tests.') | ||||
argparser.add_argument('--all', action='store_true', | ||||
help='Include slow tests not run by default.') | ||||
argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, | ||||
help='Run test sections in parallel.') | ||||
argparser.add_argument('--xunit', action='store_true', | ||||
help='Produce Xunit XML results') | ||||
argparser.add_argument('--coverage', nargs='?', const=True, default=False, | ||||
help="Measure test coverage. Specify 'html' or " | ||||
"'xml' to get reports.") | ||||
def default_options(): | ||||
"""Get an argparse Namespace object with the default arguments, to pass to | ||||
:func:`run_iptestall`. | ||||
""" | ||||
options = argparser.parse_args([]) | ||||
options.extra_args = [] | ||||
return options | ||||
|
r12607 | |||
def main(): | ||||
|
r12638 | # Arguments after -- should be passed through to nose. Argparse treats | ||
# everything after -- as regular positional arguments, so we separate them | ||||
# first. | ||||
try: | ||||
ix = sys.argv.index('--') | ||||
except ValueError: | ||||
to_parse = sys.argv[1:] | ||||
extra_args = [] | ||||
else: | ||||
to_parse = sys.argv[1:ix] | ||||
extra_args = sys.argv[ix+1:] | ||||
|
r13740 | options = argparser.parse_args(to_parse) | ||
|
r12638 | options.extra_args = extra_args | ||
|
r12608 | |||
|
r12623 | run_iptestall(options) | ||
|
r12607 | |||
if __name__ == '__main__': | ||||
main() | ||||