Show More
@@ -1,532 +1,532 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """IPython Test Process Controller |
|
2 | """IPython Test Process Controller | |
3 |
|
3 | |||
4 | This module runs one or more subprocesses which will actually run the IPython |
|
4 | This module runs one or more subprocesses which will actually run the IPython | |
5 | test suite. |
|
5 | test suite. | |
6 |
|
6 | |||
7 | """ |
|
7 | """ | |
8 |
|
8 | |||
9 | # Copyright (c) IPython Development Team. |
|
9 | # Copyright (c) IPython Development Team. | |
10 | # Distributed under the terms of the Modified BSD License. |
|
10 | # Distributed under the terms of the Modified BSD License. | |
11 |
|
11 | |||
12 | from __future__ import print_function |
|
12 | from __future__ import print_function | |
13 |
|
13 | |||
14 | import argparse |
|
14 | import argparse | |
15 | import json |
|
15 | import json | |
16 | import multiprocessing.pool |
|
16 | import multiprocessing.pool | |
17 | import os |
|
17 | import os | |
18 | import stat |
|
18 | import stat | |
19 | import re |
|
19 | import re | |
20 | import requests |
|
20 | import requests | |
21 | import shutil |
|
21 | import shutil | |
22 | import signal |
|
22 | import signal | |
23 | import sys |
|
23 | import sys | |
24 | import subprocess |
|
24 | import subprocess | |
25 | import time |
|
25 | import time | |
26 |
|
26 | |||
27 | from .iptest import ( |
|
27 | from .iptest import ( | |
28 | have, test_group_names as py_test_group_names, test_sections, StreamCapturer, |
|
28 | have, test_group_names as py_test_group_names, test_sections, StreamCapturer, | |
29 | test_for, |
|
29 | test_for, | |
30 | ) |
|
30 | ) | |
31 | from IPython.utils.path import compress_user |
|
31 | from IPython.utils.path import compress_user | |
32 | from IPython.utils.py3compat import bytes_to_str |
|
32 | from IPython.utils.py3compat import bytes_to_str | |
33 | from IPython.utils.sysinfo import get_sys_info |
|
33 | from IPython.utils.sysinfo import get_sys_info | |
34 | from IPython.utils.tempdir import TemporaryDirectory |
|
34 | from IPython.utils.tempdir import TemporaryDirectory | |
35 | from IPython.utils.text import strip_ansi |
|
35 | from IPython.utils.text import strip_ansi | |
36 |
|
36 | |||
37 | try: |
|
37 | try: | |
38 | # Python >= 3.3 |
|
38 | # Python >= 3.3 | |
39 | from subprocess import TimeoutExpired |
|
39 | from subprocess import TimeoutExpired | |
40 | def popen_wait(p, timeout): |
|
40 | def popen_wait(p, timeout): | |
41 | return p.wait(timeout) |
|
41 | return p.wait(timeout) | |
42 | except ImportError: |
|
42 | except ImportError: | |
43 | class TimeoutExpired(Exception): |
|
43 | class TimeoutExpired(Exception): | |
44 | pass |
|
44 | pass | |
45 | def popen_wait(p, timeout): |
|
45 | def popen_wait(p, timeout): | |
46 | """backport of Popen.wait from Python 3""" |
|
46 | """backport of Popen.wait from Python 3""" | |
47 | for i in range(int(10 * timeout)): |
|
47 | for i in range(int(10 * timeout)): | |
48 | if p.poll() is not None: |
|
48 | if p.poll() is not None: | |
49 | return |
|
49 | return | |
50 | time.sleep(0.1) |
|
50 | time.sleep(0.1) | |
51 | if p.poll() is None: |
|
51 | if p.poll() is None: | |
52 | raise TimeoutExpired |
|
52 | raise TimeoutExpired | |
53 |
|
53 | |||
54 | NOTEBOOK_SHUTDOWN_TIMEOUT = 10 |
|
54 | NOTEBOOK_SHUTDOWN_TIMEOUT = 10 | |
55 |
|
55 | |||
56 | class TestController(object): |
|
56 | class TestController(object): | |
57 | """Run tests in a subprocess |
|
57 | """Run tests in a subprocess | |
58 | """ |
|
58 | """ | |
59 | #: str, IPython test suite to be executed. |
|
59 | #: str, IPython test suite to be executed. | |
60 | section = None |
|
60 | section = None | |
61 | #: list, command line arguments to be executed |
|
61 | #: list, command line arguments to be executed | |
62 | cmd = None |
|
62 | cmd = None | |
63 | #: dict, extra environment variables to set for the subprocess |
|
63 | #: dict, extra environment variables to set for the subprocess | |
64 | env = None |
|
64 | env = None | |
65 | #: list, TemporaryDirectory instances to clear up when the process finishes |
|
65 | #: list, TemporaryDirectory instances to clear up when the process finishes | |
66 | dirs = None |
|
66 | dirs = None | |
67 | #: subprocess.Popen instance |
|
67 | #: subprocess.Popen instance | |
68 | process = None |
|
68 | process = None | |
69 | #: str, process stdout+stderr |
|
69 | #: str, process stdout+stderr | |
70 | stdout = None |
|
70 | stdout = None | |
71 |
|
71 | |||
72 | def __init__(self): |
|
72 | def __init__(self): | |
73 | self.cmd = [] |
|
73 | self.cmd = [] | |
74 | self.env = {} |
|
74 | self.env = {} | |
75 | self.dirs = [] |
|
75 | self.dirs = [] | |
76 |
|
76 | |||
77 | def setup(self): |
|
77 | def setup(self): | |
78 | """Create temporary directories etc. |
|
78 | """Create temporary directories etc. | |
79 |
|
79 | |||
80 | This is only called when we know the test group will be run. Things |
|
80 | This is only called when we know the test group will be run. Things | |
81 | created here may be cleaned up by self.cleanup(). |
|
81 | created here may be cleaned up by self.cleanup(). | |
82 | """ |
|
82 | """ | |
83 | pass |
|
83 | pass | |
84 |
|
84 | |||
85 | def launch(self, buffer_output=False, capture_output=False): |
|
85 | def launch(self, buffer_output=False, capture_output=False): | |
86 | # print('*** ENV:', self.env) # dbg |
|
86 | # print('*** ENV:', self.env) # dbg | |
87 | # print('*** CMD:', self.cmd) # dbg |
|
87 | # print('*** CMD:', self.cmd) # dbg | |
88 | env = os.environ.copy() |
|
88 | env = os.environ.copy() | |
89 | env.update(self.env) |
|
89 | env.update(self.env) | |
90 | if buffer_output: |
|
90 | if buffer_output: | |
91 | capture_output = True |
|
91 | capture_output = True | |
92 | self.stdout_capturer = c = StreamCapturer(echo=not buffer_output) |
|
92 | self.stdout_capturer = c = StreamCapturer(echo=not buffer_output) | |
93 | c.start() |
|
93 | c.start() | |
94 | stdout = c.writefd if capture_output else None |
|
94 | stdout = c.writefd if capture_output else None | |
95 | stderr = subprocess.STDOUT if capture_output else None |
|
95 | stderr = subprocess.STDOUT if capture_output else None | |
96 | self.process = subprocess.Popen(self.cmd, stdout=stdout, |
|
96 | self.process = subprocess.Popen(self.cmd, stdout=stdout, | |
97 | stderr=stderr, env=env) |
|
97 | stderr=stderr, env=env) | |
98 |
|
98 | |||
99 | def wait(self): |
|
99 | def wait(self): | |
100 | self.process.wait() |
|
100 | self.process.wait() | |
101 | self.stdout_capturer.halt() |
|
101 | self.stdout_capturer.halt() | |
102 | self.stdout = self.stdout_capturer.get_buffer() |
|
102 | self.stdout = self.stdout_capturer.get_buffer() | |
103 | return self.process.returncode |
|
103 | return self.process.returncode | |
104 |
|
104 | |||
105 | def print_extra_info(self): |
|
105 | def print_extra_info(self): | |
106 | """Print extra information about this test run. |
|
106 | """Print extra information about this test run. | |
107 |
|
107 | |||
108 | If we're running in parallel and showing the concise view, this is only |
|
108 | If we're running in parallel and showing the concise view, this is only | |
109 | called if the test group fails. Otherwise, it's called before the test |
|
109 | called if the test group fails. Otherwise, it's called before the test | |
110 | group is started. |
|
110 | group is started. | |
111 |
|
111 | |||
112 | The base implementation does nothing, but it can be overridden by |
|
112 | The base implementation does nothing, but it can be overridden by | |
113 | subclasses. |
|
113 | subclasses. | |
114 | """ |
|
114 | """ | |
115 | return |
|
115 | return | |
116 |
|
116 | |||
117 | def cleanup_process(self): |
|
117 | def cleanup_process(self): | |
118 | """Cleanup on exit by killing any leftover processes.""" |
|
118 | """Cleanup on exit by killing any leftover processes.""" | |
119 | subp = self.process |
|
119 | subp = self.process | |
120 | if subp is None or (subp.poll() is not None): |
|
120 | if subp is None or (subp.poll() is not None): | |
121 | return # Process doesn't exist, or is already dead. |
|
121 | return # Process doesn't exist, or is already dead. | |
122 |
|
122 | |||
123 | try: |
|
123 | try: | |
124 | print('Cleaning up stale PID: %d' % subp.pid) |
|
124 | print('Cleaning up stale PID: %d' % subp.pid) | |
125 | subp.kill() |
|
125 | subp.kill() | |
126 | except: # (OSError, WindowsError) ? |
|
126 | except: # (OSError, WindowsError) ? | |
127 | # This is just a best effort, if we fail or the process was |
|
127 | # This is just a best effort, if we fail or the process was | |
128 | # really gone, ignore it. |
|
128 | # really gone, ignore it. | |
129 | pass |
|
129 | pass | |
130 | else: |
|
130 | else: | |
131 | for i in range(10): |
|
131 | for i in range(10): | |
132 | if subp.poll() is None: |
|
132 | if subp.poll() is None: | |
133 | time.sleep(0.1) |
|
133 | time.sleep(0.1) | |
134 | else: |
|
134 | else: | |
135 | break |
|
135 | break | |
136 |
|
136 | |||
137 | if subp.poll() is None: |
|
137 | if subp.poll() is None: | |
138 | # The process did not die... |
|
138 | # The process did not die... | |
139 | print('... failed. Manual cleanup may be required.') |
|
139 | print('... failed. Manual cleanup may be required.') | |
140 |
|
140 | |||
141 | def cleanup(self): |
|
141 | def cleanup(self): | |
142 | "Kill process if it's still alive, and clean up temporary directories" |
|
142 | "Kill process if it's still alive, and clean up temporary directories" | |
143 | self.cleanup_process() |
|
143 | self.cleanup_process() | |
144 | for td in self.dirs: |
|
144 | for td in self.dirs: | |
145 | td.cleanup() |
|
145 | td.cleanup() | |
146 |
|
146 | |||
147 | __del__ = cleanup |
|
147 | __del__ = cleanup | |
148 |
|
148 | |||
149 |
|
149 | |||
150 | class PyTestController(TestController): |
|
150 | class PyTestController(TestController): | |
151 | """Run Python tests using IPython.testing.iptest""" |
|
151 | """Run Python tests using IPython.testing.iptest""" | |
152 | #: str, Python command to execute in subprocess |
|
152 | #: str, Python command to execute in subprocess | |
153 | pycmd = None |
|
153 | pycmd = None | |
154 |
|
154 | |||
155 | def __init__(self, section, options): |
|
155 | def __init__(self, section, options): | |
156 | """Create new test runner.""" |
|
156 | """Create new test runner.""" | |
157 | TestController.__init__(self) |
|
157 | TestController.__init__(self) | |
158 | self.section = section |
|
158 | self.section = section | |
159 | # pycmd is put into cmd[2] in PyTestController.launch() |
|
159 | # pycmd is put into cmd[2] in PyTestController.launch() | |
160 | self.cmd = [sys.executable, '-c', None, section] |
|
160 | self.cmd = [sys.executable, '-c', None, section] | |
161 | self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" |
|
161 | self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" | |
162 | self.options = options |
|
162 | self.options = options | |
163 |
|
163 | |||
164 | def setup(self): |
|
164 | def setup(self): | |
165 | ipydir = TemporaryDirectory() |
|
165 | ipydir = TemporaryDirectory() | |
166 | self.dirs.append(ipydir) |
|
166 | self.dirs.append(ipydir) | |
167 | self.env['IPYTHONDIR'] = ipydir.name |
|
167 | self.env['IPYTHONDIR'] = ipydir.name | |
168 | self.workingdir = workingdir = TemporaryDirectory() |
|
168 | self.workingdir = workingdir = TemporaryDirectory() | |
169 | self.dirs.append(workingdir) |
|
169 | self.dirs.append(workingdir) | |
170 | self.env['IPTEST_WORKING_DIR'] = workingdir.name |
|
170 | self.env['IPTEST_WORKING_DIR'] = workingdir.name | |
171 | # This means we won't get odd effects from our own matplotlib config |
|
171 | # This means we won't get odd effects from our own matplotlib config | |
172 | self.env['MPLCONFIGDIR'] = workingdir.name |
|
172 | self.env['MPLCONFIGDIR'] = workingdir.name | |
173 | # For security reasons (http://bugs.python.org/issue16202), use |
|
173 | # For security reasons (http://bugs.python.org/issue16202), use | |
174 | # a temporary directory to which other users have no access. |
|
174 | # a temporary directory to which other users have no access. | |
175 | self.env['TMPDIR'] = workingdir.name |
|
175 | self.env['TMPDIR'] = workingdir.name | |
176 |
|
176 | |||
177 | # Add a non-accessible directory to PATH (see gh-7053) |
|
177 | # Add a non-accessible directory to PATH (see gh-7053) | |
178 | noaccess = os.path.join(self.workingdir.name, "_no_access_") |
|
178 | noaccess = os.path.join(self.workingdir.name, "_no_access_") | |
179 | self.noaccess = noaccess |
|
179 | self.noaccess = noaccess | |
180 | os.mkdir(noaccess, 0) |
|
180 | os.mkdir(noaccess, 0) | |
181 |
|
181 | |||
182 | PATH = os.environ.get('PATH', '') |
|
182 | PATH = os.environ.get('PATH', '') | |
183 | if PATH: |
|
183 | if PATH: | |
184 | PATH = noaccess + os.pathsep + PATH |
|
184 | PATH = noaccess + os.pathsep + PATH | |
185 | else: |
|
185 | else: | |
186 | PATH = noaccess |
|
186 | PATH = noaccess | |
187 | self.env['PATH'] = PATH |
|
187 | self.env['PATH'] = PATH | |
188 |
|
188 | |||
189 | # From options: |
|
189 | # From options: | |
190 | if self.options.xunit: |
|
190 | if self.options.xunit: | |
191 | self.add_xunit() |
|
191 | self.add_xunit() | |
192 | if self.options.coverage: |
|
192 | if self.options.coverage: | |
193 | self.add_coverage() |
|
193 | self.add_coverage() | |
194 | self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams |
|
194 | self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams | |
195 | self.cmd.extend(self.options.extra_args) |
|
195 | self.cmd.extend(self.options.extra_args) | |
196 |
|
196 | |||
197 | def cleanup(self): |
|
197 | def cleanup(self): | |
198 | """ |
|
198 | """ | |
199 | Make the non-accessible directory created in setup() accessible |
|
199 | Make the non-accessible directory created in setup() accessible | |
200 | again, otherwise deleting the workingdir will fail. |
|
200 | again, otherwise deleting the workingdir will fail. | |
201 | """ |
|
201 | """ | |
202 | os.chmod(self.noaccess, stat.S_IRWXU) |
|
202 | os.chmod(self.noaccess, stat.S_IRWXU) | |
203 | TestController.cleanup(self) |
|
203 | TestController.cleanup(self) | |
204 |
|
204 | |||
205 | @property |
|
205 | @property | |
206 | def will_run(self): |
|
206 | def will_run(self): | |
207 | try: |
|
207 | try: | |
208 | return test_sections[self.section].will_run |
|
208 | return test_sections[self.section].will_run | |
209 | except KeyError: |
|
209 | except KeyError: | |
210 | return True |
|
210 | return True | |
211 |
|
211 | |||
212 | def add_xunit(self): |
|
212 | def add_xunit(self): | |
213 | xunit_file = os.path.abspath(self.section + '.xunit.xml') |
|
213 | xunit_file = os.path.abspath(self.section + '.xunit.xml') | |
214 | self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) |
|
214 | self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) | |
215 |
|
215 | |||
216 | def add_coverage(self): |
|
216 | def add_coverage(self): | |
217 | try: |
|
217 | try: | |
218 | sources = test_sections[self.section].includes |
|
218 | sources = test_sections[self.section].includes | |
219 | except KeyError: |
|
219 | except KeyError: | |
220 | sources = ['IPython'] |
|
220 | sources = ['IPython'] | |
221 |
|
221 | |||
222 | coverage_rc = ("[run]\n" |
|
222 | coverage_rc = ("[run]\n" | |
223 | "data_file = {data_file}\n" |
|
223 | "data_file = {data_file}\n" | |
224 | "source =\n" |
|
224 | "source =\n" | |
225 | " {source}\n" |
|
225 | " {source}\n" | |
226 | ).format(data_file=os.path.abspath('.coverage.'+self.section), |
|
226 | ).format(data_file=os.path.abspath('.coverage.'+self.section), | |
227 | source="\n ".join(sources)) |
|
227 | source="\n ".join(sources)) | |
228 | config_file = os.path.join(self.workingdir.name, '.coveragerc') |
|
228 | config_file = os.path.join(self.workingdir.name, '.coveragerc') | |
229 | with open(config_file, 'w') as f: |
|
229 | with open(config_file, 'w') as f: | |
230 | f.write(coverage_rc) |
|
230 | f.write(coverage_rc) | |
231 |
|
231 | |||
232 | self.env['COVERAGE_PROCESS_START'] = config_file |
|
232 | self.env['COVERAGE_PROCESS_START'] = config_file | |
233 | self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd |
|
233 | self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd | |
234 |
|
234 | |||
235 | def launch(self, buffer_output=False): |
|
235 | def launch(self, buffer_output=False): | |
236 | self.cmd[2] = self.pycmd |
|
236 | self.cmd[2] = self.pycmd | |
237 | super(PyTestController, self).launch(buffer_output=buffer_output) |
|
237 | super(PyTestController, self).launch(buffer_output=buffer_output) | |
238 |
|
238 | |||
239 |
|
239 | |||
240 | def prepare_controllers(options): |
|
240 | def prepare_controllers(options): | |
241 | """Returns two lists of TestController instances, those to run, and those |
|
241 | """Returns two lists of TestController instances, those to run, and those | |
242 | not to run.""" |
|
242 | not to run.""" | |
243 | testgroups = options.testgroups |
|
243 | testgroups = options.testgroups | |
244 | if not testgroups: |
|
244 | if not testgroups: | |
245 | testgroups = py_test_group_names |
|
245 | testgroups = py_test_group_names | |
246 |
|
246 | |||
247 | controllers = [PyTestController(name, options) for name in testgroups] |
|
247 | controllers = [PyTestController(name, options) for name in testgroups] | |
248 |
|
248 | |||
249 | to_run = [c for c in controllers if c.will_run] |
|
249 | to_run = [c for c in controllers if c.will_run] | |
250 | not_run = [c for c in controllers if not c.will_run] |
|
250 | not_run = [c for c in controllers if not c.will_run] | |
251 | return to_run, not_run |
|
251 | return to_run, not_run | |
252 |
|
252 | |||
253 | def do_run(controller, buffer_output=True): |
|
253 | def do_run(controller, buffer_output=True): | |
254 | """Setup and run a test controller. |
|
254 | """Setup and run a test controller. | |
255 |
|
255 | |||
256 | If buffer_output is True, no output is displayed, to avoid it appearing |
|
256 | If buffer_output is True, no output is displayed, to avoid it appearing | |
257 | interleaved. In this case, the caller is responsible for displaying test |
|
257 | interleaved. In this case, the caller is responsible for displaying test | |
258 | output on failure. |
|
258 | output on failure. | |
259 |
|
259 | |||
260 | Returns |
|
260 | Returns | |
261 | ------- |
|
261 | ------- | |
262 | controller : TestController |
|
262 | controller : TestController | |
263 | The same controller as passed in, as a convenience for using map() type |
|
263 | The same controller as passed in, as a convenience for using map() type | |
264 | APIs. |
|
264 | APIs. | |
265 | exitcode : int |
|
265 | exitcode : int | |
266 | The exit code of the test subprocess. Non-zero indicates failure. |
|
266 | The exit code of the test subprocess. Non-zero indicates failure. | |
267 | """ |
|
267 | """ | |
268 | try: |
|
268 | try: | |
269 | try: |
|
269 | try: | |
270 | controller.setup() |
|
270 | controller.setup() | |
271 | if not buffer_output: |
|
271 | if not buffer_output: | |
272 | controller.print_extra_info() |
|
272 | controller.print_extra_info() | |
273 | controller.launch(buffer_output=buffer_output) |
|
273 | controller.launch(buffer_output=buffer_output) | |
274 | except Exception: |
|
274 | except Exception: | |
275 | import traceback |
|
275 | import traceback | |
276 | traceback.print_exc() |
|
276 | traceback.print_exc() | |
277 | return controller, 1 # signal failure |
|
277 | return controller, 1 # signal failure | |
278 |
|
278 | |||
279 | exitcode = controller.wait() |
|
279 | exitcode = controller.wait() | |
280 | return controller, exitcode |
|
280 | return controller, exitcode | |
281 |
|
281 | |||
282 | except KeyboardInterrupt: |
|
282 | except KeyboardInterrupt: | |
283 | return controller, -signal.SIGINT |
|
283 | return controller, -signal.SIGINT | |
284 | finally: |
|
284 | finally: | |
285 | controller.cleanup() |
|
285 | controller.cleanup() | |
286 |
|
286 | |||
287 | def report(): |
|
287 | def report(): | |
288 | """Return a string with a summary report of test-related variables.""" |
|
288 | """Return a string with a summary report of test-related variables.""" | |
289 | inf = get_sys_info() |
|
289 | inf = get_sys_info() | |
290 | out = [] |
|
290 | out = [] | |
291 | def _add(name, value): |
|
291 | def _add(name, value): | |
292 | out.append((name, value)) |
|
292 | out.append((name, value)) | |
293 |
|
293 | |||
294 | _add('IPython version', inf['ipython_version']) |
|
294 | _add('IPython version', inf['ipython_version']) | |
295 | _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source'])) |
|
295 | _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source'])) | |
296 | _add('IPython package', compress_user(inf['ipython_path'])) |
|
296 | _add('IPython package', compress_user(inf['ipython_path'])) | |
297 | _add('Python version', inf['sys_version'].replace('\n','')) |
|
297 | _add('Python version', inf['sys_version'].replace('\n','')) | |
298 | _add('sys.executable', compress_user(inf['sys_executable'])) |
|
298 | _add('sys.executable', compress_user(inf['sys_executable'])) | |
299 | _add('Platform', inf['platform']) |
|
299 | _add('Platform', inf['platform']) | |
300 |
|
300 | |||
301 | width = max(len(n) for (n,v) in out) |
|
301 | width = max(len(n) for (n,v) in out) | |
302 | out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] |
|
302 | out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] | |
303 |
|
303 | |||
304 | avail = [] |
|
304 | avail = [] | |
305 | not_avail = [] |
|
305 | not_avail = [] | |
306 |
|
306 | |||
307 | for k, is_avail in have.items(): |
|
307 | for k, is_avail in have.items(): | |
308 | if is_avail: |
|
308 | if is_avail: | |
309 | avail.append(k) |
|
309 | avail.append(k) | |
310 | else: |
|
310 | else: | |
311 | not_avail.append(k) |
|
311 | not_avail.append(k) | |
312 |
|
312 | |||
313 | if avail: |
|
313 | if avail: | |
314 | out.append('\nTools and libraries available at test time:\n') |
|
314 | out.append('\nTools and libraries available at test time:\n') | |
315 | avail.sort() |
|
315 | avail.sort() | |
316 | out.append(' ' + ' '.join(avail)+'\n') |
|
316 | out.append(' ' + ' '.join(avail)+'\n') | |
317 |
|
317 | |||
318 | if not_avail: |
|
318 | if not_avail: | |
319 | out.append('\nTools and libraries NOT available at test time:\n') |
|
319 | out.append('\nTools and libraries NOT available at test time:\n') | |
320 | not_avail.sort() |
|
320 | not_avail.sort() | |
321 | out.append(' ' + ' '.join(not_avail)+'\n') |
|
321 | out.append(' ' + ' '.join(not_avail)+'\n') | |
322 |
|
322 | |||
323 | return ''.join(out) |
|
323 | return ''.join(out) | |
324 |
|
324 | |||
325 | def run_iptestall(options): |
|
325 | def run_iptestall(options): | |
326 | """Run the entire IPython test suite by calling nose and trial. |
|
326 | """Run the entire IPython test suite by calling nose and trial. | |
327 |
|
327 | |||
328 | This function constructs :class:`IPTester` instances for all IPython |
|
328 | This function constructs :class:`IPTester` instances for all IPython | |
329 | modules and package and then runs each of them. This causes the modules |
|
329 | modules and package and then runs each of them. This causes the modules | |
330 | and packages of IPython to be tested each in their own subprocess using |
|
330 | and packages of IPython to be tested each in their own subprocess using | |
331 | nose. |
|
331 | nose. | |
332 |
|
332 | |||
333 | Parameters |
|
333 | Parameters | |
334 | ---------- |
|
334 | ---------- | |
335 |
|
335 | |||
336 | All parameters are passed as attributes of the options object. |
|
336 | All parameters are passed as attributes of the options object. | |
337 |
|
337 | |||
338 | testgroups : list of str |
|
338 | testgroups : list of str | |
339 | Run only these sections of the test suite. If empty, run all the available |
|
339 | Run only these sections of the test suite. If empty, run all the available | |
340 | sections. |
|
340 | sections. | |
341 |
|
341 | |||
342 | fast : int or None |
|
342 | fast : int or None | |
343 | Run the test suite in parallel, using n simultaneous processes. If None |
|
343 | Run the test suite in parallel, using n simultaneous processes. If None | |
344 | is passed, one process is used per CPU core. Default 1 (i.e. sequential) |
|
344 | is passed, one process is used per CPU core. Default 1 (i.e. sequential) | |
345 |
|
345 | |||
346 | inc_slow : bool |
|
346 | inc_slow : bool | |
347 | Include slow tests. By default, these tests aren't run. |
|
347 | Include slow tests. By default, these tests aren't run. | |
348 |
|
348 | |||
349 | url : unicode |
|
349 | url : unicode | |
350 | Address:port to use when running the JS tests. |
|
350 | Address:port to use when running the JS tests. | |
351 |
|
351 | |||
352 | xunit : bool |
|
352 | xunit : bool | |
353 | Produce Xunit XML output. This is written to multiple foo.xunit.xml files. |
|
353 | Produce Xunit XML output. This is written to multiple foo.xunit.xml files. | |
354 |
|
354 | |||
355 | coverage : bool or str |
|
355 | coverage : bool or str | |
356 | Measure code coverage from tests. True will store the raw coverage data, |
|
356 | Measure code coverage from tests. True will store the raw coverage data, | |
357 | or pass 'html' or 'xml' to get reports. |
|
357 | or pass 'html' or 'xml' to get reports. | |
358 |
|
358 | |||
359 | extra_args : list |
|
359 | extra_args : list | |
360 | Extra arguments to pass to the test subprocesses, e.g. '-v' |
|
360 | Extra arguments to pass to the test subprocesses, e.g. '-v' | |
361 | """ |
|
361 | """ | |
362 | to_run, not_run = prepare_controllers(options) |
|
362 | to_run, not_run = prepare_controllers(options) | |
363 |
|
363 | |||
364 | def justify(ltext, rtext, width=70, fill='-'): |
|
364 | def justify(ltext, rtext, width=70, fill='-'): | |
365 | ltext += ' ' |
|
365 | ltext += ' ' | |
366 | rtext = (' ' + rtext).rjust(width - len(ltext), fill) |
|
366 | rtext = (' ' + rtext).rjust(width - len(ltext), fill) | |
367 | return ltext + rtext |
|
367 | return ltext + rtext | |
368 |
|
368 | |||
369 | # Run all test runners, tracking execution time |
|
369 | # Run all test runners, tracking execution time | |
370 | failed = [] |
|
370 | failed = [] | |
371 | t_start = time.time() |
|
371 | t_start = time.time() | |
372 |
|
372 | |||
373 | print() |
|
373 | print() | |
374 | if options.fast == 1: |
|
374 | if options.fast == 1: | |
375 | # This actually means sequential, i.e. with 1 job |
|
375 | # This actually means sequential, i.e. with 1 job | |
376 | for controller in to_run: |
|
376 | for controller in to_run: | |
377 | print('Test group:', controller.section) |
|
377 | print('Test group:', controller.section) | |
378 | sys.stdout.flush() # Show in correct order when output is piped |
|
378 | sys.stdout.flush() # Show in correct order when output is piped | |
379 | controller, res = do_run(controller, buffer_output=False) |
|
379 | controller, res = do_run(controller, buffer_output=False) | |
380 | if res: |
|
380 | if res: | |
381 | failed.append(controller) |
|
381 | failed.append(controller) | |
382 | if res == -signal.SIGINT: |
|
382 | if res == -signal.SIGINT: | |
383 | print("Interrupted") |
|
383 | print("Interrupted") | |
384 | break |
|
384 | break | |
385 | print() |
|
385 | print() | |
386 |
|
386 | |||
387 | else: |
|
387 | else: | |
388 | # Run tests concurrently |
|
388 | # Run tests concurrently | |
389 | try: |
|
389 | try: | |
390 | pool = multiprocessing.pool.ThreadPool(options.fast) |
|
390 | pool = multiprocessing.pool.ThreadPool(options.fast) | |
391 | for (controller, res) in pool.imap_unordered(do_run, to_run): |
|
391 | for (controller, res) in pool.imap_unordered(do_run, to_run): | |
392 | res_string = 'OK' if res == 0 else 'FAILED' |
|
392 | res_string = 'OK' if res == 0 else 'FAILED' | |
393 | print(justify('Test group: ' + controller.section, res_string)) |
|
393 | print(justify('Test group: ' + controller.section, res_string)) | |
394 | if res: |
|
394 | if res: | |
395 | controller.print_extra_info() |
|
395 | controller.print_extra_info() | |
396 | print(bytes_to_str(controller.stdout)) |
|
396 | print(bytes_to_str(controller.stdout)) | |
397 | failed.append(controller) |
|
397 | failed.append(controller) | |
398 | if res == -signal.SIGINT: |
|
398 | if res == -signal.SIGINT: | |
399 | print("Interrupted") |
|
399 | print("Interrupted") | |
400 | break |
|
400 | break | |
401 | except KeyboardInterrupt: |
|
401 | except KeyboardInterrupt: | |
402 | return |
|
402 | return | |
403 |
|
403 | |||
404 | for controller in not_run: |
|
404 | for controller in not_run: | |
405 | print(justify('Test group: ' + controller.section, 'NOT RUN')) |
|
405 | print(justify('Test group: ' + controller.section, 'NOT RUN')) | |
406 |
|
406 | |||
407 | t_end = time.time() |
|
407 | t_end = time.time() | |
408 | t_tests = t_end - t_start |
|
408 | t_tests = t_end - t_start | |
409 | nrunners = len(to_run) |
|
409 | nrunners = len(to_run) | |
410 | nfail = len(failed) |
|
410 | nfail = len(failed) | |
411 | # summarize results |
|
411 | # summarize results | |
412 | print('_'*70) |
|
412 | print('_'*70) | |
413 | print('Test suite completed for system with the following information:') |
|
413 | print('Test suite completed for system with the following information:') | |
414 | print(report()) |
|
414 | print(report()) | |
415 | took = "Took %.3fs." % t_tests |
|
415 | took = "Took %.3fs." % t_tests | |
416 | print('Status: ', end='') |
|
416 | print('Status: ', end='') | |
417 | if not failed: |
|
417 | if not failed: | |
418 | print('OK (%d test groups).' % nrunners, took) |
|
418 | print('OK (%d test groups).' % nrunners, took) | |
419 | else: |
|
419 | else: | |
420 | # If anything went wrong, point out what command to rerun manually to |
|
420 | # If anything went wrong, point out what command to rerun manually to | |
421 | # see the actual errors and individual summary |
|
421 | # see the actual errors and individual summary | |
422 | failed_sections = [c.section for c in failed] |
|
422 | failed_sections = [c.section for c in failed] | |
423 | print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, |
|
423 | print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, | |
424 | nrunners, ', '.join(failed_sections)), took) |
|
424 | nrunners, ', '.join(failed_sections)), took) | |
425 | print() |
|
425 | print() | |
426 | print('You may wish to rerun these, with:') |
|
426 | print('You may wish to rerun these, with:') | |
427 | print(' iptest', *failed_sections) |
|
427 | print(' iptest', *failed_sections) | |
428 | print() |
|
428 | print() | |
429 |
|
429 | |||
430 | if options.coverage: |
|
430 | if options.coverage: | |
431 | from coverage import coverage, CoverageException |
|
431 | from coverage import coverage, CoverageException | |
432 | cov = coverage(data_file='.coverage') |
|
432 | cov = coverage(data_file='.coverage') | |
433 | cov.combine() |
|
433 | cov.combine() | |
434 | cov.save() |
|
434 | cov.save() | |
435 |
|
435 | |||
436 | # Coverage HTML report |
|
436 | # Coverage HTML report | |
437 | if options.coverage == 'html': |
|
437 | if options.coverage == 'html': | |
438 | html_dir = 'ipy_htmlcov' |
|
438 | html_dir = 'ipy_htmlcov' | |
439 | shutil.rmtree(html_dir, ignore_errors=True) |
|
439 | shutil.rmtree(html_dir, ignore_errors=True) | |
440 | print("Writing HTML coverage report to %s/ ... " % html_dir, end="") |
|
440 | print("Writing HTML coverage report to %s/ ... " % html_dir, end="") | |
441 | sys.stdout.flush() |
|
441 | sys.stdout.flush() | |
442 |
|
442 | |||
443 | # Custom HTML reporter to clean up module names. |
|
443 | # Custom HTML reporter to clean up module names. | |
444 | from coverage.html import HtmlReporter |
|
444 | from coverage.html import HtmlReporter | |
445 | class CustomHtmlReporter(HtmlReporter): |
|
445 | class CustomHtmlReporter(HtmlReporter): | |
446 | def find_code_units(self, morfs): |
|
446 | def find_code_units(self, morfs): | |
447 | super(CustomHtmlReporter, self).find_code_units(morfs) |
|
447 | super(CustomHtmlReporter, self).find_code_units(morfs) | |
448 | for cu in self.code_units: |
|
448 | for cu in self.code_units: | |
449 | nameparts = cu.name.split(os.sep) |
|
449 | nameparts = cu.name.split(os.sep) | |
450 | if 'IPython' not in nameparts: |
|
450 | if 'IPython' not in nameparts: | |
451 | continue |
|
451 | continue | |
452 | ix = nameparts.index('IPython') |
|
452 | ix = nameparts.index('IPython') | |
453 | cu.name = '.'.join(nameparts[ix:]) |
|
453 | cu.name = '.'.join(nameparts[ix:]) | |
454 |
|
454 | |||
455 | # Reimplement the html_report method with our custom reporter |
|
455 | # Reimplement the html_report method with our custom reporter | |
456 |
cov. |
|
456 | cov.get_data() | |
457 | cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir, |
|
457 | cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir, | |
458 | html_title='IPython test coverage', |
|
458 | html_title='IPython test coverage', | |
459 | ) |
|
459 | ) | |
460 | reporter = CustomHtmlReporter(cov, cov.config) |
|
460 | reporter = CustomHtmlReporter(cov, cov.config) | |
461 | reporter.report(None) |
|
461 | reporter.report(None) | |
462 | print('done.') |
|
462 | print('done.') | |
463 |
|
463 | |||
464 | # Coverage XML report |
|
464 | # Coverage XML report | |
465 | elif options.coverage == 'xml': |
|
465 | elif options.coverage == 'xml': | |
466 | try: |
|
466 | try: | |
467 | cov.xml_report(outfile='ipy_coverage.xml') |
|
467 | cov.xml_report(outfile='ipy_coverage.xml') | |
468 | except CoverageException as e: |
|
468 | except CoverageException as e: | |
469 | print('Generating coverage report failed. Are you running javascript tests only?') |
|
469 | print('Generating coverage report failed. Are you running javascript tests only?') | |
470 | import traceback |
|
470 | import traceback | |
471 | traceback.print_exc() |
|
471 | traceback.print_exc() | |
472 |
|
472 | |||
473 | if failed: |
|
473 | if failed: | |
474 | # Ensure that our exit code indicates failure |
|
474 | # Ensure that our exit code indicates failure | |
475 | sys.exit(1) |
|
475 | sys.exit(1) | |
476 |
|
476 | |||
477 | argparser = argparse.ArgumentParser(description='Run IPython test suite') |
|
477 | argparser = argparse.ArgumentParser(description='Run IPython test suite') | |
478 | argparser.add_argument('testgroups', nargs='*', |
|
478 | argparser.add_argument('testgroups', nargs='*', | |
479 | help='Run specified groups of tests. If omitted, run ' |
|
479 | help='Run specified groups of tests. If omitted, run ' | |
480 | 'all tests.') |
|
480 | 'all tests.') | |
481 | argparser.add_argument('--all', action='store_true', |
|
481 | argparser.add_argument('--all', action='store_true', | |
482 | help='Include slow tests not run by default.') |
|
482 | help='Include slow tests not run by default.') | |
483 | argparser.add_argument('--url', help="URL to use for the JS tests.") |
|
483 | argparser.add_argument('--url', help="URL to use for the JS tests.") | |
484 | argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, |
|
484 | argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, | |
485 | help='Run test sections in parallel. This starts as many ' |
|
485 | help='Run test sections in parallel. This starts as many ' | |
486 | 'processes as you have cores, or you can specify a number.') |
|
486 | 'processes as you have cores, or you can specify a number.') | |
487 | argparser.add_argument('--xunit', action='store_true', |
|
487 | argparser.add_argument('--xunit', action='store_true', | |
488 | help='Produce Xunit XML results') |
|
488 | help='Produce Xunit XML results') | |
489 | argparser.add_argument('--coverage', nargs='?', const=True, default=False, |
|
489 | argparser.add_argument('--coverage', nargs='?', const=True, default=False, | |
490 | help="Measure test coverage. Specify 'html' or " |
|
490 | help="Measure test coverage. Specify 'html' or " | |
491 | "'xml' to get reports.") |
|
491 | "'xml' to get reports.") | |
492 | argparser.add_argument('--subproc-streams', default='capture', |
|
492 | argparser.add_argument('--subproc-streams', default='capture', | |
493 | help="What to do with stdout/stderr from subprocesses. " |
|
493 | help="What to do with stdout/stderr from subprocesses. " | |
494 | "'capture' (default), 'show' and 'discard' are the options.") |
|
494 | "'capture' (default), 'show' and 'discard' are the options.") | |
495 |
|
495 | |||
496 | def default_options(): |
|
496 | def default_options(): | |
497 | """Get an argparse Namespace object with the default arguments, to pass to |
|
497 | """Get an argparse Namespace object with the default arguments, to pass to | |
498 | :func:`run_iptestall`. |
|
498 | :func:`run_iptestall`. | |
499 | """ |
|
499 | """ | |
500 | options = argparser.parse_args([]) |
|
500 | options = argparser.parse_args([]) | |
501 | options.extra_args = [] |
|
501 | options.extra_args = [] | |
502 | return options |
|
502 | return options | |
503 |
|
503 | |||
504 | def main(): |
|
504 | def main(): | |
505 | # iptest doesn't work correctly if the working directory is the |
|
505 | # iptest doesn't work correctly if the working directory is the | |
506 | # root of the IPython source tree. Tell the user to avoid |
|
506 | # root of the IPython source tree. Tell the user to avoid | |
507 | # frustration. |
|
507 | # frustration. | |
508 | if os.path.exists(os.path.join(os.getcwd(), |
|
508 | if os.path.exists(os.path.join(os.getcwd(), | |
509 | 'IPython', 'testing', '__main__.py')): |
|
509 | 'IPython', 'testing', '__main__.py')): | |
510 | print("Don't run iptest from the IPython source directory", |
|
510 | print("Don't run iptest from the IPython source directory", | |
511 | file=sys.stderr) |
|
511 | file=sys.stderr) | |
512 | sys.exit(1) |
|
512 | sys.exit(1) | |
513 | # Arguments after -- should be passed through to nose. Argparse treats |
|
513 | # Arguments after -- should be passed through to nose. Argparse treats | |
514 | # everything after -- as regular positional arguments, so we separate them |
|
514 | # everything after -- as regular positional arguments, so we separate them | |
515 | # first. |
|
515 | # first. | |
516 | try: |
|
516 | try: | |
517 | ix = sys.argv.index('--') |
|
517 | ix = sys.argv.index('--') | |
518 | except ValueError: |
|
518 | except ValueError: | |
519 | to_parse = sys.argv[1:] |
|
519 | to_parse = sys.argv[1:] | |
520 | extra_args = [] |
|
520 | extra_args = [] | |
521 | else: |
|
521 | else: | |
522 | to_parse = sys.argv[1:ix] |
|
522 | to_parse = sys.argv[1:ix] | |
523 | extra_args = sys.argv[ix+1:] |
|
523 | extra_args = sys.argv[ix+1:] | |
524 |
|
524 | |||
525 | options = argparser.parse_args(to_parse) |
|
525 | options = argparser.parse_args(to_parse) | |
526 | options.extra_args = extra_args |
|
526 | options.extra_args = extra_args | |
527 |
|
527 | |||
528 | run_iptestall(options) |
|
528 | run_iptestall(options) | |
529 |
|
529 | |||
530 |
|
530 | |||
531 | if __name__ == '__main__': |
|
531 | if __name__ == '__main__': | |
532 | main() |
|
532 | main() |
General Comments 0
You need to be logged in to leave comments.
Login now