Show More
@@ -1,493 +1,496 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """IPython Test Process Controller |
|
2 | """IPython Test Process Controller | |
3 |
|
3 | |||
4 | This module runs one or more subprocesses which will actually run the IPython |
|
4 | This module runs one or more subprocesses which will actually run the IPython | |
5 | test suite. |
|
5 | test suite. | |
6 |
|
6 | |||
7 | """ |
|
7 | """ | |
8 |
|
8 | |||
9 | # Copyright (c) IPython Development Team. |
|
9 | # Copyright (c) IPython Development Team. | |
10 | # Distributed under the terms of the Modified BSD License. |
|
10 | # Distributed under the terms of the Modified BSD License. | |
11 |
|
11 | |||
12 |
|
12 | |||
13 | import argparse |
|
13 | import argparse | |
14 | import multiprocessing.pool |
|
14 | import multiprocessing.pool | |
15 | import os |
|
15 | import os | |
16 | import stat |
|
16 | import stat | |
17 | import shutil |
|
17 | import shutil | |
18 | import signal |
|
18 | import signal | |
19 | import sys |
|
19 | import sys | |
20 | import subprocess |
|
20 | import subprocess | |
21 | import time |
|
21 | import time | |
22 |
|
22 | |||
23 | from .iptest import ( |
|
23 | from .iptest import ( | |
24 | have, test_group_names as py_test_group_names, test_sections, StreamCapturer, |
|
24 | have, test_group_names as py_test_group_names, test_sections, StreamCapturer, | |
25 | ) |
|
25 | ) | |
26 | from IPython.utils.path import compress_user |
|
26 | from IPython.utils.path import compress_user | |
27 | from IPython.utils.py3compat import decode |
|
27 | from IPython.utils.py3compat import decode | |
28 | from IPython.utils.sysinfo import get_sys_info |
|
28 | from IPython.utils.sysinfo import get_sys_info | |
29 | from IPython.utils.tempdir import TemporaryDirectory |
|
29 | from IPython.utils.tempdir import TemporaryDirectory | |
30 | from pathlib import Path |
|
30 | from pathlib import Path | |
|
31 | from typing import Dict | |||
31 |
|
32 | |||
32 | class TestController: |
|
33 | class TestController: | |
33 | """Run tests in a subprocess |
|
34 | """Run tests in a subprocess | |
34 | """ |
|
35 | """ | |
35 | #: str, IPython test suite to be executed. |
|
36 | #: str, IPython test suite to be executed. | |
36 | section = None |
|
37 | section = None | |
37 | #: list, command line arguments to be executed |
|
38 | #: list, command line arguments to be executed | |
38 | cmd = None |
|
39 | cmd = None | |
39 | #: dict, extra environment variables to set for the subprocess |
|
40 | #: dict, extra environment variables to set for the subprocess | |
40 | env = None |
|
41 | env: Dict[str, str] = {} | |
41 | #: list, TemporaryDirectory instances to clear up when the process finishes |
|
42 | #: list, TemporaryDirectory instances to clear up when the process finishes | |
42 | dirs = None |
|
43 | dirs = None | |
43 | #: subprocess.Popen instance |
|
44 | #: subprocess.Popen instance | |
44 | process = None |
|
45 | process = None | |
45 | #: str, process stdout+stderr |
|
46 | #: str, process stdout+stderr | |
46 | stdout = None |
|
47 | stdout = None | |
47 |
|
48 | |||
48 | def __init__(self): |
|
49 | def __init__(self): | |
49 | self.cmd = [] |
|
50 | self.cmd = [] | |
50 | self.env = {} |
|
51 | self.env = {} | |
51 | self.dirs = [] |
|
52 | self.dirs = [] | |
52 |
|
53 | |||
53 | def setUp(self): |
|
54 | def setUp(self): | |
54 | """Create temporary directories etc. |
|
55 | """Create temporary directories etc. | |
55 |
|
56 | |||
56 | This is only called when we know the test group will be run. Things |
|
57 | This is only called when we know the test group will be run. Things | |
57 | created here may be cleaned up by self.cleanup(). |
|
58 | created here may be cleaned up by self.cleanup(). | |
58 | """ |
|
59 | """ | |
59 | pass |
|
60 | pass | |
60 |
|
61 | |||
61 | def launch(self, buffer_output=False, capture_output=False): |
|
62 | def launch(self, buffer_output=False, capture_output=False): | |
62 | # print('*** ENV:', self.env) # dbg |
|
63 | # print('*** ENV:', self.env) # dbg | |
63 | # print('*** CMD:', self.cmd) # dbg |
|
64 | # print('*** CMD:', self.cmd) # dbg | |
64 | env = os.environ.copy() |
|
65 | env = os.environ.copy() | |
65 | env.update(self.env) |
|
66 | env.update(self.env) | |
66 | if buffer_output: |
|
67 | if buffer_output: | |
67 | capture_output = True |
|
68 | capture_output = True | |
68 | self.stdout_capturer = c = StreamCapturer(echo=not buffer_output) |
|
69 | self.stdout_capturer = c = StreamCapturer(echo=not buffer_output) | |
69 | c.start() |
|
70 | c.start() | |
70 | stdout = c.writefd if capture_output else None |
|
71 | stdout = c.writefd if capture_output else None | |
71 | stderr = subprocess.STDOUT if capture_output else None |
|
72 | stderr = subprocess.STDOUT if capture_output else None | |
|
73 | for k, v in env.items(): | |||
|
74 | assert isinstance(v, str), f"env[{repr(k)}] is not a str: {v}" | |||
72 | self.process = subprocess.Popen(self.cmd, stdout=stdout, |
|
75 | self.process = subprocess.Popen(self.cmd, stdout=stdout, | |
73 | stderr=stderr, env=env) |
|
76 | stderr=stderr, env=env) | |
74 |
|
77 | |||
75 | def wait(self): |
|
78 | def wait(self): | |
76 | self.process.wait() |
|
79 | self.process.wait() | |
77 | self.stdout_capturer.halt() |
|
80 | self.stdout_capturer.halt() | |
78 | self.stdout = self.stdout_capturer.get_buffer() |
|
81 | self.stdout = self.stdout_capturer.get_buffer() | |
79 | return self.process.returncode |
|
82 | return self.process.returncode | |
80 |
|
83 | |||
81 | def cleanup_process(self): |
|
84 | def cleanup_process(self): | |
82 | """Cleanup on exit by killing any leftover processes.""" |
|
85 | """Cleanup on exit by killing any leftover processes.""" | |
83 | subp = self.process |
|
86 | subp = self.process | |
84 | if subp is None or (subp.poll() is not None): |
|
87 | if subp is None or (subp.poll() is not None): | |
85 | return # Process doesn't exist, or is already dead. |
|
88 | return # Process doesn't exist, or is already dead. | |
86 |
|
89 | |||
87 | try: |
|
90 | try: | |
88 | print('Cleaning up stale PID: %d' % subp.pid) |
|
91 | print('Cleaning up stale PID: %d' % subp.pid) | |
89 | subp.kill() |
|
92 | subp.kill() | |
90 | except: # (OSError, WindowsError) ? |
|
93 | except: # (OSError, WindowsError) ? | |
91 | # This is just a best effort, if we fail or the process was |
|
94 | # This is just a best effort, if we fail or the process was | |
92 | # really gone, ignore it. |
|
95 | # really gone, ignore it. | |
93 | pass |
|
96 | pass | |
94 | else: |
|
97 | else: | |
95 | for i in range(10): |
|
98 | for i in range(10): | |
96 | if subp.poll() is None: |
|
99 | if subp.poll() is None: | |
97 | time.sleep(0.1) |
|
100 | time.sleep(0.1) | |
98 | else: |
|
101 | else: | |
99 | break |
|
102 | break | |
100 |
|
103 | |||
101 | if subp.poll() is None: |
|
104 | if subp.poll() is None: | |
102 | # The process did not die... |
|
105 | # The process did not die... | |
103 | print('... failed. Manual cleanup may be required.') |
|
106 | print('... failed. Manual cleanup may be required.') | |
104 |
|
107 | |||
105 | def cleanup(self): |
|
108 | def cleanup(self): | |
106 | "Kill process if it's still alive, and clean up temporary directories" |
|
109 | "Kill process if it's still alive, and clean up temporary directories" | |
107 | self.cleanup_process() |
|
110 | self.cleanup_process() | |
108 | for td in self.dirs: |
|
111 | for td in self.dirs: | |
109 | td.cleanup() |
|
112 | td.cleanup() | |
110 |
|
113 | |||
111 | __del__ = cleanup |
|
114 | __del__ = cleanup | |
112 |
|
115 | |||
113 |
|
116 | |||
114 | class PyTestController(TestController): |
|
117 | class PyTestController(TestController): | |
115 | """Run Python tests using IPython.testing.iptest""" |
|
118 | """Run Python tests using IPython.testing.iptest""" | |
116 | #: str, Python command to execute in subprocess |
|
119 | #: str, Python command to execute in subprocess | |
117 | pycmd = None |
|
120 | pycmd = None | |
118 |
|
121 | |||
119 | def __init__(self, section, options): |
|
122 | def __init__(self, section, options): | |
120 | """Create new test runner.""" |
|
123 | """Create new test runner.""" | |
121 | TestController.__init__(self) |
|
124 | TestController.__init__(self) | |
122 | self.section = section |
|
125 | self.section = section | |
123 | # pycmd is put into cmd[2] in PyTestController.launch() |
|
126 | # pycmd is put into cmd[2] in PyTestController.launch() | |
124 | self.cmd = [sys.executable, '-c', None, section] |
|
127 | self.cmd = [sys.executable, '-c', None, section] | |
125 | self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" |
|
128 | self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" | |
126 | self.options = options |
|
129 | self.options = options | |
127 |
|
130 | |||
128 | def setup(self): |
|
131 | def setup(self): | |
129 | ipydir = TemporaryDirectory() |
|
132 | ipydir = TemporaryDirectory() | |
130 | self.dirs.append(ipydir) |
|
133 | self.dirs.append(ipydir) | |
131 | self.env['IPYTHONDIR'] = ipydir.name |
|
134 | self.env['IPYTHONDIR'] = ipydir.name | |
132 | self.workingdir = workingdir = TemporaryDirectory() |
|
135 | self.workingdir = workingdir = TemporaryDirectory() | |
133 | self.dirs.append(workingdir) |
|
136 | self.dirs.append(workingdir) | |
134 | self.env['IPTEST_WORKING_DIR'] = workingdir.name |
|
137 | self.env['IPTEST_WORKING_DIR'] = workingdir.name | |
135 | # This means we won't get odd effects from our own matplotlib config |
|
138 | # This means we won't get odd effects from our own matplotlib config | |
136 | self.env['MPLCONFIGDIR'] = workingdir.name |
|
139 | self.env['MPLCONFIGDIR'] = workingdir.name | |
137 | # For security reasons (http://bugs.python.org/issue16202), use |
|
140 | # For security reasons (http://bugs.python.org/issue16202), use | |
138 | # a temporary directory to which other users have no access. |
|
141 | # a temporary directory to which other users have no access. | |
139 | self.env['TMPDIR'] = workingdir.name |
|
142 | self.env['TMPDIR'] = workingdir.name | |
140 |
|
143 | |||
141 | # Add a non-accessible directory to PATH (see gh-7053) |
|
144 | # Add a non-accessible directory to PATH (see gh-7053) | |
142 | noaccess = Path(self.workingdir.name) / "_no_access_" |
|
145 | noaccess = Path(self.workingdir.name) / "_no_access_" | |
143 | self.noaccess = noaccess.resolve() |
|
146 | self.noaccess = noaccess.resolve() | |
144 |
|
147 | |||
145 | noaccess.mkdir(0, exist_ok=True) |
|
148 | noaccess.mkdir(0, exist_ok=True) | |
146 |
|
149 | |||
147 | PATH = os.environ.get('PATH', '') |
|
150 | PATH = os.environ.get('PATH', '') | |
148 | if PATH: |
|
151 | if PATH: | |
149 | PATH = noaccess / PATH |
|
152 | PATH = noaccess / PATH | |
150 | else: |
|
153 | else: | |
151 | PATH = noaccess |
|
154 | PATH = noaccess | |
152 |
self.env[ |
|
155 | self.env["PATH"] = str(PATH) | |
153 |
|
156 | |||
154 | # From options: |
|
157 | # From options: | |
155 | if self.options.xunit: |
|
158 | if self.options.xunit: | |
156 | self.add_xunit() |
|
159 | self.add_xunit() | |
157 | if self.options.coverage: |
|
160 | if self.options.coverage: | |
158 | self.add_coverage() |
|
161 | self.add_coverage() | |
159 | self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams |
|
162 | self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams | |
160 | self.cmd.extend(self.options.extra_args) |
|
163 | self.cmd.extend(self.options.extra_args) | |
161 |
|
164 | |||
162 | def cleanup(self): |
|
165 | def cleanup(self): | |
163 | """ |
|
166 | """ | |
164 | Make the non-accessible directory created in setup() accessible |
|
167 | Make the non-accessible directory created in setup() accessible | |
165 | again, otherwise deleting the workingdir will fail. |
|
168 | again, otherwise deleting the workingdir will fail. | |
166 | """ |
|
169 | """ | |
167 | self.noaccess.chmod(stat.S_IRWXU) |
|
170 | self.noaccess.chmod(stat.S_IRWXU) | |
168 | TestController.cleanup(self) |
|
171 | TestController.cleanup(self) | |
169 |
|
172 | |||
170 | @property |
|
173 | @property | |
171 | def will_run(self): |
|
174 | def will_run(self): | |
172 | try: |
|
175 | try: | |
173 | return test_sections[self.section].will_run |
|
176 | return test_sections[self.section].will_run | |
174 | except KeyError: |
|
177 | except KeyError: | |
175 | return True |
|
178 | return True | |
176 |
|
179 | |||
177 | def add_xunit(self): |
|
180 | def add_xunit(self): | |
178 | xunit_file = Path(f"{self.section}.xunit.xml") |
|
181 | xunit_file = Path(f"{self.section}.xunit.xml") | |
179 | self.cmd.extend(["--with-xunit", "--xunit-file", xunit_file.absolute()]) |
|
182 | self.cmd.extend(["--with-xunit", "--xunit-file", xunit_file.absolute()]) | |
180 |
|
183 | |||
181 | def add_coverage(self): |
|
184 | def add_coverage(self): | |
182 | try: |
|
185 | try: | |
183 | sources = test_sections[self.section].includes |
|
186 | sources = test_sections[self.section].includes | |
184 | except KeyError: |
|
187 | except KeyError: | |
185 | sources = ["IPython"] |
|
188 | sources = ["IPython"] | |
186 |
|
189 | |||
187 | coverage_rc = ( |
|
190 | coverage_rc = ( | |
188 | "[run]\n" "data_file = {data_file}\n" "source =\n" " {source}\n" |
|
191 | "[run]\n" "data_file = {data_file}\n" "source =\n" " {source}\n" | |
189 | ).format( |
|
192 | ).format( | |
190 | data_file=Path(f".coverage.{self.section}").absolute(), |
|
193 | data_file=Path(f".coverage.{self.section}").absolute(), | |
191 | source="\n ".join(sources), |
|
194 | source="\n ".join(sources), | |
192 | ) |
|
195 | ) | |
193 | config_file = Path(self.workingdir.name) / ".coveragerc" |
|
196 | config_file: Path = Path(self.workingdir.name) / ".coveragerc" | |
194 | config_file.touch(exist_ok=True) |
|
197 | config_file.touch(exist_ok=True) | |
195 | config_file.write_text(coverage_rc) |
|
198 | config_file.write_text(coverage_rc) | |
196 |
|
199 | |||
197 | self.env["COVERAGE_PROCESS_START"] = config_file.resolve() |
|
200 | self.env["COVERAGE_PROCESS_START"] = str(config_file.resolve()) | |
198 | self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd |
|
201 | self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd | |
199 |
|
202 | |||
200 | def launch(self, buffer_output=False): |
|
203 | def launch(self, buffer_output=False): | |
201 | self.cmd[2] = self.pycmd |
|
204 | self.cmd[2] = self.pycmd | |
202 | super(PyTestController, self).launch(buffer_output=buffer_output) |
|
205 | super(PyTestController, self).launch(buffer_output=buffer_output) | |
203 |
|
206 | |||
204 |
|
207 | |||
205 | def prepare_controllers(options): |
|
208 | def prepare_controllers(options): | |
206 | """Returns two lists of TestController instances, those to run, and those |
|
209 | """Returns two lists of TestController instances, those to run, and those | |
207 | not to run.""" |
|
210 | not to run.""" | |
208 | testgroups = options.testgroups |
|
211 | testgroups = options.testgroups | |
209 | if not testgroups: |
|
212 | if not testgroups: | |
210 | testgroups = py_test_group_names |
|
213 | testgroups = py_test_group_names | |
211 |
|
214 | |||
212 | controllers = [PyTestController(name, options) for name in testgroups] |
|
215 | controllers = [PyTestController(name, options) for name in testgroups] | |
213 |
|
216 | |||
214 | to_run = [c for c in controllers if c.will_run] |
|
217 | to_run = [c for c in controllers if c.will_run] | |
215 | not_run = [c for c in controllers if not c.will_run] |
|
218 | not_run = [c for c in controllers if not c.will_run] | |
216 | return to_run, not_run |
|
219 | return to_run, not_run | |
217 |
|
220 | |||
218 | def do_run(controller, buffer_output=True): |
|
221 | def do_run(controller, buffer_output=True): | |
219 | """Setup and run a test controller. |
|
222 | """Setup and run a test controller. | |
220 |
|
223 | |||
221 | If buffer_output is True, no output is displayed, to avoid it appearing |
|
224 | If buffer_output is True, no output is displayed, to avoid it appearing | |
222 | interleaved. In this case, the caller is responsible for displaying test |
|
225 | interleaved. In this case, the caller is responsible for displaying test | |
223 | output on failure. |
|
226 | output on failure. | |
224 |
|
227 | |||
225 | Returns |
|
228 | Returns | |
226 | ------- |
|
229 | ------- | |
227 | controller : TestController |
|
230 | controller : TestController | |
228 | The same controller as passed in, as a convenience for using map() type |
|
231 | The same controller as passed in, as a convenience for using map() type | |
229 | APIs. |
|
232 | APIs. | |
230 | exitcode : int |
|
233 | exitcode : int | |
231 | The exit code of the test subprocess. Non-zero indicates failure. |
|
234 | The exit code of the test subprocess. Non-zero indicates failure. | |
232 | """ |
|
235 | """ | |
233 | try: |
|
236 | try: | |
234 | try: |
|
237 | try: | |
235 | controller.setup() |
|
238 | controller.setup() | |
236 | controller.launch(buffer_output=buffer_output) |
|
239 | controller.launch(buffer_output=buffer_output) | |
237 | except Exception: |
|
240 | except Exception: | |
238 | import traceback |
|
241 | import traceback | |
239 | traceback.print_exc() |
|
242 | traceback.print_exc() | |
240 | return controller, 1 # signal failure |
|
243 | return controller, 1 # signal failure | |
241 |
|
244 | |||
242 | exitcode = controller.wait() |
|
245 | exitcode = controller.wait() | |
243 | return controller, exitcode |
|
246 | return controller, exitcode | |
244 |
|
247 | |||
245 | except KeyboardInterrupt: |
|
248 | except KeyboardInterrupt: | |
246 | return controller, -signal.SIGINT |
|
249 | return controller, -signal.SIGINT | |
247 | finally: |
|
250 | finally: | |
248 | controller.cleanup() |
|
251 | controller.cleanup() | |
249 |
|
252 | |||
250 | def report(): |
|
253 | def report(): | |
251 | """Return a string with a summary report of test-related variables.""" |
|
254 | """Return a string with a summary report of test-related variables.""" | |
252 | inf = get_sys_info() |
|
255 | inf = get_sys_info() | |
253 | out = [] |
|
256 | out = [] | |
254 | def _add(name, value): |
|
257 | def _add(name, value): | |
255 | out.append((name, value)) |
|
258 | out.append((name, value)) | |
256 |
|
259 | |||
257 | _add('IPython version', inf['ipython_version']) |
|
260 | _add('IPython version', inf['ipython_version']) | |
258 | _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source'])) |
|
261 | _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source'])) | |
259 | _add('IPython package', compress_user(inf['ipython_path'])) |
|
262 | _add('IPython package', compress_user(inf['ipython_path'])) | |
260 | _add('Python version', inf['sys_version'].replace('\n','')) |
|
263 | _add('Python version', inf['sys_version'].replace('\n','')) | |
261 | _add('sys.executable', compress_user(inf['sys_executable'])) |
|
264 | _add('sys.executable', compress_user(inf['sys_executable'])) | |
262 | _add('Platform', inf['platform']) |
|
265 | _add('Platform', inf['platform']) | |
263 |
|
266 | |||
264 | width = max(len(n) for (n,v) in out) |
|
267 | width = max(len(n) for (n,v) in out) | |
265 | out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] |
|
268 | out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] | |
266 |
|
269 | |||
267 | avail = [] |
|
270 | avail = [] | |
268 | not_avail = [] |
|
271 | not_avail = [] | |
269 |
|
272 | |||
270 | for k, is_avail in have.items(): |
|
273 | for k, is_avail in have.items(): | |
271 | if is_avail: |
|
274 | if is_avail: | |
272 | avail.append(k) |
|
275 | avail.append(k) | |
273 | else: |
|
276 | else: | |
274 | not_avail.append(k) |
|
277 | not_avail.append(k) | |
275 |
|
278 | |||
276 | if avail: |
|
279 | if avail: | |
277 | out.append('\nTools and libraries available at test time:\n') |
|
280 | out.append('\nTools and libraries available at test time:\n') | |
278 | avail.sort() |
|
281 | avail.sort() | |
279 | out.append(' ' + ' '.join(avail)+'\n') |
|
282 | out.append(' ' + ' '.join(avail)+'\n') | |
280 |
|
283 | |||
281 | if not_avail: |
|
284 | if not_avail: | |
282 | out.append('\nTools and libraries NOT available at test time:\n') |
|
285 | out.append('\nTools and libraries NOT available at test time:\n') | |
283 | not_avail.sort() |
|
286 | not_avail.sort() | |
284 | out.append(' ' + ' '.join(not_avail)+'\n') |
|
287 | out.append(' ' + ' '.join(not_avail)+'\n') | |
285 |
|
288 | |||
286 | return ''.join(out) |
|
289 | return ''.join(out) | |
287 |
|
290 | |||
288 | def run_iptestall(options): |
|
291 | def run_iptestall(options): | |
289 | """Run the entire IPython test suite by calling nose and trial. |
|
292 | """Run the entire IPython test suite by calling nose and trial. | |
290 |
|
293 | |||
291 | This function constructs :class:`IPTester` instances for all IPython |
|
294 | This function constructs :class:`IPTester` instances for all IPython | |
292 | modules and package and then runs each of them. This causes the modules |
|
295 | modules and package and then runs each of them. This causes the modules | |
293 | and packages of IPython to be tested each in their own subprocess using |
|
296 | and packages of IPython to be tested each in their own subprocess using | |
294 | nose. |
|
297 | nose. | |
295 |
|
298 | |||
296 | Parameters |
|
299 | Parameters | |
297 | ---------- |
|
300 | ---------- | |
298 |
|
301 | |||
299 | All parameters are passed as attributes of the options object. |
|
302 | All parameters are passed as attributes of the options object. | |
300 |
|
303 | |||
301 | testgroups : list of str |
|
304 | testgroups : list of str | |
302 | Run only these sections of the test suite. If empty, run all the available |
|
305 | Run only these sections of the test suite. If empty, run all the available | |
303 | sections. |
|
306 | sections. | |
304 |
|
307 | |||
305 | fast : int or None |
|
308 | fast : int or None | |
306 | Run the test suite in parallel, using n simultaneous processes. If None |
|
309 | Run the test suite in parallel, using n simultaneous processes. If None | |
307 | is passed, one process is used per CPU core. Default 1 (i.e. sequential) |
|
310 | is passed, one process is used per CPU core. Default 1 (i.e. sequential) | |
308 |
|
311 | |||
309 | inc_slow : bool |
|
312 | inc_slow : bool | |
310 | Include slow tests. By default, these tests aren't run. |
|
313 | Include slow tests. By default, these tests aren't run. | |
311 |
|
314 | |||
312 | url : unicode |
|
315 | url : unicode | |
313 | Address:port to use when running the JS tests. |
|
316 | Address:port to use when running the JS tests. | |
314 |
|
317 | |||
315 | xunit : bool |
|
318 | xunit : bool | |
316 | Produce Xunit XML output. This is written to multiple foo.xunit.xml files. |
|
319 | Produce Xunit XML output. This is written to multiple foo.xunit.xml files. | |
317 |
|
320 | |||
318 | coverage : bool or str |
|
321 | coverage : bool or str | |
319 | Measure code coverage from tests. True will store the raw coverage data, |
|
322 | Measure code coverage from tests. True will store the raw coverage data, | |
320 | or pass 'html' or 'xml' to get reports. |
|
323 | or pass 'html' or 'xml' to get reports. | |
321 |
|
324 | |||
322 | extra_args : list |
|
325 | extra_args : list | |
323 | Extra arguments to pass to the test subprocesses, e.g. '-v' |
|
326 | Extra arguments to pass to the test subprocesses, e.g. '-v' | |
324 | """ |
|
327 | """ | |
325 | to_run, not_run = prepare_controllers(options) |
|
328 | to_run, not_run = prepare_controllers(options) | |
326 |
|
329 | |||
327 | def justify(ltext, rtext, width=70, fill='-'): |
|
330 | def justify(ltext, rtext, width=70, fill='-'): | |
328 | ltext += ' ' |
|
331 | ltext += ' ' | |
329 | rtext = (' ' + rtext).rjust(width - len(ltext), fill) |
|
332 | rtext = (' ' + rtext).rjust(width - len(ltext), fill) | |
330 | return ltext + rtext |
|
333 | return ltext + rtext | |
331 |
|
334 | |||
332 | # Run all test runners, tracking execution time |
|
335 | # Run all test runners, tracking execution time | |
333 | failed = [] |
|
336 | failed = [] | |
334 | t_start = time.time() |
|
337 | t_start = time.time() | |
335 |
|
338 | |||
336 | print() |
|
339 | print() | |
337 | if options.fast == 1: |
|
340 | if options.fast == 1: | |
338 | # This actually means sequential, i.e. with 1 job |
|
341 | # This actually means sequential, i.e. with 1 job | |
339 | for controller in to_run: |
|
342 | for controller in to_run: | |
340 | print('Test group:', controller.section) |
|
343 | print('Test group:', controller.section) | |
341 | sys.stdout.flush() # Show in correct order when output is piped |
|
344 | sys.stdout.flush() # Show in correct order when output is piped | |
342 | controller, res = do_run(controller, buffer_output=False) |
|
345 | controller, res = do_run(controller, buffer_output=False) | |
343 | if res: |
|
346 | if res: | |
344 | failed.append(controller) |
|
347 | failed.append(controller) | |
345 | if res == -signal.SIGINT: |
|
348 | if res == -signal.SIGINT: | |
346 | print("Interrupted") |
|
349 | print("Interrupted") | |
347 | break |
|
350 | break | |
348 | print() |
|
351 | print() | |
349 |
|
352 | |||
350 | else: |
|
353 | else: | |
351 | # Run tests concurrently |
|
354 | # Run tests concurrently | |
352 | try: |
|
355 | try: | |
353 | pool = multiprocessing.pool.ThreadPool(options.fast) |
|
356 | pool = multiprocessing.pool.ThreadPool(options.fast) | |
354 | for (controller, res) in pool.imap_unordered(do_run, to_run): |
|
357 | for (controller, res) in pool.imap_unordered(do_run, to_run): | |
355 | res_string = 'OK' if res == 0 else 'FAILED' |
|
358 | res_string = 'OK' if res == 0 else 'FAILED' | |
356 | print(justify('Test group: ' + controller.section, res_string)) |
|
359 | print(justify('Test group: ' + controller.section, res_string)) | |
357 | if res: |
|
360 | if res: | |
358 | print(decode(controller.stdout)) |
|
361 | print(decode(controller.stdout)) | |
359 | failed.append(controller) |
|
362 | failed.append(controller) | |
360 | if res == -signal.SIGINT: |
|
363 | if res == -signal.SIGINT: | |
361 | print("Interrupted") |
|
364 | print("Interrupted") | |
362 | break |
|
365 | break | |
363 | except KeyboardInterrupt: |
|
366 | except KeyboardInterrupt: | |
364 | return |
|
367 | return | |
365 |
|
368 | |||
366 | for controller in not_run: |
|
369 | for controller in not_run: | |
367 | print(justify('Test group: ' + controller.section, 'NOT RUN')) |
|
370 | print(justify('Test group: ' + controller.section, 'NOT RUN')) | |
368 |
|
371 | |||
369 | t_end = time.time() |
|
372 | t_end = time.time() | |
370 | t_tests = t_end - t_start |
|
373 | t_tests = t_end - t_start | |
371 | nrunners = len(to_run) |
|
374 | nrunners = len(to_run) | |
372 | nfail = len(failed) |
|
375 | nfail = len(failed) | |
373 | # summarize results |
|
376 | # summarize results | |
374 | print('_'*70) |
|
377 | print('_'*70) | |
375 | print('Test suite completed for system with the following information:') |
|
378 | print('Test suite completed for system with the following information:') | |
376 | print(report()) |
|
379 | print(report()) | |
377 | took = "Took %.3fs." % t_tests |
|
380 | took = "Took %.3fs." % t_tests | |
378 | print('Status: ', end='') |
|
381 | print('Status: ', end='') | |
379 | if not failed: |
|
382 | if not failed: | |
380 | print('OK (%d test groups).' % nrunners, took) |
|
383 | print('OK (%d test groups).' % nrunners, took) | |
381 | else: |
|
384 | else: | |
382 | # If anything went wrong, point out what command to rerun manually to |
|
385 | # If anything went wrong, point out what command to rerun manually to | |
383 | # see the actual errors and individual summary |
|
386 | # see the actual errors and individual summary | |
384 | failed_sections = [c.section for c in failed] |
|
387 | failed_sections = [c.section for c in failed] | |
385 | print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, |
|
388 | print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, | |
386 | nrunners, ', '.join(failed_sections)), took) |
|
389 | nrunners, ', '.join(failed_sections)), took) | |
387 | print() |
|
390 | print() | |
388 | print('You may wish to rerun these, with:') |
|
391 | print('You may wish to rerun these, with:') | |
389 | print(' iptest', *failed_sections) |
|
392 | print(' iptest', *failed_sections) | |
390 | print() |
|
393 | print() | |
391 |
|
394 | |||
392 | if options.coverage: |
|
395 | if options.coverage: | |
393 | from coverage import coverage, CoverageException |
|
396 | from coverage import coverage, CoverageException | |
394 | cov = coverage(data_file='.coverage') |
|
397 | cov = coverage(data_file='.coverage') | |
395 | cov.combine() |
|
398 | cov.combine() | |
396 | cov.save() |
|
399 | cov.save() | |
397 |
|
400 | |||
398 | # Coverage HTML report |
|
401 | # Coverage HTML report | |
399 | if options.coverage == 'html': |
|
402 | if options.coverage == 'html': | |
400 | html_dir = 'ipy_htmlcov' |
|
403 | html_dir = 'ipy_htmlcov' | |
401 | shutil.rmtree(html_dir, ignore_errors=True) |
|
404 | shutil.rmtree(html_dir, ignore_errors=True) | |
402 | print("Writing HTML coverage report to %s/ ... " % html_dir, end="") |
|
405 | print("Writing HTML coverage report to %s/ ... " % html_dir, end="") | |
403 | sys.stdout.flush() |
|
406 | sys.stdout.flush() | |
404 |
|
407 | |||
405 | # Custom HTML reporter to clean up module names. |
|
408 | # Custom HTML reporter to clean up module names. | |
406 | from coverage.html import HtmlReporter |
|
409 | from coverage.html import HtmlReporter | |
407 | class CustomHtmlReporter(HtmlReporter): |
|
410 | class CustomHtmlReporter(HtmlReporter): | |
408 | def find_code_units(self, morfs): |
|
411 | def find_code_units(self, morfs): | |
409 | super(CustomHtmlReporter, self).find_code_units(morfs) |
|
412 | super(CustomHtmlReporter, self).find_code_units(morfs) | |
410 | for cu in self.code_units: |
|
413 | for cu in self.code_units: | |
411 | nameparts = cu.name.split(os.sep) |
|
414 | nameparts = cu.name.split(os.sep) | |
412 | if 'IPython' not in nameparts: |
|
415 | if 'IPython' not in nameparts: | |
413 | continue |
|
416 | continue | |
414 | ix = nameparts.index('IPython') |
|
417 | ix = nameparts.index('IPython') | |
415 | cu.name = '.'.join(nameparts[ix:]) |
|
418 | cu.name = '.'.join(nameparts[ix:]) | |
416 |
|
419 | |||
417 | # Reimplement the html_report method with our custom reporter |
|
420 | # Reimplement the html_report method with our custom reporter | |
418 | cov.get_data() |
|
421 | cov.get_data() | |
419 | cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir, |
|
422 | cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir, | |
420 | html_title='IPython test coverage', |
|
423 | html_title='IPython test coverage', | |
421 | ) |
|
424 | ) | |
422 | reporter = CustomHtmlReporter(cov, cov.config) |
|
425 | reporter = CustomHtmlReporter(cov, cov.config) | |
423 | reporter.report(None) |
|
426 | reporter.report(None) | |
424 | print('done.') |
|
427 | print('done.') | |
425 |
|
428 | |||
426 | # Coverage XML report |
|
429 | # Coverage XML report | |
427 | elif options.coverage == 'xml': |
|
430 | elif options.coverage == 'xml': | |
428 | try: |
|
431 | try: | |
429 | cov.xml_report(outfile='ipy_coverage.xml') |
|
432 | cov.xml_report(outfile='ipy_coverage.xml') | |
430 | except CoverageException as e: |
|
433 | except CoverageException as e: | |
431 | print('Generating coverage report failed. Are you running javascript tests only?') |
|
434 | print('Generating coverage report failed. Are you running javascript tests only?') | |
432 | import traceback |
|
435 | import traceback | |
433 | traceback.print_exc() |
|
436 | traceback.print_exc() | |
434 |
|
437 | |||
435 | if failed: |
|
438 | if failed: | |
436 | # Ensure that our exit code indicates failure |
|
439 | # Ensure that our exit code indicates failure | |
437 | sys.exit(1) |
|
440 | sys.exit(1) | |
438 |
|
441 | |||
439 | argparser = argparse.ArgumentParser(description='Run IPython test suite') |
|
442 | argparser = argparse.ArgumentParser(description='Run IPython test suite') | |
440 | argparser.add_argument('testgroups', nargs='*', |
|
443 | argparser.add_argument('testgroups', nargs='*', | |
441 | help='Run specified groups of tests. If omitted, run ' |
|
444 | help='Run specified groups of tests. If omitted, run ' | |
442 | 'all tests.') |
|
445 | 'all tests.') | |
443 | argparser.add_argument('--all', action='store_true', |
|
446 | argparser.add_argument('--all', action='store_true', | |
444 | help='Include slow tests not run by default.') |
|
447 | help='Include slow tests not run by default.') | |
445 | argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, |
|
448 | argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, | |
446 | help='Run test sections in parallel. This starts as many ' |
|
449 | help='Run test sections in parallel. This starts as many ' | |
447 | 'processes as you have cores, or you can specify a number.') |
|
450 | 'processes as you have cores, or you can specify a number.') | |
448 | argparser.add_argument('--xunit', action='store_true', |
|
451 | argparser.add_argument('--xunit', action='store_true', | |
449 | help='Produce Xunit XML results') |
|
452 | help='Produce Xunit XML results') | |
450 | argparser.add_argument('--coverage', nargs='?', const=True, default=False, |
|
453 | argparser.add_argument('--coverage', nargs='?', const=True, default=False, | |
451 | help="Measure test coverage. Specify 'html' or " |
|
454 | help="Measure test coverage. Specify 'html' or " | |
452 | "'xml' to get reports.") |
|
455 | "'xml' to get reports.") | |
453 | argparser.add_argument('--subproc-streams', default='capture', |
|
456 | argparser.add_argument('--subproc-streams', default='capture', | |
454 | help="What to do with stdout/stderr from subprocesses. " |
|
457 | help="What to do with stdout/stderr from subprocesses. " | |
455 | "'capture' (default), 'show' and 'discard' are the options.") |
|
458 | "'capture' (default), 'show' and 'discard' are the options.") | |
456 |
|
459 | |||
457 | def default_options(): |
|
460 | def default_options(): | |
458 | """Get an argparse Namespace object with the default arguments, to pass to |
|
461 | """Get an argparse Namespace object with the default arguments, to pass to | |
459 | :func:`run_iptestall`. |
|
462 | :func:`run_iptestall`. | |
460 | """ |
|
463 | """ | |
461 | options = argparser.parse_args([]) |
|
464 | options = argparser.parse_args([]) | |
462 | options.extra_args = [] |
|
465 | options.extra_args = [] | |
463 | return options |
|
466 | return options | |
464 |
|
467 | |||
465 | def main(): |
|
468 | def main(): | |
466 | # iptest doesn't work correctly if the working directory is the |
|
469 | # iptest doesn't work correctly if the working directory is the | |
467 | # root of the IPython source tree. Tell the user to avoid |
|
470 | # root of the IPython source tree. Tell the user to avoid | |
468 | # frustration. |
|
471 | # frustration. | |
469 | main_file = Path.cwd() / Path("IPython/testing/__main__.py") |
|
472 | main_file = Path.cwd() / Path("IPython/testing/__main__.py") | |
470 |
|
473 | |||
471 | if main_file.exists(): |
|
474 | if main_file.exists(): | |
472 | print("Don't run iptest from the IPython source directory", file=sys.stderr) |
|
475 | print("Don't run iptest from the IPython source directory", file=sys.stderr) | |
473 | sys.exit(1) |
|
476 | sys.exit(1) | |
474 | # Arguments after -- should be passed through to nose. Argparse treats |
|
477 | # Arguments after -- should be passed through to nose. Argparse treats | |
475 | # everything after -- as regular positional arguments, so we separate them |
|
478 | # everything after -- as regular positional arguments, so we separate them | |
476 | # first. |
|
479 | # first. | |
477 | try: |
|
480 | try: | |
478 | ix = sys.argv.index('--') |
|
481 | ix = sys.argv.index('--') | |
479 | except ValueError: |
|
482 | except ValueError: | |
480 | to_parse = sys.argv[1:] |
|
483 | to_parse = sys.argv[1:] | |
481 | extra_args = [] |
|
484 | extra_args = [] | |
482 | else: |
|
485 | else: | |
483 | to_parse = sys.argv[1:ix] |
|
486 | to_parse = sys.argv[1:ix] | |
484 | extra_args = sys.argv[ix+1:] |
|
487 | extra_args = sys.argv[ix+1:] | |
485 |
|
488 | |||
486 | options = argparser.parse_args(to_parse) |
|
489 | options = argparser.parse_args(to_parse) | |
487 | options.extra_args = extra_args |
|
490 | options.extra_args = extra_args | |
488 |
|
491 | |||
489 | run_iptestall(options) |
|
492 | run_iptestall(options) | |
490 |
|
493 | |||
491 |
|
494 | |||
492 | if __name__ == '__main__': |
|
495 | if __name__ == '__main__': | |
493 | main() |
|
496 | main() |
General Comments 0
You need to be logged in to leave comments.
Login now