Show More
@@ -1,278 +1,284 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """IPython Test Process Controller |
|
2 | """IPython Test Process Controller | |
3 |
|
3 | |||
4 | This module runs one or more subprocesses which will actually run the IPython |
|
4 | This module runs one or more subprocesses which will actually run the IPython | |
5 | test suite. |
|
5 | test suite. | |
6 |
|
6 | |||
7 | """ |
|
7 | """ | |
8 |
|
8 | |||
9 | #----------------------------------------------------------------------------- |
|
9 | #----------------------------------------------------------------------------- | |
10 | # Copyright (C) 2009-2011 The IPython Development Team |
|
10 | # Copyright (C) 2009-2011 The IPython Development Team | |
11 | # |
|
11 | # | |
12 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | # Distributed under the terms of the BSD License. The full license is in | |
13 | # the file COPYING, distributed as part of this software. |
|
13 | # the file COPYING, distributed as part of this software. | |
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 |
|
15 | |||
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 | # Imports |
|
17 | # Imports | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | from __future__ import print_function |
|
19 | from __future__ import print_function | |
20 |
|
20 | |||
21 | import argparse |
|
21 | import argparse | |
22 | import multiprocessing.pool |
|
22 | import multiprocessing.pool | |
23 | import os |
|
23 | import os | |
24 | import signal |
|
24 | import signal | |
25 | import sys |
|
25 | import sys | |
26 | import subprocess |
|
26 | import subprocess | |
27 | import time |
|
27 | import time | |
28 |
|
28 | |||
29 | from .iptest import have, test_group_names, test_sections |
|
29 | from .iptest import have, test_group_names, test_sections | |
30 | from IPython.utils import py3compat |
|
30 | from IPython.utils import py3compat | |
31 | from IPython.utils.sysinfo import sys_info |
|
31 | from IPython.utils.sysinfo import sys_info | |
32 | from IPython.utils.tempdir import TemporaryDirectory |
|
32 | from IPython.utils.tempdir import TemporaryDirectory | |
33 |
|
33 | |||
34 |
|
34 | |||
35 | class IPTestController(object): |
|
35 | class IPTestController(object): | |
36 | """Run iptest in a subprocess |
|
36 | """Run iptest in a subprocess | |
37 | """ |
|
37 | """ | |
38 | #: str, IPython test suite to be executed. |
|
38 | #: str, IPython test suite to be executed. | |
39 | section = None |
|
39 | section = None | |
40 | #: list, command line arguments to be executed |
|
40 | #: list, command line arguments to be executed | |
41 | cmd = None |
|
41 | cmd = None | |
42 | #: dict, extra environment variables to set for the subprocess |
|
42 | #: dict, extra environment variables to set for the subprocess | |
43 | env = None |
|
43 | env = None | |
44 | #: list, TemporaryDirectory instances to clear up when the process finishes |
|
44 | #: list, TemporaryDirectory instances to clear up when the process finishes | |
45 | dirs = None |
|
45 | dirs = None | |
46 | #: subprocess.Popen instance |
|
46 | #: subprocess.Popen instance | |
47 | process = None |
|
47 | process = None | |
48 | buffer_output = False |
|
48 | buffer_output = False | |
49 |
|
49 | |||
50 | def __init__(self, section): |
|
50 | def __init__(self, section): | |
51 | """Create new test runner.""" |
|
51 | """Create new test runner.""" | |
52 | self.section = section |
|
52 | self.section = section | |
53 | self.cmd = [sys.executable, '-m', 'IPython.testing.iptest', section] |
|
53 | self.cmd = [sys.executable, '-m', 'IPython.testing.iptest', section] | |
54 | self.env = {} |
|
54 | self.env = {} | |
55 | self.dirs = [] |
|
55 | self.dirs = [] | |
56 | ipydir = TemporaryDirectory() |
|
56 | ipydir = TemporaryDirectory() | |
57 | self.dirs.append(ipydir) |
|
57 | self.dirs.append(ipydir) | |
58 | self.env['IPYTHONDIR'] = ipydir.name |
|
58 | self.env['IPYTHONDIR'] = ipydir.name | |
59 | workingdir = TemporaryDirectory() |
|
59 | workingdir = TemporaryDirectory() | |
60 | self.dirs.append(workingdir) |
|
60 | self.dirs.append(workingdir) | |
61 | self.env['IPTEST_WORKING_DIR'] = workingdir.name |
|
61 | self.env['IPTEST_WORKING_DIR'] = workingdir.name | |
62 |
|
62 | |||
63 | def add_xunit(self): |
|
63 | def add_xunit(self): | |
64 | xunit_file = os.path.abspath(self.section + '.xunit.xml') |
|
64 | xunit_file = os.path.abspath(self.section + '.xunit.xml') | |
65 | self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) |
|
65 | self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) | |
66 |
|
66 | |||
67 | def add_coverage(self, xml=True): |
|
67 | def add_coverage(self, xml=True): | |
68 | self.cmd.extend(['--with-coverage', '--cover-package', self.section]) |
|
68 | self.cmd.extend(['--with-coverage', '--cover-package', self.section]) | |
69 | if xml: |
|
69 | if xml: | |
70 | coverage_xml = os.path.abspath(self.section + ".coverage.xml") |
|
70 | coverage_xml = os.path.abspath(self.section + ".coverage.xml") | |
71 | self.cmd.extend(['--cover-xml', '--cover-xml-file', coverage_xml]) |
|
71 | self.cmd.extend(['--cover-xml', '--cover-xml-file', coverage_xml]) | |
72 |
|
72 | |||
73 |
|
73 | |||
74 | def launch(self): |
|
74 | def launch(self): | |
75 | # print('*** ENV:', self.env) # dbg |
|
75 | # print('*** ENV:', self.env) # dbg | |
76 | # print('*** CMD:', self.cmd) # dbg |
|
76 | # print('*** CMD:', self.cmd) # dbg | |
77 | env = os.environ.copy() |
|
77 | env = os.environ.copy() | |
78 | env.update(self.env) |
|
78 | env.update(self.env) | |
79 | output = subprocess.PIPE if self.buffer_output else None |
|
79 | output = subprocess.PIPE if self.buffer_output else None | |
|
80 | stdout = subprocess.STDOUT if self.buffer_output else None | |||
80 | self.process = subprocess.Popen(self.cmd, stdout=output, |
|
81 | self.process = subprocess.Popen(self.cmd, stdout=output, | |
81 |
stderr= |
|
82 | stderr=stdout, env=env) | |
82 |
|
83 | |||
83 |
def |
|
84 | def wait(self): | |
84 | """Run the stored commands""" |
|
85 | self.stdout, _ = self.process.communicate() | |
85 | try: |
|
86 | return self.process.returncode | |
86 | retcode = self._run_cmd() |
|
|||
87 | except KeyboardInterrupt: |
|
|||
88 | return -signal.SIGINT |
|
|||
89 | except: |
|
|||
90 | import traceback |
|
|||
91 | traceback.print_exc() |
|
|||
92 | return 1 # signal failure |
|
|||
93 |
|
||||
94 | if self.coverage_xml: |
|
|||
95 | subprocess.call(["coverage", "xml", "-o", self.coverage_xml]) |
|
|||
96 | return retcode |
|
|||
97 |
|
87 | |||
98 | def cleanup_process(self): |
|
88 | def cleanup_process(self): | |
99 | """Cleanup on exit by killing any leftover processes.""" |
|
89 | """Cleanup on exit by killing any leftover processes.""" | |
100 | subp = self.process |
|
90 | subp = self.process | |
101 | if subp is None or (subp.poll() is not None): |
|
91 | if subp is None or (subp.poll() is not None): | |
102 | return # Process doesn't exist, or is already dead. |
|
92 | return # Process doesn't exist, or is already dead. | |
103 |
|
93 | |||
104 | try: |
|
94 | try: | |
105 | print('Cleaning up stale PID: %d' % subp.pid) |
|
95 | print('Cleaning up stale PID: %d' % subp.pid) | |
106 | subp.kill() |
|
96 | subp.kill() | |
107 | except: # (OSError, WindowsError) ? |
|
97 | except: # (OSError, WindowsError) ? | |
108 | # This is just a best effort, if we fail or the process was |
|
98 | # This is just a best effort, if we fail or the process was | |
109 | # really gone, ignore it. |
|
99 | # really gone, ignore it. | |
110 | pass |
|
100 | pass | |
111 | else: |
|
101 | else: | |
112 | for i in range(10): |
|
102 | for i in range(10): | |
113 | if subp.poll() is None: |
|
103 | if subp.poll() is None: | |
114 | time.sleep(0.1) |
|
104 | time.sleep(0.1) | |
115 | else: |
|
105 | else: | |
116 | break |
|
106 | break | |
117 |
|
107 | |||
118 | if subp.poll() is None: |
|
108 | if subp.poll() is None: | |
119 | # The process did not die... |
|
109 | # The process did not die... | |
120 | print('... failed. Manual cleanup may be required.') |
|
110 | print('... failed. Manual cleanup may be required.') | |
121 |
|
111 | |||
122 | def cleanup(self): |
|
112 | def cleanup(self): | |
123 | "Kill process if it's still alive, and clean up temporary directories" |
|
113 | "Kill process if it's still alive, and clean up temporary directories" | |
124 | self.cleanup_process() |
|
114 | self.cleanup_process() | |
125 | for td in self.dirs: |
|
115 | for td in self.dirs: | |
126 | td.cleanup() |
|
116 | td.cleanup() | |
127 |
|
117 | |||
128 | __del__ = cleanup |
|
118 | __del__ = cleanup | |
129 |
|
119 | |||
130 | def test_controllers_to_run(inc_slow=False): |
|
120 | def test_controllers_to_run(inc_slow=False): | |
131 | """Returns an ordered list of IPTestController instances to be run.""" |
|
121 | """Returns an ordered list of IPTestController instances to be run.""" | |
132 | res = [] |
|
122 | res = [] | |
133 | if not inc_slow: |
|
123 | if not inc_slow: | |
134 | test_sections['parallel'].enabled = False |
|
124 | test_sections['parallel'].enabled = False | |
|
125 | ||||
135 | for name in test_group_names: |
|
126 | for name in test_group_names: | |
136 | if test_sections[name].will_run: |
|
127 | if test_sections[name].will_run: | |
137 | res.append(IPTestController(name)) |
|
128 | res.append(IPTestController(name)) | |
138 | return res |
|
129 | return res | |
139 |
|
130 | |||
140 | def do_run(controller): |
|
131 | def do_run(controller): | |
141 | try: |
|
132 | try: | |
142 | try: |
|
133 | try: | |
143 | controller.launch() |
|
134 | controller.launch() | |
144 | except Exception: |
|
135 | except Exception: | |
145 | import traceback |
|
136 | import traceback | |
146 | traceback.print_exc() |
|
137 | traceback.print_exc() | |
147 | return controller, 1 # signal failure |
|
138 | return controller, 1 # signal failure | |
148 |
|
139 | |||
149 |
exitcode = controller. |
|
140 | exitcode = controller.wait() | |
150 | controller.cleanup() |
|
|||
151 | return controller, exitcode |
|
141 | return controller, exitcode | |
152 |
|
142 | |||
153 | except KeyboardInterrupt: |
|
143 | except KeyboardInterrupt: | |
154 | controller.cleanup() |
|
|||
155 | return controller, -signal.SIGINT |
|
144 | return controller, -signal.SIGINT | |
|
145 | finally: | |||
|
146 | controller.cleanup() | |||
156 |
|
147 | |||
157 | def report(): |
|
148 | def report(): | |
158 | """Return a string with a summary report of test-related variables.""" |
|
149 | """Return a string with a summary report of test-related variables.""" | |
159 |
|
150 | |||
160 | out = [ sys_info(), '\n'] |
|
151 | out = [ sys_info(), '\n'] | |
161 |
|
152 | |||
162 | avail = [] |
|
153 | avail = [] | |
163 | not_avail = [] |
|
154 | not_avail = [] | |
164 |
|
155 | |||
165 | for k, is_avail in have.items(): |
|
156 | for k, is_avail in have.items(): | |
166 | if is_avail: |
|
157 | if is_avail: | |
167 | avail.append(k) |
|
158 | avail.append(k) | |
168 | else: |
|
159 | else: | |
169 | not_avail.append(k) |
|
160 | not_avail.append(k) | |
170 |
|
161 | |||
171 | if avail: |
|
162 | if avail: | |
172 | out.append('\nTools and libraries available at test time:\n') |
|
163 | out.append('\nTools and libraries available at test time:\n') | |
173 | avail.sort() |
|
164 | avail.sort() | |
174 | out.append(' ' + ' '.join(avail)+'\n') |
|
165 | out.append(' ' + ' '.join(avail)+'\n') | |
175 |
|
166 | |||
176 | if not_avail: |
|
167 | if not_avail: | |
177 | out.append('\nTools and libraries NOT available at test time:\n') |
|
168 | out.append('\nTools and libraries NOT available at test time:\n') | |
178 | not_avail.sort() |
|
169 | not_avail.sort() | |
179 | out.append(' ' + ' '.join(not_avail)+'\n') |
|
170 | out.append(' ' + ' '.join(not_avail)+'\n') | |
180 |
|
171 | |||
181 | return ''.join(out) |
|
172 | return ''.join(out) | |
182 |
|
173 | |||
183 | def run_iptestall(inc_slow=False, jobs=1, xunit=False, coverage=False): |
|
174 | def run_iptestall(inc_slow=False, jobs=1, xunit=False, coverage=False): | |
184 | """Run the entire IPython test suite by calling nose and trial. |
|
175 | """Run the entire IPython test suite by calling nose and trial. | |
185 |
|
176 | |||
186 | This function constructs :class:`IPTester` instances for all IPython |
|
177 | This function constructs :class:`IPTester` instances for all IPython | |
187 | modules and package and then runs each of them. This causes the modules |
|
178 | modules and package and then runs each of them. This causes the modules | |
188 | and packages of IPython to be tested each in their own subprocess using |
|
179 | and packages of IPython to be tested each in their own subprocess using | |
189 | nose. |
|
180 | nose. | |
190 |
|
181 | |||
191 | Parameters |
|
182 | Parameters | |
192 | ---------- |
|
183 | ---------- | |
193 |
|
184 | |||
194 | inc_slow : bool, optional |
|
185 | inc_slow : bool, optional | |
195 | Include slow tests, like IPython.parallel. By default, these tests aren't |
|
186 | Include slow tests, like IPython.parallel. By default, these tests aren't | |
196 | run. |
|
187 | run. | |
197 |
|
188 | |||
198 | fast : bool, option |
|
189 | fast : bool, option | |
199 | Run the test suite in parallel, if True, using as many threads as there |
|
190 | Run the test suite in parallel, if True, using as many threads as there | |
200 | are processors |
|
191 | are processors | |
201 | """ |
|
192 | """ | |
202 | pool = multiprocessing.pool.ThreadPool(jobs) |
|
|||
203 | if jobs != 1: |
|
193 | if jobs != 1: | |
204 | IPTestController.buffer_output = True |
|
194 | IPTestController.buffer_output = True | |
205 |
|
195 | |||
206 | controllers = test_controllers_to_run(inc_slow=inc_slow) |
|
196 | controllers = test_controllers_to_run(inc_slow=inc_slow) | |
207 |
|
197 | |||
208 | # Run all test runners, tracking execution time |
|
198 | # Run all test runners, tracking execution time | |
209 | failed = [] |
|
199 | failed = [] | |
210 | t_start = time.time() |
|
200 | t_start = time.time() | |
211 |
|
201 | |||
212 | print('*'*70) |
|
202 | print('*'*70) | |
|
203 | if jobs == 1: | |||
|
204 | for controller in controllers: | |||
|
205 | print('IPython test group:', controller.section) | |||
|
206 | controller, res = do_run(controller) | |||
|
207 | if res: | |||
|
208 | failed.append(controller) | |||
|
209 | if res == -signal.SIGINT: | |||
|
210 | print("Interrupted") | |||
|
211 | break | |||
|
212 | print() | |||
|
213 | ||||
|
214 | else: | |||
|
215 | try: | |||
|
216 | pool = multiprocessing.pool.ThreadPool(jobs) | |||
213 | for (controller, res) in pool.imap_unordered(do_run, controllers): |
|
217 | for (controller, res) in pool.imap_unordered(do_run, controllers): | |
214 | tgroup = 'IPython test group: ' + controller.section |
|
218 | tgroup = 'IPython test group: ' + controller.section + ' ' | |
215 | res_string = 'OK' if res == 0 else 'FAILED' |
|
219 | res_string = ' OK' if res == 0 else ' FAILED' | |
216 | res_string = res_string.rjust(70 - len(tgroup), '.') |
|
220 | res_string = res_string.rjust(70 - len(tgroup), '.') | |
217 | print(tgroup + res_string) |
|
221 | print(tgroup + res_string) | |
218 | if res: |
|
222 | if res: | |
|
223 | print(controller.stdout) | |||
219 | failed.append(controller) |
|
224 | failed.append(controller) | |
220 | if res == -signal.SIGINT: |
|
225 | if res == -signal.SIGINT: | |
221 | print("Interrupted") |
|
226 | print("Interrupted") | |
222 | break |
|
227 | break | |
|
228 | except KeyboardInterrupt: | |||
|
229 | return | |||
223 |
|
230 | |||
224 | t_end = time.time() |
|
231 | t_end = time.time() | |
225 | t_tests = t_end - t_start |
|
232 | t_tests = t_end - t_start | |
226 | nrunners = len(controllers) |
|
233 | nrunners = len(controllers) | |
227 | nfail = len(failed) |
|
234 | nfail = len(failed) | |
228 | # summarize results |
|
235 | # summarize results | |
229 | print() |
|
|||
230 | print('*'*70) |
|
236 | print('*'*70) | |
231 | print('Test suite completed for system with the following information:') |
|
237 | print('Test suite completed for system with the following information:') | |
232 | print(report()) |
|
238 | print(report()) | |
233 | print('Ran %s test groups in %.3fs' % (nrunners, t_tests)) |
|
239 | print('Ran %s test groups in %.3fs' % (nrunners, t_tests)) | |
234 | print() |
|
240 | print() | |
235 | print('Status:') |
|
241 | print('Status:') | |
236 | if not failed: |
|
242 | if not failed: | |
237 | print('OK') |
|
243 | print('OK') | |
238 | else: |
|
244 | else: | |
239 | # If anything went wrong, point out what command to rerun manually to |
|
245 | # If anything went wrong, point out what command to rerun manually to | |
240 | # see the actual errors and individual summary |
|
246 | # see the actual errors and individual summary | |
241 | print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners)) |
|
247 | print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners)) | |
242 | for controller in failed: |
|
248 | for controller in failed: | |
243 | print('-'*40) |
|
249 | print('-'*40) | |
244 | print('Runner failed:', controller.section) |
|
250 | print('Runner failed:', controller.section) | |
245 | print('You may wish to rerun this one individually, with:') |
|
251 | print('You may wish to rerun this one individually, with:') | |
246 | failed_call_args = [py3compat.cast_unicode(x) for x in controller.cmd] |
|
252 | failed_call_args = [py3compat.cast_unicode(x) for x in controller.cmd] | |
247 | print(u' '.join(failed_call_args)) |
|
253 | print(u' '.join(failed_call_args)) | |
248 | print() |
|
254 | print() | |
249 | # Ensure that our exit code indicates failure |
|
255 | # Ensure that our exit code indicates failure | |
250 | sys.exit(1) |
|
256 | sys.exit(1) | |
251 |
|
257 | |||
252 |
|
258 | |||
253 | def main(): |
|
259 | def main(): | |
254 | if len(sys.argv) > 1 and (sys.argv[1] in test_sections): |
|
260 | if len(sys.argv) > 1 and (sys.argv[1] in test_sections): | |
255 | from .iptest import run_iptest |
|
261 | from .iptest import run_iptest | |
256 | # This is in-process |
|
262 | # This is in-process | |
257 | run_iptest() |
|
263 | run_iptest() | |
258 | return |
|
264 | return | |
259 |
|
265 | |||
260 | parser = argparse.ArgumentParser(description='Run IPython test suite') |
|
266 | parser = argparse.ArgumentParser(description='Run IPython test suite') | |
261 | parser.add_argument('--all', action='store_true', |
|
267 | parser.add_argument('--all', action='store_true', | |
262 | help='Include slow tests not run by default.') |
|
268 | help='Include slow tests not run by default.') | |
263 | parser.add_argument('-j', '--fast', nargs='?', const=None, default=1, |
|
269 | parser.add_argument('-j', '--fast', nargs='?', const=None, default=1, | |
264 | help='Run test sections in parallel.') |
|
270 | help='Run test sections in parallel.') | |
265 | parser.add_argument('--xunit', action='store_true', |
|
271 | parser.add_argument('--xunit', action='store_true', | |
266 | help='Produce Xunit XML results') |
|
272 | help='Produce Xunit XML results') | |
267 | parser.add_argument('--coverage', action='store_true', |
|
273 | parser.add_argument('--coverage', action='store_true', | |
268 | help='Measure test coverage.') |
|
274 | help='Measure test coverage.') | |
269 |
|
275 | |||
270 | options = parser.parse_args() |
|
276 | options = parser.parse_args() | |
271 |
|
277 | |||
272 | # This starts subprocesses |
|
278 | # This starts subprocesses | |
273 | run_iptestall(inc_slow=options.all, jobs=options.fast, |
|
279 | run_iptestall(inc_slow=options.all, jobs=options.fast, | |
274 | xunit=options.xunit, coverage=options.coverage) |
|
280 | xunit=options.xunit, coverage=options.coverage) | |
275 |
|
281 | |||
276 |
|
282 | |||
277 | if __name__ == '__main__': |
|
283 | if __name__ == '__main__': | |
278 | main() |
|
284 | main() |
General Comments 0
You need to be logged in to leave comments.
Login now