##// END OF EJS Templates
Don't use fork to start the notebook in js tests...
MinRK -
Show More
@@ -1,569 +1,595 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Process Controller
2 """IPython Test Process Controller
3
3
4 This module runs one or more subprocesses which will actually run the IPython
4 This module runs one or more subprocesses which will actually run the IPython
5 test suite.
5 test suite.
6
6
7 """
7 """
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2009-2011 The IPython Development Team
10 # Copyright (C) 2009-2011 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 from __future__ import print_function
19 from __future__ import print_function
20
20
21 import argparse
21 import argparse
22 import json
22 import multiprocessing.pool
23 import multiprocessing.pool
23 from multiprocessing import Process, Queue
24 import os
24 import os
25 import shutil
25 import shutil
26 import signal
26 import signal
27 import sys
27 import sys
28 import subprocess
28 import subprocess
29 import time
29 import time
30
30
31 from .iptest import have, test_group_names as py_test_group_names, test_sections
31 from .iptest import have, test_group_names as py_test_group_names, test_sections, StreamCapturer
32 from IPython.utils.path import compress_user
32 from IPython.utils.path import compress_user
33 from IPython.utils.py3compat import bytes_to_str
33 from IPython.utils.py3compat import bytes_to_str
34 from IPython.utils.sysinfo import get_sys_info
34 from IPython.utils.sysinfo import get_sys_info
35 from IPython.utils.tempdir import TemporaryDirectory
35 from IPython.utils.tempdir import TemporaryDirectory
36
36
37
37
38 class TestController(object):
38 class TestController(object):
39 """Run tests in a subprocess
39 """Run tests in a subprocess
40 """
40 """
41 #: str, IPython test suite to be executed.
41 #: str, IPython test suite to be executed.
42 section = None
42 section = None
43 #: list, command line arguments to be executed
43 #: list, command line arguments to be executed
44 cmd = None
44 cmd = None
45 #: dict, extra environment variables to set for the subprocess
45 #: dict, extra environment variables to set for the subprocess
46 env = None
46 env = None
47 #: list, TemporaryDirectory instances to clear up when the process finishes
47 #: list, TemporaryDirectory instances to clear up when the process finishes
48 dirs = None
48 dirs = None
49 #: subprocess.Popen instance
49 #: subprocess.Popen instance
50 process = None
50 process = None
51 #: str, process stdout+stderr
51 #: str, process stdout+stderr
52 stdout = None
52 stdout = None
53
53
54 def __init__(self):
54 def __init__(self):
55 self.cmd = []
55 self.cmd = []
56 self.env = {}
56 self.env = {}
57 self.dirs = []
57 self.dirs = []
58
58
59 def setup(self):
59 def setup(self):
60 """Create temporary directories etc.
60 """Create temporary directories etc.
61
61
62 This is only called when we know the test group will be run. Things
62 This is only called when we know the test group will be run. Things
63 created here may be cleaned up by self.cleanup().
63 created here may be cleaned up by self.cleanup().
64 """
64 """
65 pass
65 pass
66
66
67 def launch(self, buffer_output=False):
67 def launch(self, buffer_output=False):
68 # print('*** ENV:', self.env) # dbg
68 # print('*** ENV:', self.env) # dbg
69 # print('*** CMD:', self.cmd) # dbg
69 # print('*** CMD:', self.cmd) # dbg
70 env = os.environ.copy()
70 env = os.environ.copy()
71 env.update(self.env)
71 env.update(self.env)
72 output = subprocess.PIPE if buffer_output else None
72 output = subprocess.PIPE if buffer_output else None
73 stdout = subprocess.STDOUT if buffer_output else None
73 stdout = subprocess.STDOUT if buffer_output else None
74 self.process = subprocess.Popen(self.cmd, stdout=output,
74 self.process = subprocess.Popen(self.cmd, stdout=output,
75 stderr=stdout, env=env)
75 stderr=stdout, env=env)
76
76
77 def wait(self):
77 def wait(self):
78 self.stdout, _ = self.process.communicate()
78 self.stdout, _ = self.process.communicate()
79 return self.process.returncode
79 return self.process.returncode
80
80
81 def print_extra_info(self):
81 def print_extra_info(self):
82 """Print extra information about this test run.
82 """Print extra information about this test run.
83
83
84 If we're running in parallel and showing the concise view, this is only
84 If we're running in parallel and showing the concise view, this is only
85 called if the test group fails. Otherwise, it's called before the test
85 called if the test group fails. Otherwise, it's called before the test
86 group is started.
86 group is started.
87
87
88 The base implementation does nothing, but it can be overridden by
88 The base implementation does nothing, but it can be overridden by
89 subclasses.
89 subclasses.
90 """
90 """
91 return
91 return
92
92
93 def cleanup_process(self):
93 def cleanup_process(self):
94 """Cleanup on exit by killing any leftover processes."""
94 """Cleanup on exit by killing any leftover processes."""
95 subp = self.process
95 subp = self.process
96 if subp is None or (subp.poll() is not None):
96 if subp is None or (subp.poll() is not None):
97 return # Process doesn't exist, or is already dead.
97 return # Process doesn't exist, or is already dead.
98
98
99 try:
99 try:
100 print('Cleaning up stale PID: %d' % subp.pid)
100 print('Cleaning up stale PID: %d' % subp.pid)
101 subp.kill()
101 subp.kill()
102 except: # (OSError, WindowsError) ?
102 except: # (OSError, WindowsError) ?
103 # This is just a best effort, if we fail or the process was
103 # This is just a best effort, if we fail or the process was
104 # really gone, ignore it.
104 # really gone, ignore it.
105 pass
105 pass
106 else:
106 else:
107 for i in range(10):
107 for i in range(10):
108 if subp.poll() is None:
108 if subp.poll() is None:
109 time.sleep(0.1)
109 time.sleep(0.1)
110 else:
110 else:
111 break
111 break
112
112
113 if subp.poll() is None:
113 if subp.poll() is None:
114 # The process did not die...
114 # The process did not die...
115 print('... failed. Manual cleanup may be required.')
115 print('... failed. Manual cleanup may be required.')
116
116
117 def cleanup(self):
117 def cleanup(self):
118 "Kill process if it's still alive, and clean up temporary directories"
118 "Kill process if it's still alive, and clean up temporary directories"
119 self.cleanup_process()
119 self.cleanup_process()
120 for td in self.dirs:
120 for td in self.dirs:
121 td.cleanup()
121 td.cleanup()
122
122
123 __del__ = cleanup
123 __del__ = cleanup
124
124
125 class PyTestController(TestController):
125 class PyTestController(TestController):
126 """Run Python tests using IPython.testing.iptest"""
126 """Run Python tests using IPython.testing.iptest"""
127 #: str, Python command to execute in subprocess
127 #: str, Python command to execute in subprocess
128 pycmd = None
128 pycmd = None
129
129
130 def __init__(self, section):
130 def __init__(self, section):
131 """Create new test runner."""
131 """Create new test runner."""
132 TestController.__init__(self)
132 TestController.__init__(self)
133 self.section = section
133 self.section = section
134 # pycmd is put into cmd[2] in PyTestController.launch()
134 # pycmd is put into cmd[2] in PyTestController.launch()
135 self.cmd = [sys.executable, '-c', None, section]
135 self.cmd = [sys.executable, '-c', None, section]
136 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
136 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
137
137
138 def setup(self):
138 def setup(self):
139 ipydir = TemporaryDirectory()
139 ipydir = TemporaryDirectory()
140 self.dirs.append(ipydir)
140 self.dirs.append(ipydir)
141 self.env['IPYTHONDIR'] = ipydir.name
141 self.env['IPYTHONDIR'] = ipydir.name
142 self.workingdir = workingdir = TemporaryDirectory()
142 self.workingdir = workingdir = TemporaryDirectory()
143 self.dirs.append(workingdir)
143 self.dirs.append(workingdir)
144 self.env['IPTEST_WORKING_DIR'] = workingdir.name
144 self.env['IPTEST_WORKING_DIR'] = workingdir.name
145 # This means we won't get odd effects from our own matplotlib config
145 # This means we won't get odd effects from our own matplotlib config
146 self.env['MPLCONFIGDIR'] = workingdir.name
146 self.env['MPLCONFIGDIR'] = workingdir.name
147
147
148 @property
148 @property
149 def will_run(self):
149 def will_run(self):
150 try:
150 try:
151 return test_sections[self.section].will_run
151 return test_sections[self.section].will_run
152 except KeyError:
152 except KeyError:
153 return True
153 return True
154
154
155 def add_xunit(self):
155 def add_xunit(self):
156 xunit_file = os.path.abspath(self.section + '.xunit.xml')
156 xunit_file = os.path.abspath(self.section + '.xunit.xml')
157 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
157 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
158
158
159 def add_coverage(self):
159 def add_coverage(self):
160 try:
160 try:
161 sources = test_sections[self.section].includes
161 sources = test_sections[self.section].includes
162 except KeyError:
162 except KeyError:
163 sources = ['IPython']
163 sources = ['IPython']
164
164
165 coverage_rc = ("[run]\n"
165 coverage_rc = ("[run]\n"
166 "data_file = {data_file}\n"
166 "data_file = {data_file}\n"
167 "source =\n"
167 "source =\n"
168 " {source}\n"
168 " {source}\n"
169 ).format(data_file=os.path.abspath('.coverage.'+self.section),
169 ).format(data_file=os.path.abspath('.coverage.'+self.section),
170 source="\n ".join(sources))
170 source="\n ".join(sources))
171 config_file = os.path.join(self.workingdir.name, '.coveragerc')
171 config_file = os.path.join(self.workingdir.name, '.coveragerc')
172 with open(config_file, 'w') as f:
172 with open(config_file, 'w') as f:
173 f.write(coverage_rc)
173 f.write(coverage_rc)
174
174
175 self.env['COVERAGE_PROCESS_START'] = config_file
175 self.env['COVERAGE_PROCESS_START'] = config_file
176 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
176 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
177
177
178 def launch(self, buffer_output=False):
178 def launch(self, buffer_output=False):
179 self.cmd[2] = self.pycmd
179 self.cmd[2] = self.pycmd
180 super(PyTestController, self).launch(buffer_output=buffer_output)
180 super(PyTestController, self).launch(buffer_output=buffer_output)
181
181
182 js_prefix = 'js/'
182 js_prefix = 'js/'
183
183
184 def get_js_test_dir():
184 def get_js_test_dir():
185 import IPython.html.tests as t
185 import IPython.html.tests as t
186 return os.path.join(os.path.dirname(t.__file__), '')
186 return os.path.join(os.path.dirname(t.__file__), '')
187
187
188 def all_js_groups():
188 def all_js_groups():
189 import glob
189 import glob
190 test_dir = get_js_test_dir()
190 test_dir = get_js_test_dir()
191 all_subdirs = glob.glob(test_dir + '*/')
191 all_subdirs = glob.glob(test_dir + '*/')
192 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs if os.path.relpath(x, test_dir) != '__pycache__']
192 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs if os.path.relpath(x, test_dir) != '__pycache__']
193
193
194 class JSController(TestController):
194 class JSController(TestController):
195 """Run CasperJS tests """
195 """Run CasperJS tests """
196 def __init__(self, section):
196 def __init__(self, section):
197 """Create new test runner."""
197 """Create new test runner."""
198 TestController.__init__(self)
198 TestController.__init__(self)
199 self.section = section
199 self.section = section
200 js_test_dir = get_js_test_dir()
200 js_test_dir = get_js_test_dir()
201 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
201 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
202 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
202 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
203 self.cmd = ['casperjs', 'test', includes, test_cases]
203 self.cmd = ['casperjs', 'test', includes, test_cases]
204
204
205 def setup(self):
205 def setup(self):
206 self.ipydir = TemporaryDirectory()
206 self.ipydir = TemporaryDirectory()
207 self.nbdir = TemporaryDirectory()
207 self.nbdir = TemporaryDirectory()
208 self.dirs.append(self.ipydir)
208 self.dirs.append(self.ipydir)
209 self.dirs.append(self.nbdir)
209 self.dirs.append(self.nbdir)
210 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
210 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
211 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
211 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
212
212
213 # start the ipython notebook, so we get the port number
213 # start the ipython notebook, so we get the port number
214 self.server_port = 0
214 self._init_server()
215 self._init_server()
215 self.cmd.append('--port=%s' % self.server_port)
216 if self.server_port:
217 self.cmd.append("--port=%i" % self.server_port)
216
218
217 def print_extra_info(self):
219 def print_extra_info(self):
218 print("Running tests with notebook directory %r" % self.nbdir.name)
220 print("Running tests with notebook directory %r" % self.nbdir.name)
219
221
220 @property
222 @property
221 def will_run(self):
223 def will_run(self):
222 return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs'])
224 return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs'])
223
225
224 def _init_server(self):
226 def _init_server(self):
225 "Start the notebook server in a separate process"
227 "Start the notebook server in a separate process"
226 self.queue = q = Queue()
228 self.server_command = command = [sys.executable,
227 self.server = Process(target=run_webapp, args=(q, self.ipydir.name, self.nbdir.name))
229 '-m', 'IPython.html',
228 self.server.start()
230 '--no-browser',
229 self.server_port = q.get()
231 '--ipython-dir', self.ipydir.name,
232 '--notebook-dir', self.nbdir.name,
233 ]
234 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
235 # which run afoul of ipc's maximum path length.
236 if sys.platform.startswith('linux'):
237 command.append('--KernelManager.transport=ipc')
238 self.stream_capturer = c = StreamCapturer()
239 c.start()
240 self.server = subprocess.Popen(command, stdout=c.writefd, stderr=subprocess.STDOUT)
241 self.server_info_file = os.path.join(self.ipydir.name,
242 'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
243 )
244 self._wait_for_server()
245
246 def _wait_for_server(self):
247 """Wait 30 seconds for the notebook server to start"""
248 for i in range(300):
249 if self.server.poll() is not None:
250 return self._failed_to_start()
251 if os.path.exists(self.server_info_file):
252 self._load_server_info()
253 return
254 time.sleep(0.1)
255 print("Notebook server-info file never arrived: %s" % self.server_info_file,
256 file=sys.stderr
257 )
258
259 def _failed_to_start(self):
260 """Notebook server exited prematurely"""
261 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
262 print("Notebook failed to start: ", file=sys.stderr)
263 print(self.server_command)
264 print(captured, file=sys.stderr)
265
266 def _load_server_info(self):
267 """Notebook server started, load connection info from JSON"""
268 with open(self.server_info_file) as f:
269 info = json.load(f)
270 self.server_port = info['port']
230
271
231 def cleanup(self):
272 def cleanup(self):
232 self.server.terminate()
273 self.stream_capturer.halt()
233 self.server.join()
274 try:
275 self.server.terminate()
276 except OSError:
277 # already dead
278 pass
279 self.server.wait()
234 TestController.cleanup(self)
280 TestController.cleanup(self)
235
281
236 def run_webapp(q, ipydir, nbdir, loglevel=0):
237 """start the IPython Notebook, and pass port back to the queue"""
238 import os
239 import IPython.html.notebookapp as nbapp
240 import sys
241 sys.stderr = open(os.devnull, 'w')
242 server = nbapp.NotebookApp()
243 args = ['--no-browser']
244 args.extend(['--ipython-dir', ipydir,
245 '--notebook-dir', nbdir,
246 '--log-level', str(loglevel),
247 ])
248 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
249 # which run afoul of ipc's maximum path length.
250 if sys.platform.startswith('linux'):
251 args.append('--KernelManager.transport=ipc')
252 server.initialize(args)
253 # communicate the port number to the parent process
254 q.put(server.port)
255 server.start()
256
282
257 def prepare_controllers(options):
283 def prepare_controllers(options):
258 """Returns two lists of TestController instances, those to run, and those
284 """Returns two lists of TestController instances, those to run, and those
259 not to run."""
285 not to run."""
260 testgroups = options.testgroups
286 testgroups = options.testgroups
261
287
262 if testgroups:
288 if testgroups:
263 py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
289 py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
264 or g.startswith('IPython.')]
290 or g.startswith('IPython.')]
265 if 'js' in testgroups:
291 if 'js' in testgroups:
266 js_testgroups = all_js_groups()
292 js_testgroups = all_js_groups()
267 else:
293 else:
268 js_testgroups = [g for g in testgroups if g not in py_testgroups]
294 js_testgroups = [g for g in testgroups if g not in py_testgroups]
269 else:
295 else:
270 py_testgroups = py_test_group_names
296 py_testgroups = py_test_group_names
271 js_testgroups = all_js_groups()
297 js_testgroups = all_js_groups()
272 if not options.all:
298 if not options.all:
273 test_sections['parallel'].enabled = False
299 test_sections['parallel'].enabled = False
274
300
275 c_js = [JSController(name) for name in js_testgroups]
301 c_js = [JSController(name) for name in js_testgroups]
276 c_py = [PyTestController(name) for name in py_testgroups]
302 c_py = [PyTestController(name) for name in py_testgroups]
277
303
278 configure_py_controllers(c_py, xunit=options.xunit,
304 configure_py_controllers(c_py, xunit=options.xunit,
279 coverage=options.coverage, subproc_streams=options.subproc_streams,
305 coverage=options.coverage, subproc_streams=options.subproc_streams,
280 extra_args=options.extra_args)
306 extra_args=options.extra_args)
281
307
282 controllers = c_py + c_js
308 controllers = c_py + c_js
283 to_run = [c for c in controllers if c.will_run]
309 to_run = [c for c in controllers if c.will_run]
284 not_run = [c for c in controllers if not c.will_run]
310 not_run = [c for c in controllers if not c.will_run]
285 return to_run, not_run
311 return to_run, not_run
286
312
287 def configure_py_controllers(controllers, xunit=False, coverage=False,
313 def configure_py_controllers(controllers, xunit=False, coverage=False,
288 subproc_streams='capture', extra_args=()):
314 subproc_streams='capture', extra_args=()):
289 """Apply options for a collection of TestController objects."""
315 """Apply options for a collection of TestController objects."""
290 for controller in controllers:
316 for controller in controllers:
291 if xunit:
317 if xunit:
292 controller.add_xunit()
318 controller.add_xunit()
293 if coverage:
319 if coverage:
294 controller.add_coverage()
320 controller.add_coverage()
295 controller.env['IPTEST_SUBPROC_STREAMS'] = subproc_streams
321 controller.env['IPTEST_SUBPROC_STREAMS'] = subproc_streams
296 controller.cmd.extend(extra_args)
322 controller.cmd.extend(extra_args)
297
323
298 def do_run(controller, buffer_output=True):
324 def do_run(controller, buffer_output=True):
299 """Setup and run a test controller.
325 """Setup and run a test controller.
300
326
301 If buffer_output is True, no output is displayed, to avoid it appearing
327 If buffer_output is True, no output is displayed, to avoid it appearing
302 interleaved. In this case, the caller is responsible for displaying test
328 interleaved. In this case, the caller is responsible for displaying test
303 output on failure.
329 output on failure.
304
330
305 Returns
331 Returns
306 -------
332 -------
307 controller : TestController
333 controller : TestController
308 The same controller as passed in, as a convenience for using map() type
334 The same controller as passed in, as a convenience for using map() type
309 APIs.
335 APIs.
310 exitcode : int
336 exitcode : int
311 The exit code of the test subprocess. Non-zero indicates failure.
337 The exit code of the test subprocess. Non-zero indicates failure.
312 """
338 """
313 try:
339 try:
314 try:
340 try:
315 controller.setup()
341 controller.setup()
316 if not buffer_output:
342 if not buffer_output:
317 controller.print_extra_info()
343 controller.print_extra_info()
318 controller.launch(buffer_output=buffer_output)
344 controller.launch(buffer_output=buffer_output)
319 except Exception:
345 except Exception:
320 import traceback
346 import traceback
321 traceback.print_exc()
347 traceback.print_exc()
322 return controller, 1 # signal failure
348 return controller, 1 # signal failure
323
349
324 exitcode = controller.wait()
350 exitcode = controller.wait()
325 return controller, exitcode
351 return controller, exitcode
326
352
327 except KeyboardInterrupt:
353 except KeyboardInterrupt:
328 return controller, -signal.SIGINT
354 return controller, -signal.SIGINT
329 finally:
355 finally:
330 controller.cleanup()
356 controller.cleanup()
331
357
332 def report():
358 def report():
333 """Return a string with a summary report of test-related variables."""
359 """Return a string with a summary report of test-related variables."""
334 inf = get_sys_info()
360 inf = get_sys_info()
335 out = []
361 out = []
336 def _add(name, value):
362 def _add(name, value):
337 out.append((name, value))
363 out.append((name, value))
338
364
339 _add('IPython version', inf['ipython_version'])
365 _add('IPython version', inf['ipython_version'])
340 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
366 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
341 _add('IPython package', compress_user(inf['ipython_path']))
367 _add('IPython package', compress_user(inf['ipython_path']))
342 _add('Python version', inf['sys_version'].replace('\n',''))
368 _add('Python version', inf['sys_version'].replace('\n',''))
343 _add('sys.executable', compress_user(inf['sys_executable']))
369 _add('sys.executable', compress_user(inf['sys_executable']))
344 _add('Platform', inf['platform'])
370 _add('Platform', inf['platform'])
345
371
346 width = max(len(n) for (n,v) in out)
372 width = max(len(n) for (n,v) in out)
347 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
373 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
348
374
349 avail = []
375 avail = []
350 not_avail = []
376 not_avail = []
351
377
352 for k, is_avail in have.items():
378 for k, is_avail in have.items():
353 if is_avail:
379 if is_avail:
354 avail.append(k)
380 avail.append(k)
355 else:
381 else:
356 not_avail.append(k)
382 not_avail.append(k)
357
383
358 if avail:
384 if avail:
359 out.append('\nTools and libraries available at test time:\n')
385 out.append('\nTools and libraries available at test time:\n')
360 avail.sort()
386 avail.sort()
361 out.append(' ' + ' '.join(avail)+'\n')
387 out.append(' ' + ' '.join(avail)+'\n')
362
388
363 if not_avail:
389 if not_avail:
364 out.append('\nTools and libraries NOT available at test time:\n')
390 out.append('\nTools and libraries NOT available at test time:\n')
365 not_avail.sort()
391 not_avail.sort()
366 out.append(' ' + ' '.join(not_avail)+'\n')
392 out.append(' ' + ' '.join(not_avail)+'\n')
367
393
368 return ''.join(out)
394 return ''.join(out)
369
395
370 def run_iptestall(options):
396 def run_iptestall(options):
371 """Run the entire IPython test suite by calling nose and trial.
397 """Run the entire IPython test suite by calling nose and trial.
372
398
373 This function constructs :class:`IPTester` instances for all IPython
399 This function constructs :class:`IPTester` instances for all IPython
374 modules and package and then runs each of them. This causes the modules
400 modules and package and then runs each of them. This causes the modules
375 and packages of IPython to be tested each in their own subprocess using
401 and packages of IPython to be tested each in their own subprocess using
376 nose.
402 nose.
377
403
378 Parameters
404 Parameters
379 ----------
405 ----------
380
406
381 All parameters are passed as attributes of the options object.
407 All parameters are passed as attributes of the options object.
382
408
383 testgroups : list of str
409 testgroups : list of str
384 Run only these sections of the test suite. If empty, run all the available
410 Run only these sections of the test suite. If empty, run all the available
385 sections.
411 sections.
386
412
387 fast : int or None
413 fast : int or None
388 Run the test suite in parallel, using n simultaneous processes. If None
414 Run the test suite in parallel, using n simultaneous processes. If None
389 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
415 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
390
416
391 inc_slow : bool
417 inc_slow : bool
392 Include slow tests, like IPython.parallel. By default, these tests aren't
418 Include slow tests, like IPython.parallel. By default, these tests aren't
393 run.
419 run.
394
420
395 xunit : bool
421 xunit : bool
396 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
422 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
397
423
398 coverage : bool or str
424 coverage : bool or str
399 Measure code coverage from tests. True will store the raw coverage data,
425 Measure code coverage from tests. True will store the raw coverage data,
400 or pass 'html' or 'xml' to get reports.
426 or pass 'html' or 'xml' to get reports.
401
427
402 extra_args : list
428 extra_args : list
403 Extra arguments to pass to the test subprocesses, e.g. '-v'
429 Extra arguments to pass to the test subprocesses, e.g. '-v'
404 """
430 """
405 to_run, not_run = prepare_controllers(options)
431 to_run, not_run = prepare_controllers(options)
406
432
407 def justify(ltext, rtext, width=70, fill='-'):
433 def justify(ltext, rtext, width=70, fill='-'):
408 ltext += ' '
434 ltext += ' '
409 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
435 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
410 return ltext + rtext
436 return ltext + rtext
411
437
412 # Run all test runners, tracking execution time
438 # Run all test runners, tracking execution time
413 failed = []
439 failed = []
414 t_start = time.time()
440 t_start = time.time()
415
441
416 print()
442 print()
417 if options.fast == 1:
443 if options.fast == 1:
418 # This actually means sequential, i.e. with 1 job
444 # This actually means sequential, i.e. with 1 job
419 for controller in to_run:
445 for controller in to_run:
420 print('Test group:', controller.section)
446 print('Test group:', controller.section)
421 sys.stdout.flush() # Show in correct order when output is piped
447 sys.stdout.flush() # Show in correct order when output is piped
422 controller, res = do_run(controller, buffer_output=False)
448 controller, res = do_run(controller, buffer_output=False)
423 if res:
449 if res:
424 failed.append(controller)
450 failed.append(controller)
425 if res == -signal.SIGINT:
451 if res == -signal.SIGINT:
426 print("Interrupted")
452 print("Interrupted")
427 break
453 break
428 print()
454 print()
429
455
430 else:
456 else:
431 # Run tests concurrently
457 # Run tests concurrently
432 try:
458 try:
433 pool = multiprocessing.pool.ThreadPool(options.fast)
459 pool = multiprocessing.pool.ThreadPool(options.fast)
434 for (controller, res) in pool.imap_unordered(do_run, to_run):
460 for (controller, res) in pool.imap_unordered(do_run, to_run):
435 res_string = 'OK' if res == 0 else 'FAILED'
461 res_string = 'OK' if res == 0 else 'FAILED'
436 print(justify('Test group: ' + controller.section, res_string))
462 print(justify('Test group: ' + controller.section, res_string))
437 if res:
463 if res:
438 controller.print_extra_info()
464 controller.print_extra_info()
439 print(bytes_to_str(controller.stdout))
465 print(bytes_to_str(controller.stdout))
440 failed.append(controller)
466 failed.append(controller)
441 if res == -signal.SIGINT:
467 if res == -signal.SIGINT:
442 print("Interrupted")
468 print("Interrupted")
443 break
469 break
444 except KeyboardInterrupt:
470 except KeyboardInterrupt:
445 return
471 return
446
472
447 for controller in not_run:
473 for controller in not_run:
448 print(justify('Test group: ' + controller.section, 'NOT RUN'))
474 print(justify('Test group: ' + controller.section, 'NOT RUN'))
449
475
450 t_end = time.time()
476 t_end = time.time()
451 t_tests = t_end - t_start
477 t_tests = t_end - t_start
452 nrunners = len(to_run)
478 nrunners = len(to_run)
453 nfail = len(failed)
479 nfail = len(failed)
454 # summarize results
480 # summarize results
455 print('_'*70)
481 print('_'*70)
456 print('Test suite completed for system with the following information:')
482 print('Test suite completed for system with the following information:')
457 print(report())
483 print(report())
458 took = "Took %.3fs." % t_tests
484 took = "Took %.3fs." % t_tests
459 print('Status: ', end='')
485 print('Status: ', end='')
460 if not failed:
486 if not failed:
461 print('OK (%d test groups).' % nrunners, took)
487 print('OK (%d test groups).' % nrunners, took)
462 else:
488 else:
463 # If anything went wrong, point out what command to rerun manually to
489 # If anything went wrong, point out what command to rerun manually to
464 # see the actual errors and individual summary
490 # see the actual errors and individual summary
465 failed_sections = [c.section for c in failed]
491 failed_sections = [c.section for c in failed]
466 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
492 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
467 nrunners, ', '.join(failed_sections)), took)
493 nrunners, ', '.join(failed_sections)), took)
468 print()
494 print()
469 print('You may wish to rerun these, with:')
495 print('You may wish to rerun these, with:')
470 print(' iptest', *failed_sections)
496 print(' iptest', *failed_sections)
471 print()
497 print()
472
498
473 if options.coverage:
499 if options.coverage:
474 from coverage import coverage
500 from coverage import coverage
475 cov = coverage(data_file='.coverage')
501 cov = coverage(data_file='.coverage')
476 cov.combine()
502 cov.combine()
477 cov.save()
503 cov.save()
478
504
479 # Coverage HTML report
505 # Coverage HTML report
480 if options.coverage == 'html':
506 if options.coverage == 'html':
481 html_dir = 'ipy_htmlcov'
507 html_dir = 'ipy_htmlcov'
482 shutil.rmtree(html_dir, ignore_errors=True)
508 shutil.rmtree(html_dir, ignore_errors=True)
483 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
509 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
484 sys.stdout.flush()
510 sys.stdout.flush()
485
511
486 # Custom HTML reporter to clean up module names.
512 # Custom HTML reporter to clean up module names.
487 from coverage.html import HtmlReporter
513 from coverage.html import HtmlReporter
488 class CustomHtmlReporter(HtmlReporter):
514 class CustomHtmlReporter(HtmlReporter):
489 def find_code_units(self, morfs):
515 def find_code_units(self, morfs):
490 super(CustomHtmlReporter, self).find_code_units(morfs)
516 super(CustomHtmlReporter, self).find_code_units(morfs)
491 for cu in self.code_units:
517 for cu in self.code_units:
492 nameparts = cu.name.split(os.sep)
518 nameparts = cu.name.split(os.sep)
493 if 'IPython' not in nameparts:
519 if 'IPython' not in nameparts:
494 continue
520 continue
495 ix = nameparts.index('IPython')
521 ix = nameparts.index('IPython')
496 cu.name = '.'.join(nameparts[ix:])
522 cu.name = '.'.join(nameparts[ix:])
497
523
498 # Reimplement the html_report method with our custom reporter
524 # Reimplement the html_report method with our custom reporter
499 cov._harvest_data()
525 cov._harvest_data()
500 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
526 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
501 html_title='IPython test coverage',
527 html_title='IPython test coverage',
502 )
528 )
503 reporter = CustomHtmlReporter(cov, cov.config)
529 reporter = CustomHtmlReporter(cov, cov.config)
504 reporter.report(None)
530 reporter.report(None)
505 print('done.')
531 print('done.')
506
532
507 # Coverage XML report
533 # Coverage XML report
508 elif options.coverage == 'xml':
534 elif options.coverage == 'xml':
509 cov.xml_report(outfile='ipy_coverage.xml')
535 cov.xml_report(outfile='ipy_coverage.xml')
510
536
511 if failed:
537 if failed:
512 # Ensure that our exit code indicates failure
538 # Ensure that our exit code indicates failure
513 sys.exit(1)
539 sys.exit(1)
514
540
515 argparser = argparse.ArgumentParser(description='Run IPython test suite')
541 argparser = argparse.ArgumentParser(description='Run IPython test suite')
516 argparser.add_argument('testgroups', nargs='*',
542 argparser.add_argument('testgroups', nargs='*',
517 help='Run specified groups of tests. If omitted, run '
543 help='Run specified groups of tests. If omitted, run '
518 'all tests.')
544 'all tests.')
519 argparser.add_argument('--all', action='store_true',
545 argparser.add_argument('--all', action='store_true',
520 help='Include slow tests not run by default.')
546 help='Include slow tests not run by default.')
521 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
547 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
522 help='Run test sections in parallel. This starts as many '
548 help='Run test sections in parallel. This starts as many '
523 'processes as you have cores, or you can specify a number.')
549 'processes as you have cores, or you can specify a number.')
524 argparser.add_argument('--xunit', action='store_true',
550 argparser.add_argument('--xunit', action='store_true',
525 help='Produce Xunit XML results')
551 help='Produce Xunit XML results')
526 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
552 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
527 help="Measure test coverage. Specify 'html' or "
553 help="Measure test coverage. Specify 'html' or "
528 "'xml' to get reports.")
554 "'xml' to get reports.")
529 argparser.add_argument('--subproc-streams', default='capture',
555 argparser.add_argument('--subproc-streams', default='capture',
530 help="What to do with stdout/stderr from subprocesses. "
556 help="What to do with stdout/stderr from subprocesses. "
531 "'capture' (default), 'show' and 'discard' are the options.")
557 "'capture' (default), 'show' and 'discard' are the options.")
532
558
533 def default_options():
559 def default_options():
534 """Get an argparse Namespace object with the default arguments, to pass to
560 """Get an argparse Namespace object with the default arguments, to pass to
535 :func:`run_iptestall`.
561 :func:`run_iptestall`.
536 """
562 """
537 options = argparser.parse_args([])
563 options = argparser.parse_args([])
538 options.extra_args = []
564 options.extra_args = []
539 return options
565 return options
540
566
541 def main():
567 def main():
542 # iptest doesn't work correctly if the working directory is the
568 # iptest doesn't work correctly if the working directory is the
543 # root of the IPython source tree. Tell the user to avoid
569 # root of the IPython source tree. Tell the user to avoid
544 # frustration.
570 # frustration.
545 if os.path.exists(os.path.join(os.getcwd(),
571 if os.path.exists(os.path.join(os.getcwd(),
546 'IPython', 'testing', '__main__.py')):
572 'IPython', 'testing', '__main__.py')):
547 print("Don't run iptest from the IPython source directory",
573 print("Don't run iptest from the IPython source directory",
548 file=sys.stderr)
574 file=sys.stderr)
549 sys.exit(1)
575 sys.exit(1)
550 # Arguments after -- should be passed through to nose. Argparse treats
576 # Arguments after -- should be passed through to nose. Argparse treats
551 # everything after -- as regular positional arguments, so we separate them
577 # everything after -- as regular positional arguments, so we separate them
552 # first.
578 # first.
553 try:
579 try:
554 ix = sys.argv.index('--')
580 ix = sys.argv.index('--')
555 except ValueError:
581 except ValueError:
556 to_parse = sys.argv[1:]
582 to_parse = sys.argv[1:]
557 extra_args = []
583 extra_args = []
558 else:
584 else:
559 to_parse = sys.argv[1:ix]
585 to_parse = sys.argv[1:ix]
560 extra_args = sys.argv[ix+1:]
586 extra_args = sys.argv[ix+1:]
561
587
562 options = argparser.parse_args(to_parse)
588 options = argparser.parse_args(to_parse)
563 options.extra_args = extra_args
589 options.extra_args = extra_args
564
590
565 run_iptestall(options)
591 run_iptestall(options)
566
592
567
593
568 if __name__ == '__main__':
594 if __name__ == '__main__':
569 main()
595 main()
General Comments 0
You need to be logged in to leave comments. Login now