##// END OF EJS Templates
Merge pull request #5740 from minrk/travis-test-js...
Thomas Kluyver -
r16472:b2b3102f merge
parent child Browse files
Show More
@@ -1,18 +1,20 b''
1 # http://travis-ci.org/#!/ipython/ipython
1 # http://travis-ci.org/#!/ipython/ipython
2 language: python
2 language: python
3 python:
3 python:
4 - 2.7
4 - 2.7
5 - 3.3
5 - 3.3
6 env:
7 - GROUP=
8 - GROUP=js
6 before_install:
9 before_install:
7 # workaround for https://github.com/travis-ci/travis-cookbooks/issues/155
10 # workaround for https://github.com/travis-ci/travis-cookbooks/issues/155
8 - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
11 - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
9 - easy_install -q pyzmq
10 - pip install jinja2 sphinx pygments tornado requests mock
11 # Pierre Carrier's PPA for PhantomJS and CasperJS
12 # Pierre Carrier's PPA for PhantomJS and CasperJS
12 - sudo add-apt-repository -y ppa:pcarrier/ppa
13 - time sudo add-apt-repository -y ppa:pcarrier/ppa
13 - sudo apt-get update
14 - time sudo apt-get update
14 - sudo apt-get install pandoc casperjs nodejs
15 - time sudo apt-get install pandoc casperjs nodejs libzmq3-dev
16 - time pip install jinja2 sphinx pygments tornado requests mock pyzmq
15 install:
17 install:
16 - python setup.py install -q
18 - time python setup.py install -q
17 script:
19 script:
18 - cd /tmp && iptest
20 - cd /tmp && iptest $GROUP
@@ -1,592 +1,594 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Process Controller
2 """IPython Test Process Controller
3
3
4 This module runs one or more subprocesses which will actually run the IPython
4 This module runs one or more subprocesses which will actually run the IPython
5 test suite.
5 test suite.
6
6
7 """
7 """
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Copyright (C) 2009-2011 The IPython Development Team
10 # Copyright (C) 2009-2011 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 from __future__ import print_function
19 from __future__ import print_function
20
20
21 import argparse
21 import argparse
22 import json
22 import json
23 import multiprocessing.pool
23 import multiprocessing.pool
24 import os
24 import os
25 import shutil
25 import shutil
26 import signal
26 import signal
27 import sys
27 import sys
28 import subprocess
28 import subprocess
29 import time
29 import time
30
30
31 from .iptest import have, test_group_names as py_test_group_names, test_sections, StreamCapturer
31 from .iptest import have, test_group_names as py_test_group_names, test_sections, StreamCapturer
32 from IPython.utils.path import compress_user
32 from IPython.utils.path import compress_user
33 from IPython.utils.py3compat import bytes_to_str
33 from IPython.utils.py3compat import bytes_to_str
34 from IPython.utils.sysinfo import get_sys_info
34 from IPython.utils.sysinfo import get_sys_info
35 from IPython.utils.tempdir import TemporaryDirectory
35 from IPython.utils.tempdir import TemporaryDirectory
36
36
37
37
38 class TestController(object):
38 class TestController(object):
39 """Run tests in a subprocess
39 """Run tests in a subprocess
40 """
40 """
41 #: str, IPython test suite to be executed.
41 #: str, IPython test suite to be executed.
42 section = None
42 section = None
43 #: list, command line arguments to be executed
43 #: list, command line arguments to be executed
44 cmd = None
44 cmd = None
45 #: dict, extra environment variables to set for the subprocess
45 #: dict, extra environment variables to set for the subprocess
46 env = None
46 env = None
47 #: list, TemporaryDirectory instances to clear up when the process finishes
47 #: list, TemporaryDirectory instances to clear up when the process finishes
48 dirs = None
48 dirs = None
49 #: subprocess.Popen instance
49 #: subprocess.Popen instance
50 process = None
50 process = None
51 #: str, process stdout+stderr
51 #: str, process stdout+stderr
52 stdout = None
52 stdout = None
53
53
54 def __init__(self):
54 def __init__(self):
55 self.cmd = []
55 self.cmd = []
56 self.env = {}
56 self.env = {}
57 self.dirs = []
57 self.dirs = []
58
58
59 def setup(self):
59 def setup(self):
60 """Create temporary directories etc.
60 """Create temporary directories etc.
61
61
62 This is only called when we know the test group will be run. Things
62 This is only called when we know the test group will be run. Things
63 created here may be cleaned up by self.cleanup().
63 created here may be cleaned up by self.cleanup().
64 """
64 """
65 pass
65 pass
66
66
67 def launch(self, buffer_output=False):
67 def launch(self, buffer_output=False):
68 # print('*** ENV:', self.env) # dbg
68 # print('*** ENV:', self.env) # dbg
69 # print('*** CMD:', self.cmd) # dbg
69 # print('*** CMD:', self.cmd) # dbg
70 env = os.environ.copy()
70 env = os.environ.copy()
71 env.update(self.env)
71 env.update(self.env)
72 output = subprocess.PIPE if buffer_output else None
72 output = subprocess.PIPE if buffer_output else None
73 stdout = subprocess.STDOUT if buffer_output else None
73 stdout = subprocess.STDOUT if buffer_output else None
74 self.process = subprocess.Popen(self.cmd, stdout=output,
74 self.process = subprocess.Popen(self.cmd, stdout=output,
75 stderr=stdout, env=env)
75 stderr=stdout, env=env)
76
76
77 def wait(self):
77 def wait(self):
78 self.stdout, _ = self.process.communicate()
78 self.stdout, _ = self.process.communicate()
79 return self.process.returncode
79 return self.process.returncode
80
80
81 def print_extra_info(self):
81 def print_extra_info(self):
82 """Print extra information about this test run.
82 """Print extra information about this test run.
83
83
84 If we're running in parallel and showing the concise view, this is only
84 If we're running in parallel and showing the concise view, this is only
85 called if the test group fails. Otherwise, it's called before the test
85 called if the test group fails. Otherwise, it's called before the test
86 group is started.
86 group is started.
87
87
88 The base implementation does nothing, but it can be overridden by
88 The base implementation does nothing, but it can be overridden by
89 subclasses.
89 subclasses.
90 """
90 """
91 return
91 return
92
92
93 def cleanup_process(self):
93 def cleanup_process(self):
94 """Cleanup on exit by killing any leftover processes."""
94 """Cleanup on exit by killing any leftover processes."""
95 subp = self.process
95 subp = self.process
96 if subp is None or (subp.poll() is not None):
96 if subp is None or (subp.poll() is not None):
97 return # Process doesn't exist, or is already dead.
97 return # Process doesn't exist, or is already dead.
98
98
99 try:
99 try:
100 print('Cleaning up stale PID: %d' % subp.pid)
100 print('Cleaning up stale PID: %d' % subp.pid)
101 subp.kill()
101 subp.kill()
102 except: # (OSError, WindowsError) ?
102 except: # (OSError, WindowsError) ?
103 # This is just a best effort, if we fail or the process was
103 # This is just a best effort, if we fail or the process was
104 # really gone, ignore it.
104 # really gone, ignore it.
105 pass
105 pass
106 else:
106 else:
107 for i in range(10):
107 for i in range(10):
108 if subp.poll() is None:
108 if subp.poll() is None:
109 time.sleep(0.1)
109 time.sleep(0.1)
110 else:
110 else:
111 break
111 break
112
112
113 if subp.poll() is None:
113 if subp.poll() is None:
114 # The process did not die...
114 # The process did not die...
115 print('... failed. Manual cleanup may be required.')
115 print('... failed. Manual cleanup may be required.')
116
116
117 def cleanup(self):
117 def cleanup(self):
118 "Kill process if it's still alive, and clean up temporary directories"
118 "Kill process if it's still alive, and clean up temporary directories"
119 self.cleanup_process()
119 self.cleanup_process()
120 for td in self.dirs:
120 for td in self.dirs:
121 td.cleanup()
121 td.cleanup()
122
122
123 __del__ = cleanup
123 __del__ = cleanup
124
124
125 class PyTestController(TestController):
125 class PyTestController(TestController):
126 """Run Python tests using IPython.testing.iptest"""
126 """Run Python tests using IPython.testing.iptest"""
127 #: str, Python command to execute in subprocess
127 #: str, Python command to execute in subprocess
128 pycmd = None
128 pycmd = None
129
129
130 def __init__(self, section, options):
130 def __init__(self, section, options):
131 """Create new test runner."""
131 """Create new test runner."""
132 TestController.__init__(self)
132 TestController.__init__(self)
133 self.section = section
133 self.section = section
134 # pycmd is put into cmd[2] in PyTestController.launch()
134 # pycmd is put into cmd[2] in PyTestController.launch()
135 self.cmd = [sys.executable, '-c', None, section]
135 self.cmd = [sys.executable, '-c', None, section]
136 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
136 self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
137 self.options = options
137 self.options = options
138
138
139 def setup(self):
139 def setup(self):
140 ipydir = TemporaryDirectory()
140 ipydir = TemporaryDirectory()
141 self.dirs.append(ipydir)
141 self.dirs.append(ipydir)
142 self.env['IPYTHONDIR'] = ipydir.name
142 self.env['IPYTHONDIR'] = ipydir.name
143 self.workingdir = workingdir = TemporaryDirectory()
143 self.workingdir = workingdir = TemporaryDirectory()
144 self.dirs.append(workingdir)
144 self.dirs.append(workingdir)
145 self.env['IPTEST_WORKING_DIR'] = workingdir.name
145 self.env['IPTEST_WORKING_DIR'] = workingdir.name
146 # This means we won't get odd effects from our own matplotlib config
146 # This means we won't get odd effects from our own matplotlib config
147 self.env['MPLCONFIGDIR'] = workingdir.name
147 self.env['MPLCONFIGDIR'] = workingdir.name
148
148
149 # From options:
149 # From options:
150 if self.options.xunit:
150 if self.options.xunit:
151 self.add_xunit()
151 self.add_xunit()
152 if self.options.coverage:
152 if self.options.coverage:
153 self.add_coverage()
153 self.add_coverage()
154 self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
154 self.env['IPTEST_SUBPROC_STREAMS'] = self.options.subproc_streams
155 self.cmd.extend(self.options.extra_args)
155 self.cmd.extend(self.options.extra_args)
156
156
157 @property
157 @property
158 def will_run(self):
158 def will_run(self):
159 try:
159 try:
160 return test_sections[self.section].will_run
160 return test_sections[self.section].will_run
161 except KeyError:
161 except KeyError:
162 return True
162 return True
163
163
164 def add_xunit(self):
164 def add_xunit(self):
165 xunit_file = os.path.abspath(self.section + '.xunit.xml')
165 xunit_file = os.path.abspath(self.section + '.xunit.xml')
166 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
166 self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file])
167
167
168 def add_coverage(self):
168 def add_coverage(self):
169 try:
169 try:
170 sources = test_sections[self.section].includes
170 sources = test_sections[self.section].includes
171 except KeyError:
171 except KeyError:
172 sources = ['IPython']
172 sources = ['IPython']
173
173
174 coverage_rc = ("[run]\n"
174 coverage_rc = ("[run]\n"
175 "data_file = {data_file}\n"
175 "data_file = {data_file}\n"
176 "source =\n"
176 "source =\n"
177 " {source}\n"
177 " {source}\n"
178 ).format(data_file=os.path.abspath('.coverage.'+self.section),
178 ).format(data_file=os.path.abspath('.coverage.'+self.section),
179 source="\n ".join(sources))
179 source="\n ".join(sources))
180 config_file = os.path.join(self.workingdir.name, '.coveragerc')
180 config_file = os.path.join(self.workingdir.name, '.coveragerc')
181 with open(config_file, 'w') as f:
181 with open(config_file, 'w') as f:
182 f.write(coverage_rc)
182 f.write(coverage_rc)
183
183
184 self.env['COVERAGE_PROCESS_START'] = config_file
184 self.env['COVERAGE_PROCESS_START'] = config_file
185 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
185 self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
186
186
187 def launch(self, buffer_output=False):
187 def launch(self, buffer_output=False):
188 self.cmd[2] = self.pycmd
188 self.cmd[2] = self.pycmd
189 super(PyTestController, self).launch(buffer_output=buffer_output)
189 super(PyTestController, self).launch(buffer_output=buffer_output)
190
190
191 js_prefix = 'js/'
191 js_prefix = 'js/'
192
192
193 def get_js_test_dir():
193 def get_js_test_dir():
194 import IPython.html.tests as t
194 import IPython.html.tests as t
195 return os.path.join(os.path.dirname(t.__file__), '')
195 return os.path.join(os.path.dirname(t.__file__), '')
196
196
197 def all_js_groups():
197 def all_js_groups():
198 import glob
198 import glob
199 test_dir = get_js_test_dir()
199 test_dir = get_js_test_dir()
200 all_subdirs = glob.glob(test_dir + '*/')
200 all_subdirs = glob.glob(test_dir + '*/')
201 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs if os.path.relpath(x, test_dir) != '__pycache__']
201 return [js_prefix+os.path.relpath(x, test_dir) for x in all_subdirs if os.path.relpath(x, test_dir) != '__pycache__']
202
202
203 class JSController(TestController):
203 class JSController(TestController):
204 """Run CasperJS tests """
204 """Run CasperJS tests """
205 def __init__(self, section):
205 def __init__(self, section):
206 """Create new test runner."""
206 """Create new test runner."""
207 TestController.__init__(self)
207 TestController.__init__(self)
208 self.section = section
208 self.section = section
209 js_test_dir = get_js_test_dir()
209 js_test_dir = get_js_test_dir()
210 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
210 includes = '--includes=' + os.path.join(js_test_dir,'util.js')
211 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
211 test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
212 self.cmd = ['casperjs', 'test', includes, test_cases]
212 self.cmd = ['casperjs', 'test', includes, test_cases]
213
213
214 def setup(self):
214 def setup(self):
215 self.ipydir = TemporaryDirectory()
215 self.ipydir = TemporaryDirectory()
216 self.nbdir = TemporaryDirectory()
216 self.nbdir = TemporaryDirectory()
217 self.dirs.append(self.ipydir)
217 self.dirs.append(self.ipydir)
218 self.dirs.append(self.nbdir)
218 self.dirs.append(self.nbdir)
219 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
219 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir1', u'sub βˆ‚ir 1a')))
220 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
220 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub βˆ‚ir2', u'sub βˆ‚ir 1b')))
221
221
222 # start the ipython notebook, so we get the port number
222 # start the ipython notebook, so we get the port number
223 self.server_port = 0
223 self.server_port = 0
224 self._init_server()
224 self._init_server()
225 if self.server_port:
225 if self.server_port:
226 self.cmd.append("--port=%i" % self.server_port)
226 self.cmd.append("--port=%i" % self.server_port)
227 else:
227 else:
228 # don't launch tests if the server didn't start
228 # don't launch tests if the server didn't start
229 self.cmd = [sys.executable, '-c', 'raise SystemExit(1)']
229 self.cmd = [sys.executable, '-c', 'raise SystemExit(1)']
230
230
231 def print_extra_info(self):
231 def print_extra_info(self):
232 print("Running tests with notebook directory %r" % self.nbdir.name)
232 print("Running tests with notebook directory %r" % self.nbdir.name)
233
233
234 @property
234 @property
235 def will_run(self):
235 def will_run(self):
236 return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3'])
236 return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3'])
237
237
238 def _init_server(self):
238 def _init_server(self):
239 "Start the notebook server in a separate process"
239 "Start the notebook server in a separate process"
240 self.server_command = command = [sys.executable,
240 self.server_command = command = [sys.executable,
241 '-m', 'IPython.html',
241 '-m', 'IPython.html',
242 '--no-browser',
242 '--no-browser',
243 '--ipython-dir', self.ipydir.name,
243 '--ipython-dir', self.ipydir.name,
244 '--notebook-dir', self.nbdir.name,
244 '--notebook-dir', self.nbdir.name,
245 ]
245 ]
246 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
246 # ipc doesn't work on Windows, and darwin has crazy-long temp paths,
247 # which run afoul of ipc's maximum path length.
247 # which run afoul of ipc's maximum path length.
248 if sys.platform.startswith('linux'):
248 if sys.platform.startswith('linux'):
249 command.append('--KernelManager.transport=ipc')
249 command.append('--KernelManager.transport=ipc')
250 self.stream_capturer = c = StreamCapturer()
250 self.stream_capturer = c = StreamCapturer()
251 c.start()
251 c.start()
252 self.server = subprocess.Popen(command, stdout=c.writefd, stderr=subprocess.STDOUT)
252 self.server = subprocess.Popen(command, stdout=c.writefd, stderr=subprocess.STDOUT)
253 self.server_info_file = os.path.join(self.ipydir.name,
253 self.server_info_file = os.path.join(self.ipydir.name,
254 'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
254 'profile_default', 'security', 'nbserver-%i.json' % self.server.pid
255 )
255 )
256 self._wait_for_server()
256 self._wait_for_server()
257
257
258 def _wait_for_server(self):
258 def _wait_for_server(self):
259 """Wait 30 seconds for the notebook server to start"""
259 """Wait 30 seconds for the notebook server to start"""
260 for i in range(300):
260 for i in range(300):
261 if self.server.poll() is not None:
261 if self.server.poll() is not None:
262 return self._failed_to_start()
262 return self._failed_to_start()
263 if os.path.exists(self.server_info_file):
263 if os.path.exists(self.server_info_file):
264 self._load_server_info()
264 self._load_server_info()
265 return
265 return
266 time.sleep(0.1)
266 time.sleep(0.1)
267 print("Notebook server-info file never arrived: %s" % self.server_info_file,
267 print("Notebook server-info file never arrived: %s" % self.server_info_file,
268 file=sys.stderr
268 file=sys.stderr
269 )
269 )
270
270
271 def _failed_to_start(self):
271 def _failed_to_start(self):
272 """Notebook server exited prematurely"""
272 """Notebook server exited prematurely"""
273 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
273 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
274 print("Notebook failed to start: ", file=sys.stderr)
274 print("Notebook failed to start: ", file=sys.stderr)
275 print(self.server_command)
275 print(self.server_command)
276 print(captured, file=sys.stderr)
276 print(captured, file=sys.stderr)
277
277
278 def _load_server_info(self):
278 def _load_server_info(self):
279 """Notebook server started, load connection info from JSON"""
279 """Notebook server started, load connection info from JSON"""
280 with open(self.server_info_file) as f:
280 with open(self.server_info_file) as f:
281 info = json.load(f)
281 info = json.load(f)
282 self.server_port = info['port']
282 self.server_port = info['port']
283
283
284 def cleanup(self):
284 def cleanup(self):
285 try:
285 try:
286 self.server.terminate()
286 self.server.terminate()
287 except OSError:
287 except OSError:
288 # already dead
288 # already dead
289 pass
289 pass
290 self.server.wait()
290 self.server.wait()
291 self.stream_capturer.halt()
291 self.stream_capturer.halt()
292 TestController.cleanup(self)
292 TestController.cleanup(self)
293
293
294
294
295 def prepare_controllers(options):
295 def prepare_controllers(options):
296 """Returns two lists of TestController instances, those to run, and those
296 """Returns two lists of TestController instances, those to run, and those
297 not to run."""
297 not to run."""
298 testgroups = options.testgroups
298 testgroups = options.testgroups
299
299
300 if testgroups:
300 if testgroups:
301 py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
301 py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
302 or g.startswith('IPython.')]
302 or g.startswith('IPython.')]
303 if 'js' in testgroups:
303 if 'js' in testgroups:
304 js_testgroups = all_js_groups()
304 js_testgroups = all_js_groups()
305 else:
305 else:
306 js_testgroups = [g for g in testgroups if g not in py_testgroups]
306 js_testgroups = [g for g in testgroups if g not in py_testgroups]
307 else:
307 else:
308 py_testgroups = py_test_group_names
308 py_testgroups = py_test_group_names
309 js_testgroups = all_js_groups()
310 if not options.all:
309 if not options.all:
310 js_testgroups = []
311 test_sections['parallel'].enabled = False
311 test_sections['parallel'].enabled = False
312 else:
313 js_testgroups = all_js_groups()
312
314
313 c_js = [JSController(name) for name in js_testgroups]
315 c_js = [JSController(name) for name in js_testgroups]
314 c_py = [PyTestController(name, options) for name in py_testgroups]
316 c_py = [PyTestController(name, options) for name in py_testgroups]
315
317
316 controllers = c_py + c_js
318 controllers = c_py + c_js
317 to_run = [c for c in controllers if c.will_run]
319 to_run = [c for c in controllers if c.will_run]
318 not_run = [c for c in controllers if not c.will_run]
320 not_run = [c for c in controllers if not c.will_run]
319 return to_run, not_run
321 return to_run, not_run
320
322
321 def do_run(controller, buffer_output=True):
323 def do_run(controller, buffer_output=True):
322 """Setup and run a test controller.
324 """Setup and run a test controller.
323
325
324 If buffer_output is True, no output is displayed, to avoid it appearing
326 If buffer_output is True, no output is displayed, to avoid it appearing
325 interleaved. In this case, the caller is responsible for displaying test
327 interleaved. In this case, the caller is responsible for displaying test
326 output on failure.
328 output on failure.
327
329
328 Returns
330 Returns
329 -------
331 -------
330 controller : TestController
332 controller : TestController
331 The same controller as passed in, as a convenience for using map() type
333 The same controller as passed in, as a convenience for using map() type
332 APIs.
334 APIs.
333 exitcode : int
335 exitcode : int
334 The exit code of the test subprocess. Non-zero indicates failure.
336 The exit code of the test subprocess. Non-zero indicates failure.
335 """
337 """
336 try:
338 try:
337 try:
339 try:
338 controller.setup()
340 controller.setup()
339 if not buffer_output:
341 if not buffer_output:
340 controller.print_extra_info()
342 controller.print_extra_info()
341 controller.launch(buffer_output=buffer_output)
343 controller.launch(buffer_output=buffer_output)
342 except Exception:
344 except Exception:
343 import traceback
345 import traceback
344 traceback.print_exc()
346 traceback.print_exc()
345 return controller, 1 # signal failure
347 return controller, 1 # signal failure
346
348
347 exitcode = controller.wait()
349 exitcode = controller.wait()
348 return controller, exitcode
350 return controller, exitcode
349
351
350 except KeyboardInterrupt:
352 except KeyboardInterrupt:
351 return controller, -signal.SIGINT
353 return controller, -signal.SIGINT
352 finally:
354 finally:
353 controller.cleanup()
355 controller.cleanup()
354
356
355 def report():
357 def report():
356 """Return a string with a summary report of test-related variables."""
358 """Return a string with a summary report of test-related variables."""
357 inf = get_sys_info()
359 inf = get_sys_info()
358 out = []
360 out = []
359 def _add(name, value):
361 def _add(name, value):
360 out.append((name, value))
362 out.append((name, value))
361
363
362 _add('IPython version', inf['ipython_version'])
364 _add('IPython version', inf['ipython_version'])
363 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
365 _add('IPython commit', "{} ({})".format(inf['commit_hash'], inf['commit_source']))
364 _add('IPython package', compress_user(inf['ipython_path']))
366 _add('IPython package', compress_user(inf['ipython_path']))
365 _add('Python version', inf['sys_version'].replace('\n',''))
367 _add('Python version', inf['sys_version'].replace('\n',''))
366 _add('sys.executable', compress_user(inf['sys_executable']))
368 _add('sys.executable', compress_user(inf['sys_executable']))
367 _add('Platform', inf['platform'])
369 _add('Platform', inf['platform'])
368
370
369 width = max(len(n) for (n,v) in out)
371 width = max(len(n) for (n,v) in out)
370 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
372 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out]
371
373
372 avail = []
374 avail = []
373 not_avail = []
375 not_avail = []
374
376
375 for k, is_avail in have.items():
377 for k, is_avail in have.items():
376 if is_avail:
378 if is_avail:
377 avail.append(k)
379 avail.append(k)
378 else:
380 else:
379 not_avail.append(k)
381 not_avail.append(k)
380
382
381 if avail:
383 if avail:
382 out.append('\nTools and libraries available at test time:\n')
384 out.append('\nTools and libraries available at test time:\n')
383 avail.sort()
385 avail.sort()
384 out.append(' ' + ' '.join(avail)+'\n')
386 out.append(' ' + ' '.join(avail)+'\n')
385
387
386 if not_avail:
388 if not_avail:
387 out.append('\nTools and libraries NOT available at test time:\n')
389 out.append('\nTools and libraries NOT available at test time:\n')
388 not_avail.sort()
390 not_avail.sort()
389 out.append(' ' + ' '.join(not_avail)+'\n')
391 out.append(' ' + ' '.join(not_avail)+'\n')
390
392
391 return ''.join(out)
393 return ''.join(out)
392
394
393 def run_iptestall(options):
395 def run_iptestall(options):
394 """Run the entire IPython test suite by calling nose and trial.
396 """Run the entire IPython test suite by calling nose and trial.
395
397
396 This function constructs :class:`IPTester` instances for all IPython
398 This function constructs :class:`IPTester` instances for all IPython
397 modules and package and then runs each of them. This causes the modules
399 modules and package and then runs each of them. This causes the modules
398 and packages of IPython to be tested each in their own subprocess using
400 and packages of IPython to be tested each in their own subprocess using
399 nose.
401 nose.
400
402
401 Parameters
403 Parameters
402 ----------
404 ----------
403
405
404 All parameters are passed as attributes of the options object.
406 All parameters are passed as attributes of the options object.
405
407
406 testgroups : list of str
408 testgroups : list of str
407 Run only these sections of the test suite. If empty, run all the available
409 Run only these sections of the test suite. If empty, run all the available
408 sections.
410 sections.
409
411
410 fast : int or None
412 fast : int or None
411 Run the test suite in parallel, using n simultaneous processes. If None
413 Run the test suite in parallel, using n simultaneous processes. If None
412 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
414 is passed, one process is used per CPU core. Default 1 (i.e. sequential)
413
415
414 inc_slow : bool
416 inc_slow : bool
415 Include slow tests, like IPython.parallel. By default, these tests aren't
417 Include slow tests, like IPython.parallel. By default, these tests aren't
416 run.
418 run.
417
419
418 xunit : bool
420 xunit : bool
419 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
421 Produce Xunit XML output. This is written to multiple foo.xunit.xml files.
420
422
421 coverage : bool or str
423 coverage : bool or str
422 Measure code coverage from tests. True will store the raw coverage data,
424 Measure code coverage from tests. True will store the raw coverage data,
423 or pass 'html' or 'xml' to get reports.
425 or pass 'html' or 'xml' to get reports.
424
426
425 extra_args : list
427 extra_args : list
426 Extra arguments to pass to the test subprocesses, e.g. '-v'
428 Extra arguments to pass to the test subprocesses, e.g. '-v'
427 """
429 """
428 to_run, not_run = prepare_controllers(options)
430 to_run, not_run = prepare_controllers(options)
429
431
430 def justify(ltext, rtext, width=70, fill='-'):
432 def justify(ltext, rtext, width=70, fill='-'):
431 ltext += ' '
433 ltext += ' '
432 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
434 rtext = (' ' + rtext).rjust(width - len(ltext), fill)
433 return ltext + rtext
435 return ltext + rtext
434
436
435 # Run all test runners, tracking execution time
437 # Run all test runners, tracking execution time
436 failed = []
438 failed = []
437 t_start = time.time()
439 t_start = time.time()
438
440
439 print()
441 print()
440 if options.fast == 1:
442 if options.fast == 1:
441 # This actually means sequential, i.e. with 1 job
443 # This actually means sequential, i.e. with 1 job
442 for controller in to_run:
444 for controller in to_run:
443 print('Test group:', controller.section)
445 print('Test group:', controller.section)
444 sys.stdout.flush() # Show in correct order when output is piped
446 sys.stdout.flush() # Show in correct order when output is piped
445 controller, res = do_run(controller, buffer_output=False)
447 controller, res = do_run(controller, buffer_output=False)
446 if res:
448 if res:
447 failed.append(controller)
449 failed.append(controller)
448 if res == -signal.SIGINT:
450 if res == -signal.SIGINT:
449 print("Interrupted")
451 print("Interrupted")
450 break
452 break
451 print()
453 print()
452
454
453 else:
455 else:
454 # Run tests concurrently
456 # Run tests concurrently
455 try:
457 try:
456 pool = multiprocessing.pool.ThreadPool(options.fast)
458 pool = multiprocessing.pool.ThreadPool(options.fast)
457 for (controller, res) in pool.imap_unordered(do_run, to_run):
459 for (controller, res) in pool.imap_unordered(do_run, to_run):
458 res_string = 'OK' if res == 0 else 'FAILED'
460 res_string = 'OK' if res == 0 else 'FAILED'
459 print(justify('Test group: ' + controller.section, res_string))
461 print(justify('Test group: ' + controller.section, res_string))
460 if res:
462 if res:
461 controller.print_extra_info()
463 controller.print_extra_info()
462 print(bytes_to_str(controller.stdout))
464 print(bytes_to_str(controller.stdout))
463 failed.append(controller)
465 failed.append(controller)
464 if res == -signal.SIGINT:
466 if res == -signal.SIGINT:
465 print("Interrupted")
467 print("Interrupted")
466 break
468 break
467 except KeyboardInterrupt:
469 except KeyboardInterrupt:
468 return
470 return
469
471
470 for controller in not_run:
472 for controller in not_run:
471 print(justify('Test group: ' + controller.section, 'NOT RUN'))
473 print(justify('Test group: ' + controller.section, 'NOT RUN'))
472
474
473 t_end = time.time()
475 t_end = time.time()
474 t_tests = t_end - t_start
476 t_tests = t_end - t_start
475 nrunners = len(to_run)
477 nrunners = len(to_run)
476 nfail = len(failed)
478 nfail = len(failed)
477 # summarize results
479 # summarize results
478 print('_'*70)
480 print('_'*70)
479 print('Test suite completed for system with the following information:')
481 print('Test suite completed for system with the following information:')
480 print(report())
482 print(report())
481 took = "Took %.3fs." % t_tests
483 took = "Took %.3fs." % t_tests
482 print('Status: ', end='')
484 print('Status: ', end='')
483 if not failed:
485 if not failed:
484 print('OK (%d test groups).' % nrunners, took)
486 print('OK (%d test groups).' % nrunners, took)
485 else:
487 else:
486 # If anything went wrong, point out what command to rerun manually to
488 # If anything went wrong, point out what command to rerun manually to
487 # see the actual errors and individual summary
489 # see the actual errors and individual summary
488 failed_sections = [c.section for c in failed]
490 failed_sections = [c.section for c in failed]
489 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
491 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
490 nrunners, ', '.join(failed_sections)), took)
492 nrunners, ', '.join(failed_sections)), took)
491 print()
493 print()
492 print('You may wish to rerun these, with:')
494 print('You may wish to rerun these, with:')
493 print(' iptest', *failed_sections)
495 print(' iptest', *failed_sections)
494 print()
496 print()
495
497
496 if options.coverage:
498 if options.coverage:
497 from coverage import coverage
499 from coverage import coverage
498 cov = coverage(data_file='.coverage')
500 cov = coverage(data_file='.coverage')
499 cov.combine()
501 cov.combine()
500 cov.save()
502 cov.save()
501
503
502 # Coverage HTML report
504 # Coverage HTML report
503 if options.coverage == 'html':
505 if options.coverage == 'html':
504 html_dir = 'ipy_htmlcov'
506 html_dir = 'ipy_htmlcov'
505 shutil.rmtree(html_dir, ignore_errors=True)
507 shutil.rmtree(html_dir, ignore_errors=True)
506 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
508 print("Writing HTML coverage report to %s/ ... " % html_dir, end="")
507 sys.stdout.flush()
509 sys.stdout.flush()
508
510
509 # Custom HTML reporter to clean up module names.
511 # Custom HTML reporter to clean up module names.
510 from coverage.html import HtmlReporter
512 from coverage.html import HtmlReporter
511 class CustomHtmlReporter(HtmlReporter):
513 class CustomHtmlReporter(HtmlReporter):
512 def find_code_units(self, morfs):
514 def find_code_units(self, morfs):
513 super(CustomHtmlReporter, self).find_code_units(morfs)
515 super(CustomHtmlReporter, self).find_code_units(morfs)
514 for cu in self.code_units:
516 for cu in self.code_units:
515 nameparts = cu.name.split(os.sep)
517 nameparts = cu.name.split(os.sep)
516 if 'IPython' not in nameparts:
518 if 'IPython' not in nameparts:
517 continue
519 continue
518 ix = nameparts.index('IPython')
520 ix = nameparts.index('IPython')
519 cu.name = '.'.join(nameparts[ix:])
521 cu.name = '.'.join(nameparts[ix:])
520
522
521 # Reimplement the html_report method with our custom reporter
523 # Reimplement the html_report method with our custom reporter
522 cov._harvest_data()
524 cov._harvest_data()
523 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
525 cov.config.from_args(omit='*{0}tests{0}*'.format(os.sep), html_dir=html_dir,
524 html_title='IPython test coverage',
526 html_title='IPython test coverage',
525 )
527 )
526 reporter = CustomHtmlReporter(cov, cov.config)
528 reporter = CustomHtmlReporter(cov, cov.config)
527 reporter.report(None)
529 reporter.report(None)
528 print('done.')
530 print('done.')
529
531
530 # Coverage XML report
532 # Coverage XML report
531 elif options.coverage == 'xml':
533 elif options.coverage == 'xml':
532 cov.xml_report(outfile='ipy_coverage.xml')
534 cov.xml_report(outfile='ipy_coverage.xml')
533
535
534 if failed:
536 if failed:
535 # Ensure that our exit code indicates failure
537 # Ensure that our exit code indicates failure
536 sys.exit(1)
538 sys.exit(1)
537
539
538 argparser = argparse.ArgumentParser(description='Run IPython test suite')
540 argparser = argparse.ArgumentParser(description='Run IPython test suite')
539 argparser.add_argument('testgroups', nargs='*',
541 argparser.add_argument('testgroups', nargs='*',
540 help='Run specified groups of tests. If omitted, run '
542 help='Run specified groups of tests. If omitted, run '
541 'all tests.')
543 'all tests.')
542 argparser.add_argument('--all', action='store_true',
544 argparser.add_argument('--all', action='store_true',
543 help='Include slow tests not run by default.')
545 help='Include slow tests not run by default.')
544 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
546 argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int,
545 help='Run test sections in parallel. This starts as many '
547 help='Run test sections in parallel. This starts as many '
546 'processes as you have cores, or you can specify a number.')
548 'processes as you have cores, or you can specify a number.')
547 argparser.add_argument('--xunit', action='store_true',
549 argparser.add_argument('--xunit', action='store_true',
548 help='Produce Xunit XML results')
550 help='Produce Xunit XML results')
549 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
551 argparser.add_argument('--coverage', nargs='?', const=True, default=False,
550 help="Measure test coverage. Specify 'html' or "
552 help="Measure test coverage. Specify 'html' or "
551 "'xml' to get reports.")
553 "'xml' to get reports.")
552 argparser.add_argument('--subproc-streams', default='capture',
554 argparser.add_argument('--subproc-streams', default='capture',
553 help="What to do with stdout/stderr from subprocesses. "
555 help="What to do with stdout/stderr from subprocesses. "
554 "'capture' (default), 'show' and 'discard' are the options.")
556 "'capture' (default), 'show' and 'discard' are the options.")
555
557
556 def default_options():
558 def default_options():
557 """Get an argparse Namespace object with the default arguments, to pass to
559 """Get an argparse Namespace object with the default arguments, to pass to
558 :func:`run_iptestall`.
560 :func:`run_iptestall`.
559 """
561 """
560 options = argparser.parse_args([])
562 options = argparser.parse_args([])
561 options.extra_args = []
563 options.extra_args = []
562 return options
564 return options
563
565
564 def main():
566 def main():
565 # iptest doesn't work correctly if the working directory is the
567 # iptest doesn't work correctly if the working directory is the
566 # root of the IPython source tree. Tell the user to avoid
568 # root of the IPython source tree. Tell the user to avoid
567 # frustration.
569 # frustration.
568 if os.path.exists(os.path.join(os.getcwd(),
570 if os.path.exists(os.path.join(os.getcwd(),
569 'IPython', 'testing', '__main__.py')):
571 'IPython', 'testing', '__main__.py')):
570 print("Don't run iptest from the IPython source directory",
572 print("Don't run iptest from the IPython source directory",
571 file=sys.stderr)
573 file=sys.stderr)
572 sys.exit(1)
574 sys.exit(1)
573 # Arguments after -- should be passed through to nose. Argparse treats
575 # Arguments after -- should be passed through to nose. Argparse treats
574 # everything after -- as regular positional arguments, so we separate them
576 # everything after -- as regular positional arguments, so we separate them
575 # first.
577 # first.
576 try:
578 try:
577 ix = sys.argv.index('--')
579 ix = sys.argv.index('--')
578 except ValueError:
580 except ValueError:
579 to_parse = sys.argv[1:]
581 to_parse = sys.argv[1:]
580 extra_args = []
582 extra_args = []
581 else:
583 else:
582 to_parse = sys.argv[1:ix]
584 to_parse = sys.argv[1:ix]
583 extra_args = sys.argv[ix+1:]
585 extra_args = sys.argv[ix+1:]
584
586
585 options = argparser.parse_args(to_parse)
587 options = argparser.parse_args(to_parse)
586 options.extra_args = extra_args
588 options.extra_args = extra_args
587
589
588 run_iptestall(options)
590 run_iptestall(options)
589
591
590
592
591 if __name__ == '__main__':
593 if __name__ == '__main__':
592 main()
594 main()
General Comments 0
You need to be logged in to leave comments. Login now