##// END OF EJS Templates
allow iptest to be interrupted...
MinRK -
Show More
@@ -1,583 +1,593 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 from __future__ import print_function
27 from __future__ import print_function
28
28
29 # Stdlib
29 # Stdlib
30 import glob
30 import glob
31 import os
31 import os
32 import os.path as path
32 import os.path as path
33 import signal
33 import signal
34 import sys
34 import sys
35 import subprocess
35 import subprocess
36 import tempfile
36 import tempfile
37 import time
37 import time
38 import warnings
38 import warnings
39
39
40 # Note: monkeypatch!
40 # Note: monkeypatch!
41 # We need to monkeypatch a small problem in nose itself first, before importing
41 # We need to monkeypatch a small problem in nose itself first, before importing
42 # it for actual use. This should get into nose upstream, but its release cycle
42 # it for actual use. This should get into nose upstream, but its release cycle
43 # is slow and we need it for our parametric tests to work correctly.
43 # is slow and we need it for our parametric tests to work correctly.
44 from IPython.testing import nosepatch
44 from IPython.testing import nosepatch
45
45
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
47 # This can be dropped once we no longer test on Python 2.6
47 # This can be dropped once we no longer test on Python 2.6
48 from IPython.testing import nose_assert_methods
48 from IPython.testing import nose_assert_methods
49
49
50 # Now, proceed to import nose itself
50 # Now, proceed to import nose itself
51 import nose.plugins.builtin
51 import nose.plugins.builtin
52 from nose.plugins.xunit import Xunit
52 from nose.plugins.xunit import Xunit
53 from nose import SkipTest
53 from nose import SkipTest
54 from nose.core import TestProgram
54 from nose.core import TestProgram
55
55
56 # Our own imports
56 # Our own imports
57 from IPython.utils import py3compat
57 from IPython.utils import py3compat
58 from IPython.utils.importstring import import_item
58 from IPython.utils.importstring import import_item
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
60 from IPython.utils.process import find_cmd, pycmd2argv
60 from IPython.utils.process import find_cmd, pycmd2argv
61 from IPython.utils.sysinfo import sys_info
61 from IPython.utils.sysinfo import sys_info
62 from IPython.utils.tempdir import TemporaryDirectory
62 from IPython.utils.tempdir import TemporaryDirectory
63 from IPython.utils.warn import warn
63 from IPython.utils.warn import warn
64
64
65 from IPython.testing import globalipapp
65 from IPython.testing import globalipapp
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
67 from IPython.external.decorators import KnownFailure, knownfailureif
67 from IPython.external.decorators import KnownFailure, knownfailureif
68
68
69 pjoin = path.join
69 pjoin = path.join
70
70
71
71
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73 # Globals
73 # Globals
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75
75
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Warnings control
78 # Warnings control
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
82 # that imports 'sets' as of today
82 # that imports 'sets' as of today
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
84 DeprecationWarning )
84 DeprecationWarning )
85
85
86 # This one also comes from Twisted
86 # This one also comes from Twisted
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
88 DeprecationWarning)
88 DeprecationWarning)
89
89
90 # Wx on Fedora11 spits these out
90 # Wx on Fedora11 spits these out
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
92 UserWarning)
92 UserWarning)
93
93
94 # ------------------------------------------------------------------------------
94 # ------------------------------------------------------------------------------
95 # Monkeypatch Xunit to count known failures as skipped.
95 # Monkeypatch Xunit to count known failures as skipped.
96 # ------------------------------------------------------------------------------
96 # ------------------------------------------------------------------------------
97 def monkeypatch_xunit():
97 def monkeypatch_xunit():
98 try:
98 try:
99 knownfailureif(True)(lambda: None)()
99 knownfailureif(True)(lambda: None)()
100 except Exception as e:
100 except Exception as e:
101 KnownFailureTest = type(e)
101 KnownFailureTest = type(e)
102
102
103 def addError(self, test, err, capt=None):
103 def addError(self, test, err, capt=None):
104 if issubclass(err[0], KnownFailureTest):
104 if issubclass(err[0], KnownFailureTest):
105 err = (SkipTest,) + err[1:]
105 err = (SkipTest,) + err[1:]
106 return self.orig_addError(test, err, capt)
106 return self.orig_addError(test, err, capt)
107
107
108 Xunit.orig_addError = Xunit.addError
108 Xunit.orig_addError = Xunit.addError
109 Xunit.addError = addError
109 Xunit.addError = addError
110
110
111 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
112 # Logic for skipping doctests
112 # Logic for skipping doctests
113 #-----------------------------------------------------------------------------
113 #-----------------------------------------------------------------------------
114 def extract_version(mod):
114 def extract_version(mod):
115 return mod.__version__
115 return mod.__version__
116
116
117 def test_for(item, min_version=None, callback=extract_version):
117 def test_for(item, min_version=None, callback=extract_version):
118 """Test to see if item is importable, and optionally check against a minimum
118 """Test to see if item is importable, and optionally check against a minimum
119 version.
119 version.
120
120
121 If min_version is given, the default behavior is to check against the
121 If min_version is given, the default behavior is to check against the
122 `__version__` attribute of the item, but specifying `callback` allows you to
122 `__version__` attribute of the item, but specifying `callback` allows you to
123 extract the value you are interested in. e.g::
123 extract the value you are interested in. e.g::
124
124
125 In [1]: import sys
125 In [1]: import sys
126
126
127 In [2]: from IPython.testing.iptest import test_for
127 In [2]: from IPython.testing.iptest import test_for
128
128
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
130 Out[3]: True
130 Out[3]: True
131
131
132 """
132 """
133 try:
133 try:
134 check = import_item(item)
134 check = import_item(item)
135 except (ImportError, RuntimeError):
135 except (ImportError, RuntimeError):
136 # GTK reports Runtime error if it can't be initialized even if it's
136 # GTK reports Runtime error if it can't be initialized even if it's
137 # importable.
137 # importable.
138 return False
138 return False
139 else:
139 else:
140 if min_version:
140 if min_version:
141 if callback:
141 if callback:
142 # extra processing step to get version to compare
142 # extra processing step to get version to compare
143 check = callback(check)
143 check = callback(check)
144
144
145 return check >= min_version
145 return check >= min_version
146 else:
146 else:
147 return True
147 return True
148
148
149 # Global dict where we can store information on what we have and what we don't
149 # Global dict where we can store information on what we have and what we don't
150 # have available at test run time
150 # have available at test run time
151 have = {}
151 have = {}
152
152
153 have['curses'] = test_for('_curses')
153 have['curses'] = test_for('_curses')
154 have['matplotlib'] = test_for('matplotlib')
154 have['matplotlib'] = test_for('matplotlib')
155 have['numpy'] = test_for('numpy')
155 have['numpy'] = test_for('numpy')
156 have['pexpect'] = test_for('IPython.external.pexpect')
156 have['pexpect'] = test_for('IPython.external.pexpect')
157 have['pymongo'] = test_for('pymongo')
157 have['pymongo'] = test_for('pymongo')
158 have['pygments'] = test_for('pygments')
158 have['pygments'] = test_for('pygments')
159 have['qt'] = test_for('IPython.external.qt')
159 have['qt'] = test_for('IPython.external.qt')
160 have['rpy2'] = test_for('rpy2')
160 have['rpy2'] = test_for('rpy2')
161 have['sqlite3'] = test_for('sqlite3')
161 have['sqlite3'] = test_for('sqlite3')
162 have['cython'] = test_for('Cython')
162 have['cython'] = test_for('Cython')
163 have['oct2py'] = test_for('oct2py')
163 have['oct2py'] = test_for('oct2py')
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
165 have['wx'] = test_for('wx')
165 have['wx'] = test_for('wx')
166 have['wx.aui'] = test_for('wx.aui')
166 have['wx.aui'] = test_for('wx.aui')
167 have['azure'] = test_for('azure')
167 have['azure'] = test_for('azure')
168
168
169 if os.name == 'nt':
169 if os.name == 'nt':
170 min_zmq = (2,1,7)
170 min_zmq = (2,1,7)
171 else:
171 else:
172 min_zmq = (2,1,4)
172 min_zmq = (2,1,4)
173
173
174 def version_tuple(mod):
174 def version_tuple(mod):
175 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
175 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
176 # turn 'dev' into 999, because Python3 rejects str-int comparisons
176 # turn 'dev' into 999, because Python3 rejects str-int comparisons
177 vs = mod.__version__.replace('dev', '.999')
177 vs = mod.__version__.replace('dev', '.999')
178 tup = tuple([int(v) for v in vs.split('.') ])
178 tup = tuple([int(v) for v in vs.split('.') ])
179 return tup
179 return tup
180
180
181 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
181 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
182
182
183 #-----------------------------------------------------------------------------
183 #-----------------------------------------------------------------------------
184 # Functions and classes
184 # Functions and classes
185 #-----------------------------------------------------------------------------
185 #-----------------------------------------------------------------------------
186
186
187 def report():
187 def report():
188 """Return a string with a summary report of test-related variables."""
188 """Return a string with a summary report of test-related variables."""
189
189
190 out = [ sys_info(), '\n']
190 out = [ sys_info(), '\n']
191
191
192 avail = []
192 avail = []
193 not_avail = []
193 not_avail = []
194
194
195 for k, is_avail in have.items():
195 for k, is_avail in have.items():
196 if is_avail:
196 if is_avail:
197 avail.append(k)
197 avail.append(k)
198 else:
198 else:
199 not_avail.append(k)
199 not_avail.append(k)
200
200
201 if avail:
201 if avail:
202 out.append('\nTools and libraries available at test time:\n')
202 out.append('\nTools and libraries available at test time:\n')
203 avail.sort()
203 avail.sort()
204 out.append(' ' + ' '.join(avail)+'\n')
204 out.append(' ' + ' '.join(avail)+'\n')
205
205
206 if not_avail:
206 if not_avail:
207 out.append('\nTools and libraries NOT available at test time:\n')
207 out.append('\nTools and libraries NOT available at test time:\n')
208 not_avail.sort()
208 not_avail.sort()
209 out.append(' ' + ' '.join(not_avail)+'\n')
209 out.append(' ' + ' '.join(not_avail)+'\n')
210
210
211 return ''.join(out)
211 return ''.join(out)
212
212
213
213
214 def make_exclude():
214 def make_exclude():
215 """Make patterns of modules and packages to exclude from testing.
215 """Make patterns of modules and packages to exclude from testing.
216
216
217 For the IPythonDoctest plugin, we need to exclude certain patterns that
217 For the IPythonDoctest plugin, we need to exclude certain patterns that
218 cause testing problems. We should strive to minimize the number of
218 cause testing problems. We should strive to minimize the number of
219 skipped modules, since this means untested code.
219 skipped modules, since this means untested code.
220
220
221 These modules and packages will NOT get scanned by nose at all for tests.
221 These modules and packages will NOT get scanned by nose at all for tests.
222 """
222 """
223 # Simple utility to make IPython paths more readably, we need a lot of
223 # Simple utility to make IPython paths more readably, we need a lot of
224 # these below
224 # these below
225 ipjoin = lambda *paths: pjoin('IPython', *paths)
225 ipjoin = lambda *paths: pjoin('IPython', *paths)
226
226
227 exclusions = [ipjoin('external'),
227 exclusions = [ipjoin('external'),
228 ipjoin('quarantine'),
228 ipjoin('quarantine'),
229 ipjoin('deathrow'),
229 ipjoin('deathrow'),
230 # This guy is probably attic material
230 # This guy is probably attic material
231 ipjoin('testing', 'mkdoctests'),
231 ipjoin('testing', 'mkdoctests'),
232 # Testing inputhook will need a lot of thought, to figure out
232 # Testing inputhook will need a lot of thought, to figure out
233 # how to have tests that don't lock up with the gui event
233 # how to have tests that don't lock up with the gui event
234 # loops in the picture
234 # loops in the picture
235 ipjoin('lib', 'inputhook'),
235 ipjoin('lib', 'inputhook'),
236 # Config files aren't really importable stand-alone
236 # Config files aren't really importable stand-alone
237 ipjoin('config', 'profile'),
237 ipjoin('config', 'profile'),
238 # The notebook 'static' directory contains JS, css and other
238 # The notebook 'static' directory contains JS, css and other
239 # files for web serving. Occasionally projects may put a .py
239 # files for web serving. Occasionally projects may put a .py
240 # file in there (MathJax ships a conf.py), so we might as
240 # file in there (MathJax ships a conf.py), so we might as
241 # well play it safe and skip the whole thing.
241 # well play it safe and skip the whole thing.
242 ipjoin('frontend', 'html', 'notebook', 'static')
242 ipjoin('frontend', 'html', 'notebook', 'static')
243 ]
243 ]
244 if not have['sqlite3']:
244 if not have['sqlite3']:
245 exclusions.append(ipjoin('core', 'tests', 'test_history'))
245 exclusions.append(ipjoin('core', 'tests', 'test_history'))
246 exclusions.append(ipjoin('core', 'history'))
246 exclusions.append(ipjoin('core', 'history'))
247 if not have['wx']:
247 if not have['wx']:
248 exclusions.append(ipjoin('lib', 'inputhookwx'))
248 exclusions.append(ipjoin('lib', 'inputhookwx'))
249
249
250 # FIXME: temporarily disable autoreload tests, as they can produce
250 # FIXME: temporarily disable autoreload tests, as they can produce
251 # spurious failures in subsequent tests (cythonmagic).
251 # spurious failures in subsequent tests (cythonmagic).
252 exclusions.append(ipjoin('extensions', 'autoreload'))
252 exclusions.append(ipjoin('extensions', 'autoreload'))
253 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
253 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
254
254
255 # We do this unconditionally, so that the test suite doesn't import
255 # We do this unconditionally, so that the test suite doesn't import
256 # gtk, changing the default encoding and masking some unicode bugs.
256 # gtk, changing the default encoding and masking some unicode bugs.
257 exclusions.append(ipjoin('lib', 'inputhookgtk'))
257 exclusions.append(ipjoin('lib', 'inputhookgtk'))
258 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
258 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
259
259
260 # These have to be skipped on win32 because the use echo, rm, cd, etc.
260 # These have to be skipped on win32 because the use echo, rm, cd, etc.
261 # See ticket https://github.com/ipython/ipython/issues/87
261 # See ticket https://github.com/ipython/ipython/issues/87
262 if sys.platform == 'win32':
262 if sys.platform == 'win32':
263 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
263 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
264 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
264 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
265
265
266 if not have['pexpect']:
266 if not have['pexpect']:
267 exclusions.extend([ipjoin('lib', 'irunner'),
267 exclusions.extend([ipjoin('lib', 'irunner'),
268 ipjoin('lib', 'tests', 'test_irunner'),
268 ipjoin('lib', 'tests', 'test_irunner'),
269 ipjoin('frontend', 'terminal', 'console'),
269 ipjoin('frontend', 'terminal', 'console'),
270 ])
270 ])
271
271
272 if not have['zmq']:
272 if not have['zmq']:
273 exclusions.append(ipjoin('zmq'))
273 exclusions.append(ipjoin('zmq'))
274 exclusions.append(ipjoin('frontend', 'qt'))
274 exclusions.append(ipjoin('frontend', 'qt'))
275 exclusions.append(ipjoin('frontend', 'html'))
275 exclusions.append(ipjoin('frontend', 'html'))
276 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
276 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
277 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
277 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
278 exclusions.append(ipjoin('parallel'))
278 exclusions.append(ipjoin('parallel'))
279 elif not have['qt'] or not have['pygments']:
279 elif not have['qt'] or not have['pygments']:
280 exclusions.append(ipjoin('frontend', 'qt'))
280 exclusions.append(ipjoin('frontend', 'qt'))
281
281
282 if not have['pymongo']:
282 if not have['pymongo']:
283 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
283 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
284 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
284 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
285
285
286 if not have['matplotlib']:
286 if not have['matplotlib']:
287 exclusions.extend([ipjoin('core', 'pylabtools'),
287 exclusions.extend([ipjoin('core', 'pylabtools'),
288 ipjoin('core', 'tests', 'test_pylabtools'),
288 ipjoin('core', 'tests', 'test_pylabtools'),
289 ipjoin('zmq', 'pylab'),
289 ipjoin('zmq', 'pylab'),
290 ])
290 ])
291
291
292 if not have['cython']:
292 if not have['cython']:
293 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
293 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
294 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
294 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
295
295
296 if not have['oct2py']:
296 if not have['oct2py']:
297 exclusions.extend([ipjoin('extensions', 'octavemagic')])
297 exclusions.extend([ipjoin('extensions', 'octavemagic')])
298 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
298 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
299
299
300 if not have['tornado']:
300 if not have['tornado']:
301 exclusions.append(ipjoin('frontend', 'html'))
301 exclusions.append(ipjoin('frontend', 'html'))
302
302
303 if not have['rpy2'] or not have['numpy']:
303 if not have['rpy2'] or not have['numpy']:
304 exclusions.append(ipjoin('extensions', 'rmagic'))
304 exclusions.append(ipjoin('extensions', 'rmagic'))
305 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
305 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
306
306
307 if not have['azure']:
307 if not have['azure']:
308 exclusions.append(ipjoin('frontend', 'html', 'notebook', 'azurenbmanager'))
308 exclusions.append(ipjoin('frontend', 'html', 'notebook', 'azurenbmanager'))
309
309
310 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
310 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
311 if sys.platform == 'win32':
311 if sys.platform == 'win32':
312 exclusions = [s.replace('\\','\\\\') for s in exclusions]
312 exclusions = [s.replace('\\','\\\\') for s in exclusions]
313
313
314 # check for any exclusions that don't seem to exist:
314 # check for any exclusions that don't seem to exist:
315 parent, _ = os.path.split(get_ipython_package_dir())
315 parent, _ = os.path.split(get_ipython_package_dir())
316 for exclusion in exclusions:
316 for exclusion in exclusions:
317 if exclusion.endswith(('deathrow', 'quarantine')):
317 if exclusion.endswith(('deathrow', 'quarantine')):
318 # ignore deathrow/quarantine, which exist in dev, but not install
318 # ignore deathrow/quarantine, which exist in dev, but not install
319 continue
319 continue
320 fullpath = pjoin(parent, exclusion)
320 fullpath = pjoin(parent, exclusion)
321 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
321 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
322 warn("Excluding nonexistent file: %r\n" % exclusion)
322 warn("Excluding nonexistent file: %r\n" % exclusion)
323
323
324 return exclusions
324 return exclusions
325
325
326
326
327 class IPTester(object):
327 class IPTester(object):
328 """Call that calls iptest or trial in a subprocess.
328 """Call that calls iptest or trial in a subprocess.
329 """
329 """
330 #: string, name of test runner that will be called
330 #: string, name of test runner that will be called
331 runner = None
331 runner = None
332 #: list, parameters for test runner
332 #: list, parameters for test runner
333 params = None
333 params = None
334 #: list, arguments of system call to be made to call test runner
334 #: list, arguments of system call to be made to call test runner
335 call_args = None
335 call_args = None
336 #: list, subprocesses we start (for cleanup)
336 #: list, subprocesses we start (for cleanup)
337 processes = None
337 processes = None
338 #: str, coverage xml output file
338 #: str, coverage xml output file
339 coverage_xml = None
339 coverage_xml = None
340
340
341 def __init__(self, runner='iptest', params=None):
341 def __init__(self, runner='iptest', params=None):
342 """Create new test runner."""
342 """Create new test runner."""
343 p = os.path
343 p = os.path
344 if runner == 'iptest':
344 if runner == 'iptest':
345 iptest_app = get_ipython_module_path('IPython.testing.iptest')
345 iptest_app = get_ipython_module_path('IPython.testing.iptest')
346 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
346 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
347 else:
347 else:
348 raise Exception('Not a valid test runner: %s' % repr(runner))
348 raise Exception('Not a valid test runner: %s' % repr(runner))
349 if params is None:
349 if params is None:
350 params = []
350 params = []
351 if isinstance(params, str):
351 if isinstance(params, str):
352 params = [params]
352 params = [params]
353 self.params = params
353 self.params = params
354
354
355 # Assemble call
355 # Assemble call
356 self.call_args = self.runner+self.params
356 self.call_args = self.runner+self.params
357
357
358 # Find the section we're testing (IPython.foo)
358 # Find the section we're testing (IPython.foo)
359 for sect in self.params:
359 for sect in self.params:
360 if sect.startswith('IPython'): break
360 if sect.startswith('IPython'): break
361 else:
361 else:
362 raise ValueError("Section not found", self.params)
362 raise ValueError("Section not found", self.params)
363
363
364 if '--with-xunit' in self.call_args:
364 if '--with-xunit' in self.call_args:
365
365
366 self.call_args.append('--xunit-file')
366 self.call_args.append('--xunit-file')
367 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
367 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
368 xunit_file = path.abspath(sect+'.xunit.xml')
368 xunit_file = path.abspath(sect+'.xunit.xml')
369 if sys.platform == 'win32':
369 if sys.platform == 'win32':
370 xunit_file = '"%s"' % xunit_file
370 xunit_file = '"%s"' % xunit_file
371 self.call_args.append(xunit_file)
371 self.call_args.append(xunit_file)
372
372
373 if '--with-xml-coverage' in self.call_args:
373 if '--with-xml-coverage' in self.call_args:
374 self.coverage_xml = path.abspath(sect+".coverage.xml")
374 self.coverage_xml = path.abspath(sect+".coverage.xml")
375 self.call_args.remove('--with-xml-coverage')
375 self.call_args.remove('--with-xml-coverage')
376 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
376 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
377
377
378 # Store anything we start to clean up on deletion
378 # Store anything we start to clean up on deletion
379 self.processes = []
379 self.processes = []
380
380
381 def _run_cmd(self):
381 def _run_cmd(self):
382 with TemporaryDirectory() as IPYTHONDIR:
382 with TemporaryDirectory() as IPYTHONDIR:
383 env = os.environ.copy()
383 env = os.environ.copy()
384 env['IPYTHONDIR'] = IPYTHONDIR
384 env['IPYTHONDIR'] = IPYTHONDIR
385 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
385 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
386 subp = subprocess.Popen(self.call_args, env=env)
386 subp = subprocess.Popen(self.call_args, env=env)
387 self.processes.append(subp)
387 self.processes.append(subp)
388 # If this fails, the process will be left in self.processes and
388 # If this fails, the process will be left in self.processes and
389 # cleaned up later, but if the wait call succeeds, then we can
389 # cleaned up later, but if the wait call succeeds, then we can
390 # clear the stored process.
390 # clear the stored process.
391 retcode = subp.wait()
391 retcode = subp.wait()
392 self.processes.pop()
392 self.processes.pop()
393 return retcode
393 return retcode
394
394
395 def run(self):
395 def run(self):
396 """Run the stored commands"""
396 """Run the stored commands"""
397 try:
397 try:
398 retcode = self._run_cmd()
398 retcode = self._run_cmd()
399 except KeyboardInterrupt:
400 return -signal.SIGINT
399 except:
401 except:
400 import traceback
402 import traceback
401 traceback.print_exc()
403 traceback.print_exc()
402 return 1 # signal failure
404 return 1 # signal failure
403
405
404 if self.coverage_xml:
406 if self.coverage_xml:
405 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
407 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
406 return retcode
408 return retcode
407
409
408 def __del__(self):
410 def __del__(self):
409 """Cleanup on exit by killing any leftover processes."""
411 """Cleanup on exit by killing any leftover processes."""
410 for subp in self.processes:
412 for subp in self.processes:
411 if subp.poll() is not None:
413 if subp.poll() is not None:
412 continue # process is already dead
414 continue # process is already dead
413
415
414 try:
416 try:
415 print('Cleaning stale PID: %d' % subp.pid)
417 print('Cleaning up stale PID: %d' % subp.pid)
416 subp.kill()
418 subp.kill()
417 except: # (OSError, WindowsError) ?
419 except: # (OSError, WindowsError) ?
418 # This is just a best effort, if we fail or the process was
420 # This is just a best effort, if we fail or the process was
419 # really gone, ignore it.
421 # really gone, ignore it.
420 pass
422 pass
423 else:
424 for i in range(10):
425 if subp.poll() is None:
426 time.sleep(0.1)
427 else:
428 break
421
429
422 if subp.poll() is None:
430 if subp.poll() is None:
423 # The process did not die...
431 # The process did not die...
424 print('... failed. Manual cleanup may be required.'
432 print('... failed. Manual cleanup may be required.')
425 % subp.pid)
426
433
427 def make_runners(inc_slow=False):
434 def make_runners(inc_slow=False):
428 """Define the top-level packages that need to be tested.
435 """Define the top-level packages that need to be tested.
429 """
436 """
430
437
431 # Packages to be tested via nose, that only depend on the stdlib
438 # Packages to be tested via nose, that only depend on the stdlib
432 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
439 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
433 'testing', 'utils', 'nbformat' ]
440 'testing', 'utils', 'nbformat' ]
434
441
435 if have['zmq']:
442 if have['zmq']:
436 nose_pkg_names.append('zmq')
443 nose_pkg_names.append('zmq')
437 if inc_slow:
444 if inc_slow:
438 nose_pkg_names.append('parallel')
445 nose_pkg_names.append('parallel')
439
446
440 # For debugging this code, only load quick stuff
447 # For debugging this code, only load quick stuff
441 #nose_pkg_names = ['core', 'extensions'] # dbg
448 #nose_pkg_names = ['core', 'extensions'] # dbg
442
449
443 # Make fully qualified package names prepending 'IPython.' to our name lists
450 # Make fully qualified package names prepending 'IPython.' to our name lists
444 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
451 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
445
452
446 # Make runners
453 # Make runners
447 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
454 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
448
455
449 return runners
456 return runners
450
457
451
458
452 def run_iptest():
459 def run_iptest():
453 """Run the IPython test suite using nose.
460 """Run the IPython test suite using nose.
454
461
455 This function is called when this script is **not** called with the form
462 This function is called when this script is **not** called with the form
456 `iptest all`. It simply calls nose with appropriate command line flags
463 `iptest all`. It simply calls nose with appropriate command line flags
457 and accepts all of the standard nose arguments.
464 and accepts all of the standard nose arguments.
458 """
465 """
459 # Apply our monkeypatch to Xunit
466 # Apply our monkeypatch to Xunit
460 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
467 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
461 monkeypatch_xunit()
468 monkeypatch_xunit()
462
469
463 warnings.filterwarnings('ignore',
470 warnings.filterwarnings('ignore',
464 'This will be removed soon. Use IPython.testing.util instead')
471 'This will be removed soon. Use IPython.testing.util instead')
465
472
466 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
473 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
467
474
468 '--with-ipdoctest',
475 '--with-ipdoctest',
469 '--ipdoctest-tests','--ipdoctest-extension=txt',
476 '--ipdoctest-tests','--ipdoctest-extension=txt',
470
477
471 # We add --exe because of setuptools' imbecility (it
478 # We add --exe because of setuptools' imbecility (it
472 # blindly does chmod +x on ALL files). Nose does the
479 # blindly does chmod +x on ALL files). Nose does the
473 # right thing and it tries to avoid executables,
480 # right thing and it tries to avoid executables,
474 # setuptools unfortunately forces our hand here. This
481 # setuptools unfortunately forces our hand here. This
475 # has been discussed on the distutils list and the
482 # has been discussed on the distutils list and the
476 # setuptools devs refuse to fix this problem!
483 # setuptools devs refuse to fix this problem!
477 '--exe',
484 '--exe',
478 ]
485 ]
479
486
480 if nose.__version__ >= '0.11':
487 if nose.__version__ >= '0.11':
481 # I don't fully understand why we need this one, but depending on what
488 # I don't fully understand why we need this one, but depending on what
482 # directory the test suite is run from, if we don't give it, 0 tests
489 # directory the test suite is run from, if we don't give it, 0 tests
483 # get run. Specifically, if the test suite is run from the source dir
490 # get run. Specifically, if the test suite is run from the source dir
484 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
491 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
485 # even if the same call done in this directory works fine). It appears
492 # even if the same call done in this directory works fine). It appears
486 # that if the requested package is in the current dir, nose bails early
493 # that if the requested package is in the current dir, nose bails early
487 # by default. Since it's otherwise harmless, leave it in by default
494 # by default. Since it's otherwise harmless, leave it in by default
488 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
495 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
489 argv.append('--traverse-namespace')
496 argv.append('--traverse-namespace')
490
497
491 # use our plugin for doctesting. It will remove the standard doctest plugin
498 # use our plugin for doctesting. It will remove the standard doctest plugin
492 # if it finds it enabled
499 # if it finds it enabled
493 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
500 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
494 # We need a global ipython running in this process
501 # We need a global ipython running in this process
495 globalipapp.start_ipython()
502 globalipapp.start_ipython()
496 # Now nose can run
503 # Now nose can run
497 TestProgram(argv=argv, addplugins=plugins)
504 TestProgram(argv=argv, addplugins=plugins)
498
505
499
506
500 def run_iptestall(inc_slow=False):
507 def run_iptestall(inc_slow=False):
501 """Run the entire IPython test suite by calling nose and trial.
508 """Run the entire IPython test suite by calling nose and trial.
502
509
503 This function constructs :class:`IPTester` instances for all IPython
510 This function constructs :class:`IPTester` instances for all IPython
504 modules and package and then runs each of them. This causes the modules
511 modules and package and then runs each of them. This causes the modules
505 and packages of IPython to be tested each in their own subprocess using
512 and packages of IPython to be tested each in their own subprocess using
506 nose.
513 nose.
507
514
508 Parameters
515 Parameters
509 ----------
516 ----------
510
517
511 inc_slow : bool, optional
518 inc_slow : bool, optional
512 Include slow tests, like IPython.parallel. By default, these tests aren't
519 Include slow tests, like IPython.parallel. By default, these tests aren't
513 run.
520 run.
514 """
521 """
515
522
516 runners = make_runners(inc_slow=inc_slow)
523 runners = make_runners(inc_slow=inc_slow)
517
524
518 # Run the test runners in a temporary dir so we can nuke it when finished
525 # Run the test runners in a temporary dir so we can nuke it when finished
519 # to clean up any junk files left over by accident. This also makes it
526 # to clean up any junk files left over by accident. This also makes it
520 # robust against being run in non-writeable directories by mistake, as the
527 # robust against being run in non-writeable directories by mistake, as the
521 # temp dir will always be user-writeable.
528 # temp dir will always be user-writeable.
522 curdir = os.getcwdu()
529 curdir = os.getcwdu()
523 testdir = tempfile.gettempdir()
530 testdir = tempfile.gettempdir()
524 os.chdir(testdir)
531 os.chdir(testdir)
525
532
526 # Run all test runners, tracking execution time
533 # Run all test runners, tracking execution time
527 failed = []
534 failed = []
528 t_start = time.time()
535 t_start = time.time()
529 try:
536 try:
530 for (name, runner) in runners:
537 for (name, runner) in runners:
531 print('*'*70)
538 print('*'*70)
532 print('IPython test group:',name)
539 print('IPython test group:',name)
533 res = runner.run()
540 res = runner.run()
534 if res:
541 if res:
535 failed.append( (name, runner) )
542 failed.append( (name, runner) )
543 if res == -signal.SIGINT:
544 print("Interrupted")
545 break
536 finally:
546 finally:
537 os.chdir(curdir)
547 os.chdir(curdir)
538 t_end = time.time()
548 t_end = time.time()
539 t_tests = t_end - t_start
549 t_tests = t_end - t_start
540 nrunners = len(runners)
550 nrunners = len(runners)
541 nfail = len(failed)
551 nfail = len(failed)
542 # summarize results
552 # summarize results
543 print()
553 print()
544 print('*'*70)
554 print('*'*70)
545 print('Test suite completed for system with the following information:')
555 print('Test suite completed for system with the following information:')
546 print(report())
556 print(report())
547 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
557 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
548 print()
558 print()
549 print('Status:')
559 print('Status:')
550 if not failed:
560 if not failed:
551 print('OK')
561 print('OK')
552 else:
562 else:
553 # If anything went wrong, point out what command to rerun manually to
563 # If anything went wrong, point out what command to rerun manually to
554 # see the actual errors and individual summary
564 # see the actual errors and individual summary
555 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
565 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
556 for name, failed_runner in failed:
566 for name, failed_runner in failed:
557 print('-'*40)
567 print('-'*40)
558 print('Runner failed:',name)
568 print('Runner failed:',name)
559 print('You may wish to rerun this one individually, with:')
569 print('You may wish to rerun this one individually, with:')
560 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
570 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
561 print(u' '.join(failed_call_args))
571 print(u' '.join(failed_call_args))
562 print()
572 print()
563 # Ensure that our exit code indicates failure
573 # Ensure that our exit code indicates failure
564 sys.exit(1)
574 sys.exit(1)
565
575
566
576
567 def main():
577 def main():
568 for arg in sys.argv[1:]:
578 for arg in sys.argv[1:]:
569 if arg.startswith('IPython'):
579 if arg.startswith('IPython'):
570 # This is in-process
580 # This is in-process
571 run_iptest()
581 run_iptest()
572 else:
582 else:
573 if "--all" in sys.argv:
583 if "--all" in sys.argv:
574 sys.argv.remove("--all")
584 sys.argv.remove("--all")
575 inc_slow = True
585 inc_slow = True
576 else:
586 else:
577 inc_slow = False
587 inc_slow = False
578 # This starts subprocesses
588 # This starts subprocesses
579 run_iptestall(inc_slow=inc_slow)
589 run_iptestall(inc_slow=inc_slow)
580
590
581
591
582 if __name__ == '__main__':
592 if __name__ == '__main__':
583 main()
593 main()
General Comments 0
You need to be logged in to leave comments. Login now