##// END OF EJS Templates
Exclude slow tests (IPython.parallel by default in iptest
Thomas Kluyver -
Show More
@@ -1,29 +1,29 b''
1 """Testing support (tools to test IPython itself).
1 """Testing support (tools to test IPython itself).
2 """
2 """
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2009-2011 The IPython Development Team
5 # Copyright (C) 2009-2011 The IPython Development Team
6 #
6 #
7 # Distributed under the terms of the BSD License. The full license is in
7 # Distributed under the terms of the BSD License. The full license is in
8 # the file COPYING, distributed as part of this software.
8 # the file COPYING, distributed as part of this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Functions
12 # Functions
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 # User-level entry point for testing
15 # User-level entry point for testing
16 def test():
16 def test(all=False):
17 """Run the entire IPython test suite.
17 """Run the entire IPython test suite.
18
18
19 For fine-grained control, you should use the :file:`iptest` script supplied
19 For fine-grained control, you should use the :file:`iptest` script supplied
20 with the IPython installation."""
20 with the IPython installation."""
21
21
22 # Do the import internally, so that this function doesn't increase total
22 # Do the import internally, so that this function doesn't increase total
23 # import time
23 # import time
24 from iptest import run_iptestall
24 from iptest import run_iptestall
25 run_iptestall()
25 run_iptestall(inc_slow=all)
26
26
27 # So nose doesn't try to run this as a test itself and we end up with an
27 # So nose doesn't try to run this as a test itself and we end up with an
28 # infinite test loop
28 # infinite test loop
29 test.__test__ = False
29 test.__test__ = False
@@ -1,566 +1,579 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 from __future__ import print_function
27 from __future__ import print_function
28
28
29 # Stdlib
29 # Stdlib
30 import glob
30 import glob
31 import os
31 import os
32 import os.path as path
32 import os.path as path
33 import signal
33 import signal
34 import sys
34 import sys
35 import subprocess
35 import subprocess
36 import tempfile
36 import tempfile
37 import time
37 import time
38 import warnings
38 import warnings
39
39
40 # Note: monkeypatch!
40 # Note: monkeypatch!
41 # We need to monkeypatch a small problem in nose itself first, before importing
41 # We need to monkeypatch a small problem in nose itself first, before importing
42 # it for actual use. This should get into nose upstream, but its release cycle
42 # it for actual use. This should get into nose upstream, but its release cycle
43 # is slow and we need it for our parametric tests to work correctly.
43 # is slow and we need it for our parametric tests to work correctly.
44 from IPython.testing import nosepatch
44 from IPython.testing import nosepatch
45
45
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
47 # This can be dropped once we no longer test on Python 2.6
47 # This can be dropped once we no longer test on Python 2.6
48 from IPython.testing import nose_assert_methods
48 from IPython.testing import nose_assert_methods
49
49
50 # Now, proceed to import nose itself
50 # Now, proceed to import nose itself
51 import nose.plugins.builtin
51 import nose.plugins.builtin
52 from nose.plugins.xunit import Xunit
52 from nose.plugins.xunit import Xunit
53 from nose import SkipTest
53 from nose import SkipTest
54 from nose.core import TestProgram
54 from nose.core import TestProgram
55
55
56 # Our own imports
56 # Our own imports
57 from IPython.utils import py3compat
57 from IPython.utils import py3compat
58 from IPython.utils.importstring import import_item
58 from IPython.utils.importstring import import_item
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
60 from IPython.utils.process import find_cmd, pycmd2argv
60 from IPython.utils.process import find_cmd, pycmd2argv
61 from IPython.utils.sysinfo import sys_info
61 from IPython.utils.sysinfo import sys_info
62 from IPython.utils.tempdir import TemporaryDirectory
62 from IPython.utils.tempdir import TemporaryDirectory
63 from IPython.utils.warn import warn
63 from IPython.utils.warn import warn
64
64
65 from IPython.testing import globalipapp
65 from IPython.testing import globalipapp
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
67 from IPython.external.decorators import KnownFailure, knownfailureif
67 from IPython.external.decorators import KnownFailure, knownfailureif
68
68
69 pjoin = path.join
69 pjoin = path.join
70
70
71
71
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73 # Globals
73 # Globals
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75
75
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Warnings control
78 # Warnings control
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
82 # that imports 'sets' as of today
82 # that imports 'sets' as of today
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
84 DeprecationWarning )
84 DeprecationWarning )
85
85
86 # This one also comes from Twisted
86 # This one also comes from Twisted
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
88 DeprecationWarning)
88 DeprecationWarning)
89
89
90 # Wx on Fedora11 spits these out
90 # Wx on Fedora11 spits these out
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
92 UserWarning)
92 UserWarning)
93
93
94 # ------------------------------------------------------------------------------
94 # ------------------------------------------------------------------------------
95 # Monkeypatch Xunit to count known failures as skipped.
95 # Monkeypatch Xunit to count known failures as skipped.
96 # ------------------------------------------------------------------------------
96 # ------------------------------------------------------------------------------
97 def monkeypatch_xunit():
97 def monkeypatch_xunit():
98 try:
98 try:
99 knownfailureif(True)(lambda: None)()
99 knownfailureif(True)(lambda: None)()
100 except Exception as e:
100 except Exception as e:
101 KnownFailureTest = type(e)
101 KnownFailureTest = type(e)
102
102
103 def addError(self, test, err, capt=None):
103 def addError(self, test, err, capt=None):
104 if issubclass(err[0], KnownFailureTest):
104 if issubclass(err[0], KnownFailureTest):
105 err = (SkipTest,) + err[1:]
105 err = (SkipTest,) + err[1:]
106 return self.orig_addError(test, err, capt)
106 return self.orig_addError(test, err, capt)
107
107
108 Xunit.orig_addError = Xunit.addError
108 Xunit.orig_addError = Xunit.addError
109 Xunit.addError = addError
109 Xunit.addError = addError
110
110
111 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
112 # Logic for skipping doctests
112 # Logic for skipping doctests
113 #-----------------------------------------------------------------------------
113 #-----------------------------------------------------------------------------
114 def extract_version(mod):
114 def extract_version(mod):
115 return mod.__version__
115 return mod.__version__
116
116
117 def test_for(item, min_version=None, callback=extract_version):
117 def test_for(item, min_version=None, callback=extract_version):
118 """Test to see if item is importable, and optionally check against a minimum
118 """Test to see if item is importable, and optionally check against a minimum
119 version.
119 version.
120
120
121 If min_version is given, the default behavior is to check against the
121 If min_version is given, the default behavior is to check against the
122 `__version__` attribute of the item, but specifying `callback` allows you to
122 `__version__` attribute of the item, but specifying `callback` allows you to
123 extract the value you are interested in. e.g::
123 extract the value you are interested in. e.g::
124
124
125 In [1]: import sys
125 In [1]: import sys
126
126
127 In [2]: from IPython.testing.iptest import test_for
127 In [2]: from IPython.testing.iptest import test_for
128
128
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
130 Out[3]: True
130 Out[3]: True
131
131
132 """
132 """
133 try:
133 try:
134 check = import_item(item)
134 check = import_item(item)
135 except (ImportError, RuntimeError):
135 except (ImportError, RuntimeError):
136 # GTK reports Runtime error if it can't be initialized even if it's
136 # GTK reports Runtime error if it can't be initialized even if it's
137 # importable.
137 # importable.
138 return False
138 return False
139 else:
139 else:
140 if min_version:
140 if min_version:
141 if callback:
141 if callback:
142 # extra processing step to get version to compare
142 # extra processing step to get version to compare
143 check = callback(check)
143 check = callback(check)
144
144
145 return check >= min_version
145 return check >= min_version
146 else:
146 else:
147 return True
147 return True
148
148
149 # Global dict where we can store information on what we have and what we don't
149 # Global dict where we can store information on what we have and what we don't
150 # have available at test run time
150 # have available at test run time
151 have = {}
151 have = {}
152
152
153 have['curses'] = test_for('_curses')
153 have['curses'] = test_for('_curses')
154 have['matplotlib'] = test_for('matplotlib')
154 have['matplotlib'] = test_for('matplotlib')
155 have['numpy'] = test_for('numpy')
155 have['numpy'] = test_for('numpy')
156 have['pexpect'] = test_for('IPython.external.pexpect')
156 have['pexpect'] = test_for('IPython.external.pexpect')
157 have['pymongo'] = test_for('pymongo')
157 have['pymongo'] = test_for('pymongo')
158 have['pygments'] = test_for('pygments')
158 have['pygments'] = test_for('pygments')
159 have['qt'] = test_for('IPython.external.qt')
159 have['qt'] = test_for('IPython.external.qt')
160 have['rpy2'] = test_for('rpy2')
160 have['rpy2'] = test_for('rpy2')
161 have['sqlite3'] = test_for('sqlite3')
161 have['sqlite3'] = test_for('sqlite3')
162 have['cython'] = test_for('Cython')
162 have['cython'] = test_for('Cython')
163 have['oct2py'] = test_for('oct2py')
163 have['oct2py'] = test_for('oct2py')
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
165 have['wx'] = test_for('wx')
165 have['wx'] = test_for('wx')
166 have['wx.aui'] = test_for('wx.aui')
166 have['wx.aui'] = test_for('wx.aui')
167
167
168 if os.name == 'nt':
168 if os.name == 'nt':
169 min_zmq = (2,1,7)
169 min_zmq = (2,1,7)
170 else:
170 else:
171 min_zmq = (2,1,4)
171 min_zmq = (2,1,4)
172
172
173 def version_tuple(mod):
173 def version_tuple(mod):
174 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
174 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
175 # turn 'dev' into 999, because Python3 rejects str-int comparisons
175 # turn 'dev' into 999, because Python3 rejects str-int comparisons
176 vs = mod.__version__.replace('dev', '.999')
176 vs = mod.__version__.replace('dev', '.999')
177 tup = tuple([int(v) for v in vs.split('.') ])
177 tup = tuple([int(v) for v in vs.split('.') ])
178 return tup
178 return tup
179
179
180 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
180 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
181
181
182 #-----------------------------------------------------------------------------
182 #-----------------------------------------------------------------------------
183 # Functions and classes
183 # Functions and classes
184 #-----------------------------------------------------------------------------
184 #-----------------------------------------------------------------------------
185
185
186 def report():
186 def report():
187 """Return a string with a summary report of test-related variables."""
187 """Return a string with a summary report of test-related variables."""
188
188
189 out = [ sys_info(), '\n']
189 out = [ sys_info(), '\n']
190
190
191 avail = []
191 avail = []
192 not_avail = []
192 not_avail = []
193
193
194 for k, is_avail in have.items():
194 for k, is_avail in have.items():
195 if is_avail:
195 if is_avail:
196 avail.append(k)
196 avail.append(k)
197 else:
197 else:
198 not_avail.append(k)
198 not_avail.append(k)
199
199
200 if avail:
200 if avail:
201 out.append('\nTools and libraries available at test time:\n')
201 out.append('\nTools and libraries available at test time:\n')
202 avail.sort()
202 avail.sort()
203 out.append(' ' + ' '.join(avail)+'\n')
203 out.append(' ' + ' '.join(avail)+'\n')
204
204
205 if not_avail:
205 if not_avail:
206 out.append('\nTools and libraries NOT available at test time:\n')
206 out.append('\nTools and libraries NOT available at test time:\n')
207 not_avail.sort()
207 not_avail.sort()
208 out.append(' ' + ' '.join(not_avail)+'\n')
208 out.append(' ' + ' '.join(not_avail)+'\n')
209
209
210 return ''.join(out)
210 return ''.join(out)
211
211
212
212
213 def make_exclude():
213 def make_exclude():
214 """Make patterns of modules and packages to exclude from testing.
214 """Make patterns of modules and packages to exclude from testing.
215
215
216 For the IPythonDoctest plugin, we need to exclude certain patterns that
216 For the IPythonDoctest plugin, we need to exclude certain patterns that
217 cause testing problems. We should strive to minimize the number of
217 cause testing problems. We should strive to minimize the number of
218 skipped modules, since this means untested code.
218 skipped modules, since this means untested code.
219
219
220 These modules and packages will NOT get scanned by nose at all for tests.
220 These modules and packages will NOT get scanned by nose at all for tests.
221 """
221 """
222 # Simple utility to make IPython paths more readably, we need a lot of
222 # Simple utility to make IPython paths more readably, we need a lot of
223 # these below
223 # these below
224 ipjoin = lambda *paths: pjoin('IPython', *paths)
224 ipjoin = lambda *paths: pjoin('IPython', *paths)
225
225
226 exclusions = [ipjoin('external'),
226 exclusions = [ipjoin('external'),
227 ipjoin('quarantine'),
227 ipjoin('quarantine'),
228 ipjoin('deathrow'),
228 ipjoin('deathrow'),
229 # This guy is probably attic material
229 # This guy is probably attic material
230 ipjoin('testing', 'mkdoctests'),
230 ipjoin('testing', 'mkdoctests'),
231 # Testing inputhook will need a lot of thought, to figure out
231 # Testing inputhook will need a lot of thought, to figure out
232 # how to have tests that don't lock up with the gui event
232 # how to have tests that don't lock up with the gui event
233 # loops in the picture
233 # loops in the picture
234 ipjoin('lib', 'inputhook'),
234 ipjoin('lib', 'inputhook'),
235 # Config files aren't really importable stand-alone
235 # Config files aren't really importable stand-alone
236 ipjoin('config', 'profile'),
236 ipjoin('config', 'profile'),
237 # The notebook 'static' directory contains JS, css and other
237 # The notebook 'static' directory contains JS, css and other
238 # files for web serving. Occasionally projects may put a .py
238 # files for web serving. Occasionally projects may put a .py
239 # file in there (MathJax ships a conf.py), so we might as
239 # file in there (MathJax ships a conf.py), so we might as
240 # well play it safe and skip the whole thing.
240 # well play it safe and skip the whole thing.
241 ipjoin('frontend', 'html', 'notebook', 'static')
241 ipjoin('frontend', 'html', 'notebook', 'static')
242 ]
242 ]
243 if not have['sqlite3']:
243 if not have['sqlite3']:
244 exclusions.append(ipjoin('core', 'tests', 'test_history'))
244 exclusions.append(ipjoin('core', 'tests', 'test_history'))
245 exclusions.append(ipjoin('core', 'history'))
245 exclusions.append(ipjoin('core', 'history'))
246 if not have['wx']:
246 if not have['wx']:
247 exclusions.append(ipjoin('lib', 'inputhookwx'))
247 exclusions.append(ipjoin('lib', 'inputhookwx'))
248
248
249 # FIXME: temporarily disable autoreload tests, as they can produce
249 # FIXME: temporarily disable autoreload tests, as they can produce
250 # spurious failures in subsequent tests (cythonmagic).
250 # spurious failures in subsequent tests (cythonmagic).
251 exclusions.append(ipjoin('extensions', 'autoreload'))
251 exclusions.append(ipjoin('extensions', 'autoreload'))
252 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
252 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
253
253
254 # We do this unconditionally, so that the test suite doesn't import
254 # We do this unconditionally, so that the test suite doesn't import
255 # gtk, changing the default encoding and masking some unicode bugs.
255 # gtk, changing the default encoding and masking some unicode bugs.
256 exclusions.append(ipjoin('lib', 'inputhookgtk'))
256 exclusions.append(ipjoin('lib', 'inputhookgtk'))
257 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
257 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
258
258
259 # These have to be skipped on win32 because the use echo, rm, cd, etc.
259 # These have to be skipped on win32 because the use echo, rm, cd, etc.
260 # See ticket https://github.com/ipython/ipython/issues/87
260 # See ticket https://github.com/ipython/ipython/issues/87
261 if sys.platform == 'win32':
261 if sys.platform == 'win32':
262 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
262 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
263 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
263 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
264
264
265 if not have['pexpect']:
265 if not have['pexpect']:
266 exclusions.extend([ipjoin('lib', 'irunner'),
266 exclusions.extend([ipjoin('lib', 'irunner'),
267 ipjoin('lib', 'tests', 'test_irunner'),
267 ipjoin('lib', 'tests', 'test_irunner'),
268 ipjoin('frontend', 'terminal', 'console'),
268 ipjoin('frontend', 'terminal', 'console'),
269 ])
269 ])
270
270
271 if not have['zmq']:
271 if not have['zmq']:
272 exclusions.append(ipjoin('zmq'))
272 exclusions.append(ipjoin('zmq'))
273 exclusions.append(ipjoin('frontend', 'qt'))
273 exclusions.append(ipjoin('frontend', 'qt'))
274 exclusions.append(ipjoin('frontend', 'html'))
274 exclusions.append(ipjoin('frontend', 'html'))
275 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
275 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
276 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
276 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
277 exclusions.append(ipjoin('parallel'))
277 exclusions.append(ipjoin('parallel'))
278 elif not have['qt'] or not have['pygments']:
278 elif not have['qt'] or not have['pygments']:
279 exclusions.append(ipjoin('frontend', 'qt'))
279 exclusions.append(ipjoin('frontend', 'qt'))
280
280
281 if not have['pymongo']:
281 if not have['pymongo']:
282 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
282 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
283 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
283 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
284
284
285 if not have['matplotlib']:
285 if not have['matplotlib']:
286 exclusions.extend([ipjoin('core', 'pylabtools'),
286 exclusions.extend([ipjoin('core', 'pylabtools'),
287 ipjoin('core', 'tests', 'test_pylabtools'),
287 ipjoin('core', 'tests', 'test_pylabtools'),
288 ipjoin('zmq', 'pylab'),
288 ipjoin('zmq', 'pylab'),
289 ])
289 ])
290
290
291 if not have['cython']:
291 if not have['cython']:
292 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
292 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
293 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
293 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
294
294
295 if not have['oct2py']:
295 if not have['oct2py']:
296 exclusions.extend([ipjoin('extensions', 'octavemagic')])
296 exclusions.extend([ipjoin('extensions', 'octavemagic')])
297 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
297 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
298
298
299 if not have['tornado']:
299 if not have['tornado']:
300 exclusions.append(ipjoin('frontend', 'html'))
300 exclusions.append(ipjoin('frontend', 'html'))
301
301
302 if not have['rpy2'] or not have['numpy']:
302 if not have['rpy2'] or not have['numpy']:
303 exclusions.append(ipjoin('extensions', 'rmagic'))
303 exclusions.append(ipjoin('extensions', 'rmagic'))
304 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
304 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
305
305
306 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
306 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
307 if sys.platform == 'win32':
307 if sys.platform == 'win32':
308 exclusions = [s.replace('\\','\\\\') for s in exclusions]
308 exclusions = [s.replace('\\','\\\\') for s in exclusions]
309
309
310 # check for any exclusions that don't seem to exist:
310 # check for any exclusions that don't seem to exist:
311 parent, _ = os.path.split(get_ipython_package_dir())
311 parent, _ = os.path.split(get_ipython_package_dir())
312 for exclusion in exclusions:
312 for exclusion in exclusions:
313 if exclusion.endswith(('deathrow', 'quarantine')):
313 if exclusion.endswith(('deathrow', 'quarantine')):
314 # ignore deathrow/quarantine, which exist in dev, but not install
314 # ignore deathrow/quarantine, which exist in dev, but not install
315 continue
315 continue
316 fullpath = pjoin(parent, exclusion)
316 fullpath = pjoin(parent, exclusion)
317 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
317 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
318 warn("Excluding nonexistent file: %r\n" % exclusion)
318 warn("Excluding nonexistent file: %r\n" % exclusion)
319
319
320 return exclusions
320 return exclusions
321
321
322
322
323 class IPTester(object):
323 class IPTester(object):
324 """Call that calls iptest or trial in a subprocess.
324 """Call that calls iptest or trial in a subprocess.
325 """
325 """
326 #: string, name of test runner that will be called
326 #: string, name of test runner that will be called
327 runner = None
327 runner = None
328 #: list, parameters for test runner
328 #: list, parameters for test runner
329 params = None
329 params = None
330 #: list, arguments of system call to be made to call test runner
330 #: list, arguments of system call to be made to call test runner
331 call_args = None
331 call_args = None
332 #: list, subprocesses we start (for cleanup)
332 #: list, subprocesses we start (for cleanup)
333 processes = None
333 processes = None
334 #: str, coverage xml output file
334 #: str, coverage xml output file
335 coverage_xml = None
335 coverage_xml = None
336
336
337 def __init__(self, runner='iptest', params=None):
337 def __init__(self, runner='iptest', params=None):
338 """Create new test runner."""
338 """Create new test runner."""
339 p = os.path
339 p = os.path
340 if runner == 'iptest':
340 if runner == 'iptest':
341 iptest_app = get_ipython_module_path('IPython.testing.iptest')
341 iptest_app = get_ipython_module_path('IPython.testing.iptest')
342 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
342 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
343 else:
343 else:
344 raise Exception('Not a valid test runner: %s' % repr(runner))
344 raise Exception('Not a valid test runner: %s' % repr(runner))
345 if params is None:
345 if params is None:
346 params = []
346 params = []
347 if isinstance(params, str):
347 if isinstance(params, str):
348 params = [params]
348 params = [params]
349 self.params = params
349 self.params = params
350
350
351 # Assemble call
351 # Assemble call
352 self.call_args = self.runner+self.params
352 self.call_args = self.runner+self.params
353
353
354 # Find the section we're testing (IPython.foo)
354 # Find the section we're testing (IPython.foo)
355 for sect in self.params:
355 for sect in self.params:
356 if sect.startswith('IPython'): break
356 if sect.startswith('IPython'): break
357 else:
357 else:
358 raise ValueError("Section not found", self.params)
358 raise ValueError("Section not found", self.params)
359
359
360 if '--with-xunit' in self.call_args:
360 if '--with-xunit' in self.call_args:
361
361
362 self.call_args.append('--xunit-file')
362 self.call_args.append('--xunit-file')
363 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
363 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
364 xunit_file = path.abspath(sect+'.xunit.xml')
364 xunit_file = path.abspath(sect+'.xunit.xml')
365 if sys.platform == 'win32':
365 if sys.platform == 'win32':
366 xunit_file = '"%s"' % xunit_file
366 xunit_file = '"%s"' % xunit_file
367 self.call_args.append(xunit_file)
367 self.call_args.append(xunit_file)
368
368
369 if '--with-xml-coverage' in self.call_args:
369 if '--with-xml-coverage' in self.call_args:
370 self.coverage_xml = path.abspath(sect+".coverage.xml")
370 self.coverage_xml = path.abspath(sect+".coverage.xml")
371 self.call_args.remove('--with-xml-coverage')
371 self.call_args.remove('--with-xml-coverage')
372 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
372 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
373
373
374 # Store anything we start to clean up on deletion
374 # Store anything we start to clean up on deletion
375 self.processes = []
375 self.processes = []
376
376
377 def _run_cmd(self):
377 def _run_cmd(self):
378 with TemporaryDirectory() as IPYTHONDIR:
378 with TemporaryDirectory() as IPYTHONDIR:
379 env = os.environ.copy()
379 env = os.environ.copy()
380 env['IPYTHONDIR'] = IPYTHONDIR
380 env['IPYTHONDIR'] = IPYTHONDIR
381 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
381 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
382 subp = subprocess.Popen(self.call_args, env=env)
382 subp = subprocess.Popen(self.call_args, env=env)
383 self.processes.append(subp)
383 self.processes.append(subp)
384 # If this fails, the process will be left in self.processes and
384 # If this fails, the process will be left in self.processes and
385 # cleaned up later, but if the wait call succeeds, then we can
385 # cleaned up later, but if the wait call succeeds, then we can
386 # clear the stored process.
386 # clear the stored process.
387 retcode = subp.wait()
387 retcode = subp.wait()
388 self.processes.pop()
388 self.processes.pop()
389 return retcode
389 return retcode
390
390
391 def run(self):
391 def run(self):
392 """Run the stored commands"""
392 """Run the stored commands"""
393 try:
393 try:
394 retcode = self._run_cmd()
394 retcode = self._run_cmd()
395 except:
395 except:
396 import traceback
396 import traceback
397 traceback.print_exc()
397 traceback.print_exc()
398 return 1 # signal failure
398 return 1 # signal failure
399
399
400 if self.coverage_xml:
400 if self.coverage_xml:
401 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
401 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
402 return retcode
402 return retcode
403
403
404 def __del__(self):
404 def __del__(self):
405 """Cleanup on exit by killing any leftover processes."""
405 """Cleanup on exit by killing any leftover processes."""
406 for subp in self.processes:
406 for subp in self.processes:
407 if subp.poll() is not None:
407 if subp.poll() is not None:
408 continue # process is already dead
408 continue # process is already dead
409
409
410 try:
410 try:
411 print('Cleaning stale PID: %d' % subp.pid)
411 print('Cleaning stale PID: %d' % subp.pid)
412 subp.kill()
412 subp.kill()
413 except: # (OSError, WindowsError) ?
413 except: # (OSError, WindowsError) ?
414 # This is just a best effort, if we fail or the process was
414 # This is just a best effort, if we fail or the process was
415 # really gone, ignore it.
415 # really gone, ignore it.
416 pass
416 pass
417
417
418 if subp.poll() is None:
418 if subp.poll() is None:
419 # The process did not die...
419 # The process did not die...
420 print('... failed. Manual cleanup may be required.'
420 print('... failed. Manual cleanup may be required.'
421 % subp.pid)
421 % subp.pid)
422
422
423 def make_runners():
423 def make_runners(inc_slow=False):
424 """Define the top-level packages that need to be tested.
424 """Define the top-level packages that need to be tested.
425 """
425 """
426
426
427 # Packages to be tested via nose, that only depend on the stdlib
427 # Packages to be tested via nose, that only depend on the stdlib
428 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
428 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
429 'testing', 'utils', 'nbformat' ]
429 'testing', 'utils', 'nbformat' ]
430
430
431 if have['zmq']:
431 if have['zmq']:
432 nose_pkg_names.append('zmq')
432 nose_pkg_names.append('zmq')
433 nose_pkg_names.append('parallel')
433 if inc_slow:
434 nose_pkg_names.append('parallel')
434
435
435 # For debugging this code, only load quick stuff
436 # For debugging this code, only load quick stuff
436 #nose_pkg_names = ['core', 'extensions'] # dbg
437 #nose_pkg_names = ['core', 'extensions'] # dbg
437
438
438 # Make fully qualified package names prepending 'IPython.' to our name lists
439 # Make fully qualified package names prepending 'IPython.' to our name lists
439 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
440 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
440
441
441 # Make runners
442 # Make runners
442 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
443 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
443
444
444 return runners
445 return runners
445
446
446
447
447 def run_iptest():
448 def run_iptest():
448 """Run the IPython test suite using nose.
449 """Run the IPython test suite using nose.
449
450
450 This function is called when this script is **not** called with the form
451 This function is called when this script is **not** called with the form
451 `iptest all`. It simply calls nose with appropriate command line flags
452 `iptest all`. It simply calls nose with appropriate command line flags
452 and accepts all of the standard nose arguments.
453 and accepts all of the standard nose arguments.
453 """
454 """
454 # Apply our monkeypatch to Xunit
455 # Apply our monkeypatch to Xunit
455 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
456 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
456 monkeypatch_xunit()
457 monkeypatch_xunit()
457
458
458 warnings.filterwarnings('ignore',
459 warnings.filterwarnings('ignore',
459 'This will be removed soon. Use IPython.testing.util instead')
460 'This will be removed soon. Use IPython.testing.util instead')
460
461
461 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
462 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
462
463
463 '--with-ipdoctest',
464 '--with-ipdoctest',
464 '--ipdoctest-tests','--ipdoctest-extension=txt',
465 '--ipdoctest-tests','--ipdoctest-extension=txt',
465
466
466 # We add --exe because of setuptools' imbecility (it
467 # We add --exe because of setuptools' imbecility (it
467 # blindly does chmod +x on ALL files). Nose does the
468 # blindly does chmod +x on ALL files). Nose does the
468 # right thing and it tries to avoid executables,
469 # right thing and it tries to avoid executables,
469 # setuptools unfortunately forces our hand here. This
470 # setuptools unfortunately forces our hand here. This
470 # has been discussed on the distutils list and the
471 # has been discussed on the distutils list and the
471 # setuptools devs refuse to fix this problem!
472 # setuptools devs refuse to fix this problem!
472 '--exe',
473 '--exe',
473 ]
474 ]
474
475
475 if nose.__version__ >= '0.11':
476 if nose.__version__ >= '0.11':
476 # I don't fully understand why we need this one, but depending on what
477 # I don't fully understand why we need this one, but depending on what
477 # directory the test suite is run from, if we don't give it, 0 tests
478 # directory the test suite is run from, if we don't give it, 0 tests
478 # get run. Specifically, if the test suite is run from the source dir
479 # get run. Specifically, if the test suite is run from the source dir
479 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
480 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
480 # even if the same call done in this directory works fine). It appears
481 # even if the same call done in this directory works fine). It appears
481 # that if the requested package is in the current dir, nose bails early
482 # that if the requested package is in the current dir, nose bails early
482 # by default. Since it's otherwise harmless, leave it in by default
483 # by default. Since it's otherwise harmless, leave it in by default
483 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
484 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
484 argv.append('--traverse-namespace')
485 argv.append('--traverse-namespace')
485
486
486 # use our plugin for doctesting. It will remove the standard doctest plugin
487 # use our plugin for doctesting. It will remove the standard doctest plugin
487 # if it finds it enabled
488 # if it finds it enabled
488 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
489 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
489 # We need a global ipython running in this process
490 # We need a global ipython running in this process
490 globalipapp.start_ipython()
491 globalipapp.start_ipython()
491 # Now nose can run
492 # Now nose can run
492 TestProgram(argv=argv, addplugins=plugins)
493 TestProgram(argv=argv, addplugins=plugins)
493
494
494
495
495 def run_iptestall():
496 def run_iptestall(inc_slow=False):
496 """Run the entire IPython test suite by calling nose and trial.
497 """Run the entire IPython test suite by calling nose and trial.
497
498
498 This function constructs :class:`IPTester` instances for all IPython
499 This function constructs :class:`IPTester` instances for all IPython
499 modules and package and then runs each of them. This causes the modules
500 modules and package and then runs each of them. This causes the modules
500 and packages of IPython to be tested each in their own subprocess using
501 and packages of IPython to be tested each in their own subprocess using
501 nose.
502 nose.
503
504 Parameters
505 ----------
506
507 inc_slow : bool, optional
508 Include slow tests, like IPython.parallel. By default, these tests aren't
509 run.
502 """
510 """
503
511
504 runners = make_runners()
512 runners = make_runners(inc_slow=inc_slow)
505
513
506 # Run the test runners in a temporary dir so we can nuke it when finished
514 # Run the test runners in a temporary dir so we can nuke it when finished
507 # to clean up any junk files left over by accident. This also makes it
515 # to clean up any junk files left over by accident. This also makes it
508 # robust against being run in non-writeable directories by mistake, as the
516 # robust against being run in non-writeable directories by mistake, as the
509 # temp dir will always be user-writeable.
517 # temp dir will always be user-writeable.
510 curdir = os.getcwdu()
518 curdir = os.getcwdu()
511 testdir = tempfile.gettempdir()
519 testdir = tempfile.gettempdir()
512 os.chdir(testdir)
520 os.chdir(testdir)
513
521
514 # Run all test runners, tracking execution time
522 # Run all test runners, tracking execution time
515 failed = []
523 failed = []
516 t_start = time.time()
524 t_start = time.time()
517 try:
525 try:
518 for (name, runner) in runners:
526 for (name, runner) in runners:
519 print('*'*70)
527 print('*'*70)
520 print('IPython test group:',name)
528 print('IPython test group:',name)
521 res = runner.run()
529 res = runner.run()
522 if res:
530 if res:
523 failed.append( (name, runner) )
531 failed.append( (name, runner) )
524 finally:
532 finally:
525 os.chdir(curdir)
533 os.chdir(curdir)
526 t_end = time.time()
534 t_end = time.time()
527 t_tests = t_end - t_start
535 t_tests = t_end - t_start
528 nrunners = len(runners)
536 nrunners = len(runners)
529 nfail = len(failed)
537 nfail = len(failed)
530 # summarize results
538 # summarize results
531 print()
539 print()
532 print('*'*70)
540 print('*'*70)
533 print('Test suite completed for system with the following information:')
541 print('Test suite completed for system with the following information:')
534 print(report())
542 print(report())
535 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
543 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
536 print()
544 print()
537 print('Status:')
545 print('Status:')
538 if not failed:
546 if not failed:
539 print('OK')
547 print('OK')
540 else:
548 else:
541 # If anything went wrong, point out what command to rerun manually to
549 # If anything went wrong, point out what command to rerun manually to
542 # see the actual errors and individual summary
550 # see the actual errors and individual summary
543 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
551 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
544 for name, failed_runner in failed:
552 for name, failed_runner in failed:
545 print('-'*40)
553 print('-'*40)
546 print('Runner failed:',name)
554 print('Runner failed:',name)
547 print('You may wish to rerun this one individually, with:')
555 print('You may wish to rerun this one individually, with:')
548 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
556 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
549 print(u' '.join(failed_call_args))
557 print(u' '.join(failed_call_args))
550 print()
558 print()
551 # Ensure that our exit code indicates failure
559 # Ensure that our exit code indicates failure
552 sys.exit(1)
560 sys.exit(1)
553
561
554
562
555 def main():
563 def main():
556 for arg in sys.argv[1:]:
564 for arg in sys.argv[1:]:
557 if arg.startswith('IPython'):
565 if arg.startswith('IPython'):
558 # This is in-process
566 # This is in-process
559 run_iptest()
567 run_iptest()
560 else:
568 else:
569 if "--all" in sys.argv:
570 sys.argv.remove("--all")
571 inc_slow = True
572 else:
573 inc_slow = False
561 # This starts subprocesses
574 # This starts subprocesses
562 run_iptestall()
575 run_iptestall(inc_slow=inc_slow)
563
576
564
577
565 if __name__ == '__main__':
578 if __name__ == '__main__':
566 main()
579 main()
General Comments 0
You need to be logged in to leave comments. Login now