##// END OF EJS Templates
exclude IPython.lib.kernel in iptest...
MinRK -
Show More
@@ -1,637 +1,638 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 from __future__ import print_function
27 from __future__ import print_function
28
28
29 # Stdlib
29 # Stdlib
30 import glob
30 import glob
31 import os
31 import os
32 import os.path as path
32 import os.path as path
33 import signal
33 import signal
34 import sys
34 import sys
35 import subprocess
35 import subprocess
36 import tempfile
36 import tempfile
37 import time
37 import time
38 import warnings
38 import warnings
39
39
40 # Note: monkeypatch!
40 # Note: monkeypatch!
41 # We need to monkeypatch a small problem in nose itself first, before importing
41 # We need to monkeypatch a small problem in nose itself first, before importing
42 # it for actual use. This should get into nose upstream, but its release cycle
42 # it for actual use. This should get into nose upstream, but its release cycle
43 # is slow and we need it for our parametric tests to work correctly.
43 # is slow and we need it for our parametric tests to work correctly.
44 from IPython.testing import nosepatch
44 from IPython.testing import nosepatch
45
45
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
46 # Monkeypatch extra assert methods into nose.tools if they're not already there.
47 # This can be dropped once we no longer test on Python 2.6
47 # This can be dropped once we no longer test on Python 2.6
48 from IPython.testing import nose_assert_methods
48 from IPython.testing import nose_assert_methods
49
49
50 # Now, proceed to import nose itself
50 # Now, proceed to import nose itself
51 import nose.plugins.builtin
51 import nose.plugins.builtin
52 from nose.plugins.xunit import Xunit
52 from nose.plugins.xunit import Xunit
53 from nose import SkipTest
53 from nose import SkipTest
54 from nose.core import TestProgram
54 from nose.core import TestProgram
55
55
56 # Our own imports
56 # Our own imports
57 from IPython.utils import py3compat
57 from IPython.utils import py3compat
58 from IPython.utils.importstring import import_item
58 from IPython.utils.importstring import import_item
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
59 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
60 from IPython.utils.process import pycmd2argv
60 from IPython.utils.process import pycmd2argv
61 from IPython.utils.sysinfo import sys_info
61 from IPython.utils.sysinfo import sys_info
62 from IPython.utils.tempdir import TemporaryDirectory
62 from IPython.utils.tempdir import TemporaryDirectory
63 from IPython.utils.warn import warn
63 from IPython.utils.warn import warn
64
64
65 from IPython.testing import globalipapp
65 from IPython.testing import globalipapp
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
66 from IPython.testing.plugin.ipdoctest import IPythonDoctest
67 from IPython.external.decorators import KnownFailure, knownfailureif
67 from IPython.external.decorators import KnownFailure, knownfailureif
68
68
69 pjoin = path.join
69 pjoin = path.join
70
70
71
71
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73 # Globals
73 # Globals
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75
75
76
76
77 #-----------------------------------------------------------------------------
77 #-----------------------------------------------------------------------------
78 # Warnings control
78 # Warnings control
79 #-----------------------------------------------------------------------------
79 #-----------------------------------------------------------------------------
80
80
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
81 # Twisted generates annoying warnings with Python 2.6, as will do other code
82 # that imports 'sets' as of today
82 # that imports 'sets' as of today
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
83 warnings.filterwarnings('ignore', 'the sets module is deprecated',
84 DeprecationWarning )
84 DeprecationWarning )
85
85
86 # This one also comes from Twisted
86 # This one also comes from Twisted
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
87 warnings.filterwarnings('ignore', 'the sha module is deprecated',
88 DeprecationWarning)
88 DeprecationWarning)
89
89
90 # Wx on Fedora11 spits these out
90 # Wx on Fedora11 spits these out
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
91 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
92 UserWarning)
92 UserWarning)
93
93
94 # ------------------------------------------------------------------------------
94 # ------------------------------------------------------------------------------
95 # Monkeypatch Xunit to count known failures as skipped.
95 # Monkeypatch Xunit to count known failures as skipped.
96 # ------------------------------------------------------------------------------
96 # ------------------------------------------------------------------------------
97 def monkeypatch_xunit():
97 def monkeypatch_xunit():
98 try:
98 try:
99 knownfailureif(True)(lambda: None)()
99 knownfailureif(True)(lambda: None)()
100 except Exception as e:
100 except Exception as e:
101 KnownFailureTest = type(e)
101 KnownFailureTest = type(e)
102
102
103 def addError(self, test, err, capt=None):
103 def addError(self, test, err, capt=None):
104 if issubclass(err[0], KnownFailureTest):
104 if issubclass(err[0], KnownFailureTest):
105 err = (SkipTest,) + err[1:]
105 err = (SkipTest,) + err[1:]
106 return self.orig_addError(test, err, capt)
106 return self.orig_addError(test, err, capt)
107
107
108 Xunit.orig_addError = Xunit.addError
108 Xunit.orig_addError = Xunit.addError
109 Xunit.addError = addError
109 Xunit.addError = addError
110
110
111 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
112 # Logic for skipping doctests
112 # Logic for skipping doctests
113 #-----------------------------------------------------------------------------
113 #-----------------------------------------------------------------------------
114 def extract_version(mod):
114 def extract_version(mod):
115 return mod.__version__
115 return mod.__version__
116
116
117 def test_for(item, min_version=None, callback=extract_version):
117 def test_for(item, min_version=None, callback=extract_version):
118 """Test to see if item is importable, and optionally check against a minimum
118 """Test to see if item is importable, and optionally check against a minimum
119 version.
119 version.
120
120
121 If min_version is given, the default behavior is to check against the
121 If min_version is given, the default behavior is to check against the
122 `__version__` attribute of the item, but specifying `callback` allows you to
122 `__version__` attribute of the item, but specifying `callback` allows you to
123 extract the value you are interested in. e.g::
123 extract the value you are interested in. e.g::
124
124
125 In [1]: import sys
125 In [1]: import sys
126
126
127 In [2]: from IPython.testing.iptest import test_for
127 In [2]: from IPython.testing.iptest import test_for
128
128
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
129 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
130 Out[3]: True
130 Out[3]: True
131
131
132 """
132 """
133 try:
133 try:
134 check = import_item(item)
134 check = import_item(item)
135 except (ImportError, RuntimeError):
135 except (ImportError, RuntimeError):
136 # GTK reports Runtime error if it can't be initialized even if it's
136 # GTK reports Runtime error if it can't be initialized even if it's
137 # importable.
137 # importable.
138 return False
138 return False
139 else:
139 else:
140 if min_version:
140 if min_version:
141 if callback:
141 if callback:
142 # extra processing step to get version to compare
142 # extra processing step to get version to compare
143 check = callback(check)
143 check = callback(check)
144
144
145 return check >= min_version
145 return check >= min_version
146 else:
146 else:
147 return True
147 return True
148
148
149 # Global dict where we can store information on what we have and what we don't
149 # Global dict where we can store information on what we have and what we don't
150 # have available at test run time
150 # have available at test run time
151 have = {}
151 have = {}
152
152
153 have['curses'] = test_for('_curses')
153 have['curses'] = test_for('_curses')
154 have['matplotlib'] = test_for('matplotlib')
154 have['matplotlib'] = test_for('matplotlib')
155 have['numpy'] = test_for('numpy')
155 have['numpy'] = test_for('numpy')
156 have['pexpect'] = test_for('IPython.external.pexpect')
156 have['pexpect'] = test_for('IPython.external.pexpect')
157 have['pymongo'] = test_for('pymongo')
157 have['pymongo'] = test_for('pymongo')
158 have['pygments'] = test_for('pygments')
158 have['pygments'] = test_for('pygments')
159 have['qt'] = test_for('IPython.external.qt')
159 have['qt'] = test_for('IPython.external.qt')
160 have['rpy2'] = test_for('rpy2')
160 have['rpy2'] = test_for('rpy2')
161 have['sqlite3'] = test_for('sqlite3')
161 have['sqlite3'] = test_for('sqlite3')
162 have['cython'] = test_for('Cython')
162 have['cython'] = test_for('Cython')
163 have['oct2py'] = test_for('oct2py')
163 have['oct2py'] = test_for('oct2py')
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
164 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
165 have['jinja2'] = test_for('jinja2')
165 have['jinja2'] = test_for('jinja2')
166 have['wx'] = test_for('wx')
166 have['wx'] = test_for('wx')
167 have['wx.aui'] = test_for('wx.aui')
167 have['wx.aui'] = test_for('wx.aui')
168 have['azure'] = test_for('azure')
168 have['azure'] = test_for('azure')
169 have['sphinx'] = test_for('sphinx')
169 have['sphinx'] = test_for('sphinx')
170
170
171 min_zmq = (2,1,11)
171 min_zmq = (2,1,11)
172
172
173 have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x())
173 have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x())
174
174
175 #-----------------------------------------------------------------------------
175 #-----------------------------------------------------------------------------
176 # Functions and classes
176 # Functions and classes
177 #-----------------------------------------------------------------------------
177 #-----------------------------------------------------------------------------
178
178
179 def report():
179 def report():
180 """Return a string with a summary report of test-related variables."""
180 """Return a string with a summary report of test-related variables."""
181
181
182 out = [ sys_info(), '\n']
182 out = [ sys_info(), '\n']
183
183
184 avail = []
184 avail = []
185 not_avail = []
185 not_avail = []
186
186
187 for k, is_avail in have.items():
187 for k, is_avail in have.items():
188 if is_avail:
188 if is_avail:
189 avail.append(k)
189 avail.append(k)
190 else:
190 else:
191 not_avail.append(k)
191 not_avail.append(k)
192
192
193 if avail:
193 if avail:
194 out.append('\nTools and libraries available at test time:\n')
194 out.append('\nTools and libraries available at test time:\n')
195 avail.sort()
195 avail.sort()
196 out.append(' ' + ' '.join(avail)+'\n')
196 out.append(' ' + ' '.join(avail)+'\n')
197
197
198 if not_avail:
198 if not_avail:
199 out.append('\nTools and libraries NOT available at test time:\n')
199 out.append('\nTools and libraries NOT available at test time:\n')
200 not_avail.sort()
200 not_avail.sort()
201 out.append(' ' + ' '.join(not_avail)+'\n')
201 out.append(' ' + ' '.join(not_avail)+'\n')
202
202
203 return ''.join(out)
203 return ''.join(out)
204
204
205
205
206 def make_exclude():
206 def make_exclude():
207 """Make patterns of modules and packages to exclude from testing.
207 """Make patterns of modules and packages to exclude from testing.
208
208
209 For the IPythonDoctest plugin, we need to exclude certain patterns that
209 For the IPythonDoctest plugin, we need to exclude certain patterns that
210 cause testing problems. We should strive to minimize the number of
210 cause testing problems. We should strive to minimize the number of
211 skipped modules, since this means untested code.
211 skipped modules, since this means untested code.
212
212
213 These modules and packages will NOT get scanned by nose at all for tests.
213 These modules and packages will NOT get scanned by nose at all for tests.
214 """
214 """
215 # Simple utility to make IPython paths more readably, we need a lot of
215 # Simple utility to make IPython paths more readably, we need a lot of
216 # these below
216 # these below
217 ipjoin = lambda *paths: pjoin('IPython', *paths)
217 ipjoin = lambda *paths: pjoin('IPython', *paths)
218
218
219 exclusions = [ipjoin('external'),
219 exclusions = [ipjoin('external'),
220 ipjoin('quarantine'),
220 ipjoin('quarantine'),
221 ipjoin('deathrow'),
221 ipjoin('deathrow'),
222 # This guy is probably attic material
222 # This guy is probably attic material
223 ipjoin('testing', 'mkdoctests'),
223 ipjoin('testing', 'mkdoctests'),
224 # Testing inputhook will need a lot of thought, to figure out
224 # Testing inputhook will need a lot of thought, to figure out
225 # how to have tests that don't lock up with the gui event
225 # how to have tests that don't lock up with the gui event
226 # loops in the picture
226 # loops in the picture
227 ipjoin('lib', 'inputhook'),
227 ipjoin('lib', 'inputhook'),
228 # Config files aren't really importable stand-alone
228 # Config files aren't really importable stand-alone
229 ipjoin('config', 'profile'),
229 ipjoin('config', 'profile'),
230 # The notebook 'static' directory contains JS, css and other
230 # The notebook 'static' directory contains JS, css and other
231 # files for web serving. Occasionally projects may put a .py
231 # files for web serving. Occasionally projects may put a .py
232 # file in there (MathJax ships a conf.py), so we might as
232 # file in there (MathJax ships a conf.py), so we might as
233 # well play it safe and skip the whole thing.
233 # well play it safe and skip the whole thing.
234 ipjoin('html', 'static'),
234 ipjoin('html', 'static'),
235 ipjoin('html', 'fabfile'),
235 ipjoin('html', 'fabfile'),
236 ]
236 ]
237 if not have['sqlite3']:
237 if not have['sqlite3']:
238 exclusions.append(ipjoin('core', 'tests', 'test_history'))
238 exclusions.append(ipjoin('core', 'tests', 'test_history'))
239 exclusions.append(ipjoin('core', 'history'))
239 exclusions.append(ipjoin('core', 'history'))
240 if not have['wx']:
240 if not have['wx']:
241 exclusions.append(ipjoin('lib', 'inputhookwx'))
241 exclusions.append(ipjoin('lib', 'inputhookwx'))
242
242
243 if 'IPython.kernel.inprocess' not in sys.argv:
243 if 'IPython.kernel.inprocess' not in sys.argv:
244 exclusions.append(ipjoin('kernel', 'inprocess'))
244 exclusions.append(ipjoin('kernel', 'inprocess'))
245
245
246 # FIXME: temporarily disable autoreload tests, as they can produce
246 # FIXME: temporarily disable autoreload tests, as they can produce
247 # spurious failures in subsequent tests (cythonmagic).
247 # spurious failures in subsequent tests (cythonmagic).
248 exclusions.append(ipjoin('extensions', 'autoreload'))
248 exclusions.append(ipjoin('extensions', 'autoreload'))
249 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
249 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
250
250
251 # We do this unconditionally, so that the test suite doesn't import
251 # We do this unconditionally, so that the test suite doesn't import
252 # gtk, changing the default encoding and masking some unicode bugs.
252 # gtk, changing the default encoding and masking some unicode bugs.
253 exclusions.append(ipjoin('lib', 'inputhookgtk'))
253 exclusions.append(ipjoin('lib', 'inputhookgtk'))
254 exclusions.append(ipjoin('kernel', 'zmq', 'gui', 'gtkembed'))
254 exclusions.append(ipjoin('kernel', 'zmq', 'gui', 'gtkembed'))
255
255
256 #Also done unconditionally, exclude nbconvert directories containing
256 #Also done unconditionally, exclude nbconvert directories containing
257 #config files used to test. Executing the config files with iptest would
257 #config files used to test. Executing the config files with iptest would
258 #cause an exception.
258 #cause an exception.
259 exclusions.append(ipjoin('nbconvert', 'tests', 'files'))
259 exclusions.append(ipjoin('nbconvert', 'tests', 'files'))
260 exclusions.append(ipjoin('nbconvert', 'exporters', 'tests', 'files'))
260 exclusions.append(ipjoin('nbconvert', 'exporters', 'tests', 'files'))
261
261
262 # These have to be skipped on win32 because the use echo, rm, cd, etc.
262 # These have to be skipped on win32 because the use echo, rm, cd, etc.
263 # See ticket https://github.com/ipython/ipython/issues/87
263 # See ticket https://github.com/ipython/ipython/issues/87
264 if sys.platform == 'win32':
264 if sys.platform == 'win32':
265 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
265 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
266 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
266 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
267
267
268 if not have['pexpect']:
268 if not have['pexpect']:
269 exclusions.extend([ipjoin('lib', 'irunner'),
269 exclusions.extend([ipjoin('lib', 'irunner'),
270 ipjoin('lib', 'tests', 'test_irunner'),
270 ipjoin('lib', 'tests', 'test_irunner'),
271 ipjoin('terminal', 'console'),
271 ipjoin('terminal', 'console'),
272 ])
272 ])
273
273
274 if not have['zmq']:
274 if not have['zmq']:
275 exclusions.append(ipjoin('lib', 'kernel'))
275 exclusions.append(ipjoin('kernel'))
276 exclusions.append(ipjoin('kernel'))
276 exclusions.append(ipjoin('qt'))
277 exclusions.append(ipjoin('qt'))
277 exclusions.append(ipjoin('html'))
278 exclusions.append(ipjoin('html'))
278 exclusions.append(ipjoin('consoleapp.py'))
279 exclusions.append(ipjoin('consoleapp.py'))
279 exclusions.append(ipjoin('terminal', 'console'))
280 exclusions.append(ipjoin('terminal', 'console'))
280 exclusions.append(ipjoin('parallel'))
281 exclusions.append(ipjoin('parallel'))
281 elif not have['qt'] or not have['pygments']:
282 elif not have['qt'] or not have['pygments']:
282 exclusions.append(ipjoin('qt'))
283 exclusions.append(ipjoin('qt'))
283
284
284 if not have['pymongo']:
285 if not have['pymongo']:
285 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
286 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
286 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
287 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
287
288
288 if not have['matplotlib']:
289 if not have['matplotlib']:
289 exclusions.extend([ipjoin('core', 'pylabtools'),
290 exclusions.extend([ipjoin('core', 'pylabtools'),
290 ipjoin('core', 'tests', 'test_pylabtools'),
291 ipjoin('core', 'tests', 'test_pylabtools'),
291 ipjoin('kernel', 'zmq', 'pylab'),
292 ipjoin('kernel', 'zmq', 'pylab'),
292 ])
293 ])
293
294
294 if not have['cython']:
295 if not have['cython']:
295 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
296 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
296 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
297 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
297
298
298 if not have['oct2py']:
299 if not have['oct2py']:
299 exclusions.extend([ipjoin('extensions', 'octavemagic')])
300 exclusions.extend([ipjoin('extensions', 'octavemagic')])
300 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
301 exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')])
301
302
302 if not have['tornado']:
303 if not have['tornado']:
303 exclusions.append(ipjoin('html'))
304 exclusions.append(ipjoin('html'))
304
305
305 if not have['jinja2']:
306 if not have['jinja2']:
306 exclusions.append(ipjoin('html', 'notebookapp'))
307 exclusions.append(ipjoin('html', 'notebookapp'))
307
308
308 if not have['rpy2'] or not have['numpy']:
309 if not have['rpy2'] or not have['numpy']:
309 exclusions.append(ipjoin('extensions', 'rmagic'))
310 exclusions.append(ipjoin('extensions', 'rmagic'))
310 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
311 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
311
312
312 if not have['azure']:
313 if not have['azure']:
313 exclusions.append(ipjoin('html', 'services', 'notebooks', 'azurenbmanager'))
314 exclusions.append(ipjoin('html', 'services', 'notebooks', 'azurenbmanager'))
314
315
315 if not all((have['pygments'], have['jinja2'], have['sphinx'])):
316 if not all((have['pygments'], have['jinja2'], have['sphinx'])):
316 exclusions.append(ipjoin('nbconvert'))
317 exclusions.append(ipjoin('nbconvert'))
317
318
318 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
319 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
319 if sys.platform == 'win32':
320 if sys.platform == 'win32':
320 exclusions = [s.replace('\\','\\\\') for s in exclusions]
321 exclusions = [s.replace('\\','\\\\') for s in exclusions]
321
322
322 # check for any exclusions that don't seem to exist:
323 # check for any exclusions that don't seem to exist:
323 parent, _ = os.path.split(get_ipython_package_dir())
324 parent, _ = os.path.split(get_ipython_package_dir())
324 for exclusion in exclusions:
325 for exclusion in exclusions:
325 if exclusion.endswith(('deathrow', 'quarantine')):
326 if exclusion.endswith(('deathrow', 'quarantine')):
326 # ignore deathrow/quarantine, which exist in dev, but not install
327 # ignore deathrow/quarantine, which exist in dev, but not install
327 continue
328 continue
328 fullpath = pjoin(parent, exclusion)
329 fullpath = pjoin(parent, exclusion)
329 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
330 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
330 warn("Excluding nonexistent file: %r" % exclusion)
331 warn("Excluding nonexistent file: %r" % exclusion)
331
332
332 return exclusions
333 return exclusions
333
334
334
335
335 class IPTester(object):
336 class IPTester(object):
336 """Call that calls iptest or trial in a subprocess.
337 """Call that calls iptest or trial in a subprocess.
337 """
338 """
338 #: string, name of test runner that will be called
339 #: string, name of test runner that will be called
339 runner = None
340 runner = None
340 #: list, parameters for test runner
341 #: list, parameters for test runner
341 params = None
342 params = None
342 #: list, arguments of system call to be made to call test runner
343 #: list, arguments of system call to be made to call test runner
343 call_args = None
344 call_args = None
344 #: list, subprocesses we start (for cleanup)
345 #: list, subprocesses we start (for cleanup)
345 processes = None
346 processes = None
346 #: str, coverage xml output file
347 #: str, coverage xml output file
347 coverage_xml = None
348 coverage_xml = None
348
349
349 def __init__(self, runner='iptest', params=None):
350 def __init__(self, runner='iptest', params=None):
350 """Create new test runner."""
351 """Create new test runner."""
351 p = os.path
352 p = os.path
352 if runner == 'iptest':
353 if runner == 'iptest':
353 iptest_app = os.path.abspath(get_ipython_module_path('IPython.testing.iptest'))
354 iptest_app = os.path.abspath(get_ipython_module_path('IPython.testing.iptest'))
354 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
355 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
355 else:
356 else:
356 raise Exception('Not a valid test runner: %s' % repr(runner))
357 raise Exception('Not a valid test runner: %s' % repr(runner))
357 if params is None:
358 if params is None:
358 params = []
359 params = []
359 if isinstance(params, str):
360 if isinstance(params, str):
360 params = [params]
361 params = [params]
361 self.params = params
362 self.params = params
362
363
363 # Assemble call
364 # Assemble call
364 self.call_args = self.runner+self.params
365 self.call_args = self.runner+self.params
365
366
366 # Find the section we're testing (IPython.foo)
367 # Find the section we're testing (IPython.foo)
367 for sect in self.params:
368 for sect in self.params:
368 if sect.startswith('IPython') or sect in special_test_suites: break
369 if sect.startswith('IPython') or sect in special_test_suites: break
369 else:
370 else:
370 raise ValueError("Section not found", self.params)
371 raise ValueError("Section not found", self.params)
371
372
372 if '--with-xunit' in self.call_args:
373 if '--with-xunit' in self.call_args:
373
374
374 self.call_args.append('--xunit-file')
375 self.call_args.append('--xunit-file')
375 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
376 # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary:
376 xunit_file = path.abspath(sect+'.xunit.xml')
377 xunit_file = path.abspath(sect+'.xunit.xml')
377 if sys.platform == 'win32':
378 if sys.platform == 'win32':
378 xunit_file = '"%s"' % xunit_file
379 xunit_file = '"%s"' % xunit_file
379 self.call_args.append(xunit_file)
380 self.call_args.append(xunit_file)
380
381
381 if '--with-xml-coverage' in self.call_args:
382 if '--with-xml-coverage' in self.call_args:
382 self.coverage_xml = path.abspath(sect+".coverage.xml")
383 self.coverage_xml = path.abspath(sect+".coverage.xml")
383 self.call_args.remove('--with-xml-coverage')
384 self.call_args.remove('--with-xml-coverage')
384 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
385 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
385
386
386 # Store anything we start to clean up on deletion
387 # Store anything we start to clean up on deletion
387 self.processes = []
388 self.processes = []
388
389
389 def _run_cmd(self):
390 def _run_cmd(self):
390 with TemporaryDirectory() as IPYTHONDIR:
391 with TemporaryDirectory() as IPYTHONDIR:
391 env = os.environ.copy()
392 env = os.environ.copy()
392 env['IPYTHONDIR'] = IPYTHONDIR
393 env['IPYTHONDIR'] = IPYTHONDIR
393 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
394 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
394 subp = subprocess.Popen(self.call_args, env=env)
395 subp = subprocess.Popen(self.call_args, env=env)
395 self.processes.append(subp)
396 self.processes.append(subp)
396 # If this fails, the process will be left in self.processes and
397 # If this fails, the process will be left in self.processes and
397 # cleaned up later, but if the wait call succeeds, then we can
398 # cleaned up later, but if the wait call succeeds, then we can
398 # clear the stored process.
399 # clear the stored process.
399 retcode = subp.wait()
400 retcode = subp.wait()
400 self.processes.pop()
401 self.processes.pop()
401 return retcode
402 return retcode
402
403
403 def run(self):
404 def run(self):
404 """Run the stored commands"""
405 """Run the stored commands"""
405 try:
406 try:
406 retcode = self._run_cmd()
407 retcode = self._run_cmd()
407 except KeyboardInterrupt:
408 except KeyboardInterrupt:
408 return -signal.SIGINT
409 return -signal.SIGINT
409 except:
410 except:
410 import traceback
411 import traceback
411 traceback.print_exc()
412 traceback.print_exc()
412 return 1 # signal failure
413 return 1 # signal failure
413
414
414 if self.coverage_xml:
415 if self.coverage_xml:
415 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
416 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
416 return retcode
417 return retcode
417
418
418 def __del__(self):
419 def __del__(self):
419 """Cleanup on exit by killing any leftover processes."""
420 """Cleanup on exit by killing any leftover processes."""
420 for subp in self.processes:
421 for subp in self.processes:
421 if subp.poll() is not None:
422 if subp.poll() is not None:
422 continue # process is already dead
423 continue # process is already dead
423
424
424 try:
425 try:
425 print('Cleaning up stale PID: %d' % subp.pid)
426 print('Cleaning up stale PID: %d' % subp.pid)
426 subp.kill()
427 subp.kill()
427 except: # (OSError, WindowsError) ?
428 except: # (OSError, WindowsError) ?
428 # This is just a best effort, if we fail or the process was
429 # This is just a best effort, if we fail or the process was
429 # really gone, ignore it.
430 # really gone, ignore it.
430 pass
431 pass
431 else:
432 else:
432 for i in range(10):
433 for i in range(10):
433 if subp.poll() is None:
434 if subp.poll() is None:
434 time.sleep(0.1)
435 time.sleep(0.1)
435 else:
436 else:
436 break
437 break
437
438
438 if subp.poll() is None:
439 if subp.poll() is None:
439 # The process did not die...
440 # The process did not die...
440 print('... failed. Manual cleanup may be required.')
441 print('... failed. Manual cleanup may be required.')
441
442
442
443
443 special_test_suites = {
444 special_test_suites = {
444 'autoreload': ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'],
445 'autoreload': ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'],
445 }
446 }
446
447
447 def make_runners(inc_slow=False):
448 def make_runners(inc_slow=False):
448 """Define the top-level packages that need to be tested.
449 """Define the top-level packages that need to be tested.
449 """
450 """
450
451
451 # Packages to be tested via nose, that only depend on the stdlib
452 # Packages to be tested via nose, that only depend on the stdlib
452 nose_pkg_names = ['config', 'core', 'extensions', 'lib', 'terminal',
453 nose_pkg_names = ['config', 'core', 'extensions', 'lib', 'terminal',
453 'testing', 'utils', 'nbformat']
454 'testing', 'utils', 'nbformat']
454
455
455 if have['qt']:
456 if have['qt']:
456 nose_pkg_names.append('qt')
457 nose_pkg_names.append('qt')
457
458
458 if have['tornado']:
459 if have['tornado']:
459 nose_pkg_names.append('html')
460 nose_pkg_names.append('html')
460
461
461 if have['zmq']:
462 if have['zmq']:
462 nose_pkg_names.append('kernel')
463 nose_pkg_names.append('kernel')
463 nose_pkg_names.append('kernel.inprocess')
464 nose_pkg_names.append('kernel.inprocess')
464 if inc_slow:
465 if inc_slow:
465 nose_pkg_names.append('parallel')
466 nose_pkg_names.append('parallel')
466
467
467 if all((have['pygments'], have['jinja2'], have['sphinx'])):
468 if all((have['pygments'], have['jinja2'], have['sphinx'])):
468 nose_pkg_names.append('nbconvert')
469 nose_pkg_names.append('nbconvert')
469
470
470 # For debugging this code, only load quick stuff
471 # For debugging this code, only load quick stuff
471 #nose_pkg_names = ['core', 'extensions'] # dbg
472 #nose_pkg_names = ['core', 'extensions'] # dbg
472
473
473 # Make fully qualified package names prepending 'IPython.' to our name lists
474 # Make fully qualified package names prepending 'IPython.' to our name lists
474 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
475 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
475
476
476 # Make runners
477 # Make runners
477 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
478 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
478
479
479 for name in special_test_suites:
480 for name in special_test_suites:
480 runners.append((name, IPTester('iptest', params=name)))
481 runners.append((name, IPTester('iptest', params=name)))
481
482
482 return runners
483 return runners
483
484
484
485
485 def run_iptest():
486 def run_iptest():
486 """Run the IPython test suite using nose.
487 """Run the IPython test suite using nose.
487
488
488 This function is called when this script is **not** called with the form
489 This function is called when this script is **not** called with the form
489 `iptest all`. It simply calls nose with appropriate command line flags
490 `iptest all`. It simply calls nose with appropriate command line flags
490 and accepts all of the standard nose arguments.
491 and accepts all of the standard nose arguments.
491 """
492 """
492 # Apply our monkeypatch to Xunit
493 # Apply our monkeypatch to Xunit
493 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
494 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
494 monkeypatch_xunit()
495 monkeypatch_xunit()
495
496
496 warnings.filterwarnings('ignore',
497 warnings.filterwarnings('ignore',
497 'This will be removed soon. Use IPython.testing.util instead')
498 'This will be removed soon. Use IPython.testing.util instead')
498
499
499 if sys.argv[1] in special_test_suites:
500 if sys.argv[1] in special_test_suites:
500 sys.argv[1:2] = special_test_suites[sys.argv[1]]
501 sys.argv[1:2] = special_test_suites[sys.argv[1]]
501 special_suite = True
502 special_suite = True
502 else:
503 else:
503 special_suite = False
504 special_suite = False
504
505
505 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
506 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
506
507
507 '--with-ipdoctest',
508 '--with-ipdoctest',
508 '--ipdoctest-tests','--ipdoctest-extension=txt',
509 '--ipdoctest-tests','--ipdoctest-extension=txt',
509
510
510 # We add --exe because of setuptools' imbecility (it
511 # We add --exe because of setuptools' imbecility (it
511 # blindly does chmod +x on ALL files). Nose does the
512 # blindly does chmod +x on ALL files). Nose does the
512 # right thing and it tries to avoid executables,
513 # right thing and it tries to avoid executables,
513 # setuptools unfortunately forces our hand here. This
514 # setuptools unfortunately forces our hand here. This
514 # has been discussed on the distutils list and the
515 # has been discussed on the distutils list and the
515 # setuptools devs refuse to fix this problem!
516 # setuptools devs refuse to fix this problem!
516 '--exe',
517 '--exe',
517 ]
518 ]
518 if '-a' not in argv and '-A' not in argv:
519 if '-a' not in argv and '-A' not in argv:
519 argv = argv + ['-a', '!crash']
520 argv = argv + ['-a', '!crash']
520
521
521 if nose.__version__ >= '0.11':
522 if nose.__version__ >= '0.11':
522 # I don't fully understand why we need this one, but depending on what
523 # I don't fully understand why we need this one, but depending on what
523 # directory the test suite is run from, if we don't give it, 0 tests
524 # directory the test suite is run from, if we don't give it, 0 tests
524 # get run. Specifically, if the test suite is run from the source dir
525 # get run. Specifically, if the test suite is run from the source dir
525 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
526 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
526 # even if the same call done in this directory works fine). It appears
527 # even if the same call done in this directory works fine). It appears
527 # that if the requested package is in the current dir, nose bails early
528 # that if the requested package is in the current dir, nose bails early
528 # by default. Since it's otherwise harmless, leave it in by default
529 # by default. Since it's otherwise harmless, leave it in by default
529 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
530 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
530 argv.append('--traverse-namespace')
531 argv.append('--traverse-namespace')
531
532
532 # use our plugin for doctesting. It will remove the standard doctest plugin
533 # use our plugin for doctesting. It will remove the standard doctest plugin
533 # if it finds it enabled
534 # if it finds it enabled
534 ipdt = IPythonDoctest() if special_suite else IPythonDoctest(make_exclude())
535 ipdt = IPythonDoctest() if special_suite else IPythonDoctest(make_exclude())
535 plugins = [ipdt, KnownFailure()]
536 plugins = [ipdt, KnownFailure()]
536
537
537 # We need a global ipython running in this process, but the special
538 # We need a global ipython running in this process, but the special
538 # in-process group spawns its own IPython kernels, so for *that* group we
539 # in-process group spawns its own IPython kernels, so for *that* group we
539 # must avoid also opening the global one (otherwise there's a conflict of
540 # must avoid also opening the global one (otherwise there's a conflict of
540 # singletons). Ultimately the solution to this problem is to refactor our
541 # singletons). Ultimately the solution to this problem is to refactor our
541 # assumptions about what needs to be a singleton and what doesn't (app
542 # assumptions about what needs to be a singleton and what doesn't (app
542 # objects should, individual shells shouldn't). But for now, this
543 # objects should, individual shells shouldn't). But for now, this
543 # workaround allows the test suite for the inprocess module to complete.
544 # workaround allows the test suite for the inprocess module to complete.
544 if not 'IPython.kernel.inprocess' in sys.argv:
545 if not 'IPython.kernel.inprocess' in sys.argv:
545 globalipapp.start_ipython()
546 globalipapp.start_ipython()
546
547
547 # Now nose can run
548 # Now nose can run
548 TestProgram(argv=argv, addplugins=plugins)
549 TestProgram(argv=argv, addplugins=plugins)
549
550
550
551
551 def run_iptestall(inc_slow=False):
552 def run_iptestall(inc_slow=False):
552 """Run the entire IPython test suite by calling nose and trial.
553 """Run the entire IPython test suite by calling nose and trial.
553
554
554 This function constructs :class:`IPTester` instances for all IPython
555 This function constructs :class:`IPTester` instances for all IPython
555 modules and package and then runs each of them. This causes the modules
556 modules and package and then runs each of them. This causes the modules
556 and packages of IPython to be tested each in their own subprocess using
557 and packages of IPython to be tested each in their own subprocess using
557 nose.
558 nose.
558
559
559 Parameters
560 Parameters
560 ----------
561 ----------
561
562
562 inc_slow : bool, optional
563 inc_slow : bool, optional
563 Include slow tests, like IPython.parallel. By default, these tests aren't
564 Include slow tests, like IPython.parallel. By default, these tests aren't
564 run.
565 run.
565 """
566 """
566
567
567 runners = make_runners(inc_slow=inc_slow)
568 runners = make_runners(inc_slow=inc_slow)
568
569
569 # Run the test runners in a temporary dir so we can nuke it when finished
570 # Run the test runners in a temporary dir so we can nuke it when finished
570 # to clean up any junk files left over by accident. This also makes it
571 # to clean up any junk files left over by accident. This also makes it
571 # robust against being run in non-writeable directories by mistake, as the
572 # robust against being run in non-writeable directories by mistake, as the
572 # temp dir will always be user-writeable.
573 # temp dir will always be user-writeable.
573 curdir = os.getcwdu()
574 curdir = os.getcwdu()
574 testdir = tempfile.gettempdir()
575 testdir = tempfile.gettempdir()
575 os.chdir(testdir)
576 os.chdir(testdir)
576
577
577 # Run all test runners, tracking execution time
578 # Run all test runners, tracking execution time
578 failed = []
579 failed = []
579 t_start = time.time()
580 t_start = time.time()
580 try:
581 try:
581 for (name, runner) in runners:
582 for (name, runner) in runners:
582 print('*'*70)
583 print('*'*70)
583 print('IPython test group:',name)
584 print('IPython test group:',name)
584 res = runner.run()
585 res = runner.run()
585 if res:
586 if res:
586 failed.append( (name, runner) )
587 failed.append( (name, runner) )
587 if res == -signal.SIGINT:
588 if res == -signal.SIGINT:
588 print("Interrupted")
589 print("Interrupted")
589 break
590 break
590 finally:
591 finally:
591 os.chdir(curdir)
592 os.chdir(curdir)
592 t_end = time.time()
593 t_end = time.time()
593 t_tests = t_end - t_start
594 t_tests = t_end - t_start
594 nrunners = len(runners)
595 nrunners = len(runners)
595 nfail = len(failed)
596 nfail = len(failed)
596 # summarize results
597 # summarize results
597 print()
598 print()
598 print('*'*70)
599 print('*'*70)
599 print('Test suite completed for system with the following information:')
600 print('Test suite completed for system with the following information:')
600 print(report())
601 print(report())
601 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
602 print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
602 print()
603 print()
603 print('Status:')
604 print('Status:')
604 if not failed:
605 if not failed:
605 print('OK')
606 print('OK')
606 else:
607 else:
607 # If anything went wrong, point out what command to rerun manually to
608 # If anything went wrong, point out what command to rerun manually to
608 # see the actual errors and individual summary
609 # see the actual errors and individual summary
609 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
610 print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
610 for name, failed_runner in failed:
611 for name, failed_runner in failed:
611 print('-'*40)
612 print('-'*40)
612 print('Runner failed:',name)
613 print('Runner failed:',name)
613 print('You may wish to rerun this one individually, with:')
614 print('You may wish to rerun this one individually, with:')
614 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
615 failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
615 print(u' '.join(failed_call_args))
616 print(u' '.join(failed_call_args))
616 print()
617 print()
617 # Ensure that our exit code indicates failure
618 # Ensure that our exit code indicates failure
618 sys.exit(1)
619 sys.exit(1)
619
620
620
621
621 def main():
622 def main():
622 for arg in sys.argv[1:]:
623 for arg in sys.argv[1:]:
623 if arg.startswith('IPython') or arg in special_test_suites:
624 if arg.startswith('IPython') or arg in special_test_suites:
624 # This is in-process
625 # This is in-process
625 run_iptest()
626 run_iptest()
626 else:
627 else:
627 if "--all" in sys.argv:
628 if "--all" in sys.argv:
628 sys.argv.remove("--all")
629 sys.argv.remove("--all")
629 inc_slow = True
630 inc_slow = True
630 else:
631 else:
631 inc_slow = False
632 inc_slow = False
632 # This starts subprocesses
633 # This starts subprocesses
633 run_iptestall(inc_slow=inc_slow)
634 run_iptestall(inc_slow=inc_slow)
634
635
635
636
636 if __name__ == '__main__':
637 if __name__ == '__main__':
637 main()
638 main()
General Comments 0
You need to be logged in to leave comments. Login now