##// END OF EJS Templates
skip autoreload tests
MinRK -
Show More
@@ -1,546 +1,551 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009-2011 The IPython Development Team
18 # Copyright (C) 2009-2011 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 # Stdlib
28 # Stdlib
29 import glob
29 import glob
30 import os
30 import os
31 import os.path as path
31 import os.path as path
32 import signal
32 import signal
33 import sys
33 import sys
34 import subprocess
34 import subprocess
35 import tempfile
35 import tempfile
36 import time
36 import time
37 import warnings
37 import warnings
38
38
39 # Note: monkeypatch!
39 # Note: monkeypatch!
40 # We need to monkeypatch a small problem in nose itself first, before importing
40 # We need to monkeypatch a small problem in nose itself first, before importing
41 # it for actual use. This should get into nose upstream, but its release cycle
41 # it for actual use. This should get into nose upstream, but its release cycle
42 # is slow and we need it for our parametric tests to work correctly.
42 # is slow and we need it for our parametric tests to work correctly.
43 from IPython.testing import nosepatch
43 from IPython.testing import nosepatch
44
44
45 # Monkeypatch extra assert methods into nose.tools if they're not already there.
45 # Monkeypatch extra assert methods into nose.tools if they're not already there.
46 # This can be dropped once we no longer test on Python 2.6
46 # This can be dropped once we no longer test on Python 2.6
47 from IPython.testing import nose_assert_methods
47 from IPython.testing import nose_assert_methods
48
48
49 # Now, proceed to import nose itself
49 # Now, proceed to import nose itself
50 import nose.plugins.builtin
50 import nose.plugins.builtin
51 from nose.plugins.xunit import Xunit
51 from nose.plugins.xunit import Xunit
52 from nose import SkipTest
52 from nose import SkipTest
53 from nose.core import TestProgram
53 from nose.core import TestProgram
54
54
55 # Our own imports
55 # Our own imports
56 from IPython.utils.importstring import import_item
56 from IPython.utils.importstring import import_item
57 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
57 from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir
58 from IPython.utils.process import find_cmd, pycmd2argv
58 from IPython.utils.process import find_cmd, pycmd2argv
59 from IPython.utils.sysinfo import sys_info
59 from IPython.utils.sysinfo import sys_info
60 from IPython.utils.warn import warn
60 from IPython.utils.warn import warn
61
61
62 from IPython.testing import globalipapp
62 from IPython.testing import globalipapp
63 from IPython.testing.plugin.ipdoctest import IPythonDoctest
63 from IPython.testing.plugin.ipdoctest import IPythonDoctest
64 from IPython.external.decorators import KnownFailure, knownfailureif
64 from IPython.external.decorators import KnownFailure, knownfailureif
65
65
66 pjoin = path.join
66 pjoin = path.join
67
67
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Globals
70 # Globals
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73
73
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75 # Warnings control
75 # Warnings control
76 #-----------------------------------------------------------------------------
76 #-----------------------------------------------------------------------------
77
77
78 # Twisted generates annoying warnings with Python 2.6, as will do other code
78 # Twisted generates annoying warnings with Python 2.6, as will do other code
79 # that imports 'sets' as of today
79 # that imports 'sets' as of today
80 warnings.filterwarnings('ignore', 'the sets module is deprecated',
80 warnings.filterwarnings('ignore', 'the sets module is deprecated',
81 DeprecationWarning )
81 DeprecationWarning )
82
82
83 # This one also comes from Twisted
83 # This one also comes from Twisted
84 warnings.filterwarnings('ignore', 'the sha module is deprecated',
84 warnings.filterwarnings('ignore', 'the sha module is deprecated',
85 DeprecationWarning)
85 DeprecationWarning)
86
86
87 # Wx on Fedora11 spits these out
87 # Wx on Fedora11 spits these out
88 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
88 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
89 UserWarning)
89 UserWarning)
90
90
91 # ------------------------------------------------------------------------------
91 # ------------------------------------------------------------------------------
92 # Monkeypatch Xunit to count known failures as skipped.
92 # Monkeypatch Xunit to count known failures as skipped.
93 # ------------------------------------------------------------------------------
93 # ------------------------------------------------------------------------------
94 def monkeypatch_xunit():
94 def monkeypatch_xunit():
95 try:
95 try:
96 knownfailureif(True)(lambda: None)()
96 knownfailureif(True)(lambda: None)()
97 except Exception as e:
97 except Exception as e:
98 KnownFailureTest = type(e)
98 KnownFailureTest = type(e)
99
99
100 def addError(self, test, err, capt=None):
100 def addError(self, test, err, capt=None):
101 if issubclass(err[0], KnownFailureTest):
101 if issubclass(err[0], KnownFailureTest):
102 err = (SkipTest,) + err[1:]
102 err = (SkipTest,) + err[1:]
103 return self.orig_addError(test, err, capt)
103 return self.orig_addError(test, err, capt)
104
104
105 Xunit.orig_addError = Xunit.addError
105 Xunit.orig_addError = Xunit.addError
106 Xunit.addError = addError
106 Xunit.addError = addError
107
107
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109 # Logic for skipping doctests
109 # Logic for skipping doctests
110 #-----------------------------------------------------------------------------
110 #-----------------------------------------------------------------------------
111 def extract_version(mod):
111 def extract_version(mod):
112 return mod.__version__
112 return mod.__version__
113
113
114 def test_for(item, min_version=None, callback=extract_version):
114 def test_for(item, min_version=None, callback=extract_version):
115 """Test to see if item is importable, and optionally check against a minimum
115 """Test to see if item is importable, and optionally check against a minimum
116 version.
116 version.
117
117
118 If min_version is given, the default behavior is to check against the
118 If min_version is given, the default behavior is to check against the
119 `__version__` attribute of the item, but specifying `callback` allows you to
119 `__version__` attribute of the item, but specifying `callback` allows you to
120 extract the value you are interested in. e.g::
120 extract the value you are interested in. e.g::
121
121
122 In [1]: import sys
122 In [1]: import sys
123
123
124 In [2]: from IPython.testing.iptest import test_for
124 In [2]: from IPython.testing.iptest import test_for
125
125
126 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
126 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
127 Out[3]: True
127 Out[3]: True
128
128
129 """
129 """
130 try:
130 try:
131 check = import_item(item)
131 check = import_item(item)
132 except (ImportError, RuntimeError):
132 except (ImportError, RuntimeError):
133 # GTK reports Runtime error if it can't be initialized even if it's
133 # GTK reports Runtime error if it can't be initialized even if it's
134 # importable.
134 # importable.
135 return False
135 return False
136 else:
136 else:
137 if min_version:
137 if min_version:
138 if callback:
138 if callback:
139 # extra processing step to get version to compare
139 # extra processing step to get version to compare
140 check = callback(check)
140 check = callback(check)
141
141
142 return check >= min_version
142 return check >= min_version
143 else:
143 else:
144 return True
144 return True
145
145
146 # Global dict where we can store information on what we have and what we don't
146 # Global dict where we can store information on what we have and what we don't
147 # have available at test run time
147 # have available at test run time
148 have = {}
148 have = {}
149
149
150 have['curses'] = test_for('_curses')
150 have['curses'] = test_for('_curses')
151 have['matplotlib'] = test_for('matplotlib')
151 have['matplotlib'] = test_for('matplotlib')
152 have['numpy'] = test_for('numpy')
152 have['numpy'] = test_for('numpy')
153 have['pexpect'] = test_for('IPython.external.pexpect')
153 have['pexpect'] = test_for('IPython.external.pexpect')
154 have['pymongo'] = test_for('pymongo')
154 have['pymongo'] = test_for('pymongo')
155 have['pygments'] = test_for('pygments')
155 have['pygments'] = test_for('pygments')
156 have['qt'] = test_for('IPython.external.qt')
156 have['qt'] = test_for('IPython.external.qt')
157 have['rpy2'] = test_for('rpy2')
157 have['rpy2'] = test_for('rpy2')
158 have['sqlite3'] = test_for('sqlite3')
158 have['sqlite3'] = test_for('sqlite3')
159 have['cython'] = test_for('Cython')
159 have['cython'] = test_for('Cython')
160 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
160 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
161 have['wx'] = test_for('wx')
161 have['wx'] = test_for('wx')
162 have['wx.aui'] = test_for('wx.aui')
162 have['wx.aui'] = test_for('wx.aui')
163
163
164 if os.name == 'nt':
164 if os.name == 'nt':
165 min_zmq = (2,1,7)
165 min_zmq = (2,1,7)
166 else:
166 else:
167 min_zmq = (2,1,4)
167 min_zmq = (2,1,4)
168
168
169 def version_tuple(mod):
169 def version_tuple(mod):
170 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
170 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
171 # turn 'dev' into 999, because Python3 rejects str-int comparisons
171 # turn 'dev' into 999, because Python3 rejects str-int comparisons
172 vs = mod.__version__.replace('dev', '.999')
172 vs = mod.__version__.replace('dev', '.999')
173 tup = tuple([int(v) for v in vs.split('.') ])
173 tup = tuple([int(v) for v in vs.split('.') ])
174 return tup
174 return tup
175
175
176 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
176 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
177
177
178 #-----------------------------------------------------------------------------
178 #-----------------------------------------------------------------------------
179 # Functions and classes
179 # Functions and classes
180 #-----------------------------------------------------------------------------
180 #-----------------------------------------------------------------------------
181
181
182 def report():
182 def report():
183 """Return a string with a summary report of test-related variables."""
183 """Return a string with a summary report of test-related variables."""
184
184
185 out = [ sys_info(), '\n']
185 out = [ sys_info(), '\n']
186
186
187 avail = []
187 avail = []
188 not_avail = []
188 not_avail = []
189
189
190 for k, is_avail in have.items():
190 for k, is_avail in have.items():
191 if is_avail:
191 if is_avail:
192 avail.append(k)
192 avail.append(k)
193 else:
193 else:
194 not_avail.append(k)
194 not_avail.append(k)
195
195
196 if avail:
196 if avail:
197 out.append('\nTools and libraries available at test time:\n')
197 out.append('\nTools and libraries available at test time:\n')
198 avail.sort()
198 avail.sort()
199 out.append(' ' + ' '.join(avail)+'\n')
199 out.append(' ' + ' '.join(avail)+'\n')
200
200
201 if not_avail:
201 if not_avail:
202 out.append('\nTools and libraries NOT available at test time:\n')
202 out.append('\nTools and libraries NOT available at test time:\n')
203 not_avail.sort()
203 not_avail.sort()
204 out.append(' ' + ' '.join(not_avail)+'\n')
204 out.append(' ' + ' '.join(not_avail)+'\n')
205
205
206 return ''.join(out)
206 return ''.join(out)
207
207
208
208
209 def make_exclude():
209 def make_exclude():
210 """Make patterns of modules and packages to exclude from testing.
210 """Make patterns of modules and packages to exclude from testing.
211
211
212 For the IPythonDoctest plugin, we need to exclude certain patterns that
212 For the IPythonDoctest plugin, we need to exclude certain patterns that
213 cause testing problems. We should strive to minimize the number of
213 cause testing problems. We should strive to minimize the number of
214 skipped modules, since this means untested code.
214 skipped modules, since this means untested code.
215
215
216 These modules and packages will NOT get scanned by nose at all for tests.
216 These modules and packages will NOT get scanned by nose at all for tests.
217 """
217 """
218 # Simple utility to make IPython paths more readably, we need a lot of
218 # Simple utility to make IPython paths more readably, we need a lot of
219 # these below
219 # these below
220 ipjoin = lambda *paths: pjoin('IPython', *paths)
220 ipjoin = lambda *paths: pjoin('IPython', *paths)
221
221
222 exclusions = [ipjoin('external'),
222 exclusions = [ipjoin('external'),
223 ipjoin('quarantine'),
223 ipjoin('quarantine'),
224 ipjoin('deathrow'),
224 ipjoin('deathrow'),
225 # This guy is probably attic material
225 # This guy is probably attic material
226 ipjoin('testing', 'mkdoctests'),
226 ipjoin('testing', 'mkdoctests'),
227 # Testing inputhook will need a lot of thought, to figure out
227 # Testing inputhook will need a lot of thought, to figure out
228 # how to have tests that don't lock up with the gui event
228 # how to have tests that don't lock up with the gui event
229 # loops in the picture
229 # loops in the picture
230 ipjoin('lib', 'inputhook'),
230 ipjoin('lib', 'inputhook'),
231 # Config files aren't really importable stand-alone
231 # Config files aren't really importable stand-alone
232 ipjoin('config', 'profile'),
232 ipjoin('config', 'profile'),
233 ]
233 ]
234 if not have['sqlite3']:
234 if not have['sqlite3']:
235 exclusions.append(ipjoin('core', 'tests', 'test_history'))
235 exclusions.append(ipjoin('core', 'tests', 'test_history'))
236 exclusions.append(ipjoin('core', 'history'))
236 exclusions.append(ipjoin('core', 'history'))
237 if not have['wx']:
237 if not have['wx']:
238 exclusions.append(ipjoin('lib', 'inputhookwx'))
238 exclusions.append(ipjoin('lib', 'inputhookwx'))
239
239
240 # FIXME: temporarily disable autoreload tests, as they can produce
241 # spurious failures in subsequent tests (cythonmagic).
242 exclusions.append(ipjoin('extensions', 'autoreload'))
243 exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload'))
244
240 # We do this unconditionally, so that the test suite doesn't import
245 # We do this unconditionally, so that the test suite doesn't import
241 # gtk, changing the default encoding and masking some unicode bugs.
246 # gtk, changing the default encoding and masking some unicode bugs.
242 exclusions.append(ipjoin('lib', 'inputhookgtk'))
247 exclusions.append(ipjoin('lib', 'inputhookgtk'))
243 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
248 exclusions.append(ipjoin('zmq', 'gui', 'gtkembed'))
244
249
245 # These have to be skipped on win32 because the use echo, rm, cd, etc.
250 # These have to be skipped on win32 because the use echo, rm, cd, etc.
246 # See ticket https://github.com/ipython/ipython/issues/87
251 # See ticket https://github.com/ipython/ipython/issues/87
247 if sys.platform == 'win32':
252 if sys.platform == 'win32':
248 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
253 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
249 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
254 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
250
255
251 if not have['pexpect']:
256 if not have['pexpect']:
252 exclusions.extend([ipjoin('scripts', 'irunner'),
257 exclusions.extend([ipjoin('scripts', 'irunner'),
253 ipjoin('lib', 'irunner'),
258 ipjoin('lib', 'irunner'),
254 ipjoin('lib', 'tests', 'test_irunner'),
259 ipjoin('lib', 'tests', 'test_irunner'),
255 ipjoin('frontend', 'terminal', 'console'),
260 ipjoin('frontend', 'terminal', 'console'),
256 ])
261 ])
257
262
258 if not have['zmq']:
263 if not have['zmq']:
259 exclusions.append(ipjoin('zmq'))
264 exclusions.append(ipjoin('zmq'))
260 exclusions.append(ipjoin('frontend', 'qt'))
265 exclusions.append(ipjoin('frontend', 'qt'))
261 exclusions.append(ipjoin('frontend', 'html'))
266 exclusions.append(ipjoin('frontend', 'html'))
262 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
267 exclusions.append(ipjoin('frontend', 'consoleapp.py'))
263 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
268 exclusions.append(ipjoin('frontend', 'terminal', 'console'))
264 exclusions.append(ipjoin('parallel'))
269 exclusions.append(ipjoin('parallel'))
265 elif not have['qt'] or not have['pygments']:
270 elif not have['qt'] or not have['pygments']:
266 exclusions.append(ipjoin('frontend', 'qt'))
271 exclusions.append(ipjoin('frontend', 'qt'))
267
272
268 if not have['pymongo']:
273 if not have['pymongo']:
269 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
274 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
270 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
275 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
271
276
272 if not have['matplotlib']:
277 if not have['matplotlib']:
273 exclusions.extend([ipjoin('core', 'pylabtools'),
278 exclusions.extend([ipjoin('core', 'pylabtools'),
274 ipjoin('core', 'tests', 'test_pylabtools'),
279 ipjoin('core', 'tests', 'test_pylabtools'),
275 ipjoin('zmq', 'pylab'),
280 ipjoin('zmq', 'pylab'),
276 ])
281 ])
277
282
278 if not have['cython']:
283 if not have['cython']:
279 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
284 exclusions.extend([ipjoin('extensions', 'cythonmagic')])
280 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
285 exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')])
281
286
282 if not have['tornado']:
287 if not have['tornado']:
283 exclusions.append(ipjoin('frontend', 'html'))
288 exclusions.append(ipjoin('frontend', 'html'))
284
289
285 if not have['rpy2'] or not have['numpy']:
290 if not have['rpy2'] or not have['numpy']:
286 exclusions.append(ipjoin('extensions', 'rmagic'))
291 exclusions.append(ipjoin('extensions', 'rmagic'))
287 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
292 exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic'))
288
293
289 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
294 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
290 if sys.platform == 'win32':
295 if sys.platform == 'win32':
291 exclusions = [s.replace('\\','\\\\') for s in exclusions]
296 exclusions = [s.replace('\\','\\\\') for s in exclusions]
292
297
293 # check for any exclusions that don't seem to exist:
298 # check for any exclusions that don't seem to exist:
294 parent, _ = os.path.split(get_ipython_package_dir())
299 parent, _ = os.path.split(get_ipython_package_dir())
295 for exclusion in exclusions:
300 for exclusion in exclusions:
296 fullpath = pjoin(parent, exclusion)
301 fullpath = pjoin(parent, exclusion)
297 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
302 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
298 warn("Excluding nonexistent file: %r\n" % exclusion)
303 warn("Excluding nonexistent file: %r\n" % exclusion)
299
304
300 return exclusions
305 return exclusions
301
306
302
307
303 class IPTester(object):
308 class IPTester(object):
304 """Call that calls iptest or trial in a subprocess.
309 """Call that calls iptest or trial in a subprocess.
305 """
310 """
306 #: string, name of test runner that will be called
311 #: string, name of test runner that will be called
307 runner = None
312 runner = None
308 #: list, parameters for test runner
313 #: list, parameters for test runner
309 params = None
314 params = None
310 #: list, arguments of system call to be made to call test runner
315 #: list, arguments of system call to be made to call test runner
311 call_args = None
316 call_args = None
312 #: list, process ids of subprocesses we start (for cleanup)
317 #: list, process ids of subprocesses we start (for cleanup)
313 pids = None
318 pids = None
314 #: str, coverage xml output file
319 #: str, coverage xml output file
315 coverage_xml = None
320 coverage_xml = None
316
321
317 def __init__(self, runner='iptest', params=None):
322 def __init__(self, runner='iptest', params=None):
318 """Create new test runner."""
323 """Create new test runner."""
319 p = os.path
324 p = os.path
320 if runner == 'iptest':
325 if runner == 'iptest':
321 iptest_app = get_ipython_module_path('IPython.testing.iptest')
326 iptest_app = get_ipython_module_path('IPython.testing.iptest')
322 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
327 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
323 else:
328 else:
324 raise Exception('Not a valid test runner: %s' % repr(runner))
329 raise Exception('Not a valid test runner: %s' % repr(runner))
325 if params is None:
330 if params is None:
326 params = []
331 params = []
327 if isinstance(params, str):
332 if isinstance(params, str):
328 params = [params]
333 params = [params]
329 self.params = params
334 self.params = params
330
335
331 # Assemble call
336 # Assemble call
332 self.call_args = self.runner+self.params
337 self.call_args = self.runner+self.params
333
338
334 # Find the section we're testing (IPython.foo)
339 # Find the section we're testing (IPython.foo)
335 for sect in self.params:
340 for sect in self.params:
336 if sect.startswith('IPython'): break
341 if sect.startswith('IPython'): break
337 else:
342 else:
338 raise ValueError("Section not found", self.params)
343 raise ValueError("Section not found", self.params)
339
344
340 if '--with-xunit' in self.call_args:
345 if '--with-xunit' in self.call_args:
341 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
346 self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml'))
342
347
343 if '--with-xml-coverage' in self.call_args:
348 if '--with-xml-coverage' in self.call_args:
344 self.coverage_xml = path.abspath(sect+".coverage.xml")
349 self.coverage_xml = path.abspath(sect+".coverage.xml")
345 self.call_args.remove('--with-xml-coverage')
350 self.call_args.remove('--with-xml-coverage')
346 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
351 self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:]
347
352
348 # Store pids of anything we start to clean up on deletion, if possible
353 # Store pids of anything we start to clean up on deletion, if possible
349 # (on posix only, since win32 has no os.kill)
354 # (on posix only, since win32 has no os.kill)
350 self.pids = []
355 self.pids = []
351
356
352 if sys.platform == 'win32':
357 if sys.platform == 'win32':
353 def _run_cmd(self):
358 def _run_cmd(self):
354 # On Windows, use os.system instead of subprocess.call, because I
359 # On Windows, use os.system instead of subprocess.call, because I
355 # was having problems with subprocess and I just don't know enough
360 # was having problems with subprocess and I just don't know enough
356 # about win32 to debug this reliably. Os.system may be the 'old
361 # about win32 to debug this reliably. Os.system may be the 'old
357 # fashioned' way to do it, but it works just fine. If someone
362 # fashioned' way to do it, but it works just fine. If someone
358 # later can clean this up that's fine, as long as the tests run
363 # later can clean this up that's fine, as long as the tests run
359 # reliably in win32.
364 # reliably in win32.
360 # What types of problems are you having. They may be related to
365 # What types of problems are you having. They may be related to
361 # running Python in unboffered mode. BG.
366 # running Python in unboffered mode. BG.
362 return os.system(' '.join(self.call_args))
367 return os.system(' '.join(self.call_args))
363 else:
368 else:
364 def _run_cmd(self):
369 def _run_cmd(self):
365 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
370 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
366 subp = subprocess.Popen(self.call_args)
371 subp = subprocess.Popen(self.call_args)
367 self.pids.append(subp.pid)
372 self.pids.append(subp.pid)
368 # If this fails, the pid will be left in self.pids and cleaned up
373 # If this fails, the pid will be left in self.pids and cleaned up
369 # later, but if the wait call succeeds, then we can clear the
374 # later, but if the wait call succeeds, then we can clear the
370 # stored pid.
375 # stored pid.
371 retcode = subp.wait()
376 retcode = subp.wait()
372 self.pids.pop()
377 self.pids.pop()
373 return retcode
378 return retcode
374
379
375 def run(self):
380 def run(self):
376 """Run the stored commands"""
381 """Run the stored commands"""
377 try:
382 try:
378 retcode = self._run_cmd()
383 retcode = self._run_cmd()
379 except:
384 except:
380 import traceback
385 import traceback
381 traceback.print_exc()
386 traceback.print_exc()
382 return 1 # signal failure
387 return 1 # signal failure
383
388
384 if self.coverage_xml:
389 if self.coverage_xml:
385 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
390 subprocess.call(["coverage", "xml", "-o", self.coverage_xml])
386 return retcode
391 return retcode
387
392
388 def __del__(self):
393 def __del__(self):
389 """Cleanup on exit by killing any leftover processes."""
394 """Cleanup on exit by killing any leftover processes."""
390
395
391 if not hasattr(os, 'kill'):
396 if not hasattr(os, 'kill'):
392 return
397 return
393
398
394 for pid in self.pids:
399 for pid in self.pids:
395 try:
400 try:
396 print 'Cleaning stale PID:', pid
401 print 'Cleaning stale PID:', pid
397 os.kill(pid, signal.SIGKILL)
402 os.kill(pid, signal.SIGKILL)
398 except OSError:
403 except OSError:
399 # This is just a best effort, if we fail or the process was
404 # This is just a best effort, if we fail or the process was
400 # really gone, ignore it.
405 # really gone, ignore it.
401 pass
406 pass
402
407
403
408
404 def make_runners():
409 def make_runners():
405 """Define the top-level packages that need to be tested.
410 """Define the top-level packages that need to be tested.
406 """
411 """
407
412
408 # Packages to be tested via nose, that only depend on the stdlib
413 # Packages to be tested via nose, that only depend on the stdlib
409 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
414 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
410 'scripts', 'testing', 'utils', 'nbformat' ]
415 'scripts', 'testing', 'utils', 'nbformat' ]
411
416
412 if have['zmq']:
417 if have['zmq']:
413 nose_pkg_names.append('zmq')
418 nose_pkg_names.append('zmq')
414 nose_pkg_names.append('parallel')
419 nose_pkg_names.append('parallel')
415
420
416 # For debugging this code, only load quick stuff
421 # For debugging this code, only load quick stuff
417 #nose_pkg_names = ['core', 'extensions'] # dbg
422 #nose_pkg_names = ['core', 'extensions'] # dbg
418
423
419 # Make fully qualified package names prepending 'IPython.' to our name lists
424 # Make fully qualified package names prepending 'IPython.' to our name lists
420 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
425 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
421
426
422 # Make runners
427 # Make runners
423 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
428 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
424
429
425 return runners
430 return runners
426
431
427
432
428 def run_iptest():
433 def run_iptest():
429 """Run the IPython test suite using nose.
434 """Run the IPython test suite using nose.
430
435
431 This function is called when this script is **not** called with the form
436 This function is called when this script is **not** called with the form
432 `iptest all`. It simply calls nose with appropriate command line flags
437 `iptest all`. It simply calls nose with appropriate command line flags
433 and accepts all of the standard nose arguments.
438 and accepts all of the standard nose arguments.
434 """
439 """
435 # Apply our monkeypatch to Xunit
440 # Apply our monkeypatch to Xunit
436 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
441 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
437 monkeypatch_xunit()
442 monkeypatch_xunit()
438
443
439 warnings.filterwarnings('ignore',
444 warnings.filterwarnings('ignore',
440 'This will be removed soon. Use IPython.testing.util instead')
445 'This will be removed soon. Use IPython.testing.util instead')
441
446
442 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
447 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
443
448
444 '--with-ipdoctest',
449 '--with-ipdoctest',
445 '--ipdoctest-tests','--ipdoctest-extension=txt',
450 '--ipdoctest-tests','--ipdoctest-extension=txt',
446
451
447 # We add --exe because of setuptools' imbecility (it
452 # We add --exe because of setuptools' imbecility (it
448 # blindly does chmod +x on ALL files). Nose does the
453 # blindly does chmod +x on ALL files). Nose does the
449 # right thing and it tries to avoid executables,
454 # right thing and it tries to avoid executables,
450 # setuptools unfortunately forces our hand here. This
455 # setuptools unfortunately forces our hand here. This
451 # has been discussed on the distutils list and the
456 # has been discussed on the distutils list and the
452 # setuptools devs refuse to fix this problem!
457 # setuptools devs refuse to fix this problem!
453 '--exe',
458 '--exe',
454 ]
459 ]
455
460
456 if nose.__version__ >= '0.11':
461 if nose.__version__ >= '0.11':
457 # I don't fully understand why we need this one, but depending on what
462 # I don't fully understand why we need this one, but depending on what
458 # directory the test suite is run from, if we don't give it, 0 tests
463 # directory the test suite is run from, if we don't give it, 0 tests
459 # get run. Specifically, if the test suite is run from the source dir
464 # get run. Specifically, if the test suite is run from the source dir
460 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
465 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
461 # even if the same call done in this directory works fine). It appears
466 # even if the same call done in this directory works fine). It appears
462 # that if the requested package is in the current dir, nose bails early
467 # that if the requested package is in the current dir, nose bails early
463 # by default. Since it's otherwise harmless, leave it in by default
468 # by default. Since it's otherwise harmless, leave it in by default
464 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
469 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
465 argv.append('--traverse-namespace')
470 argv.append('--traverse-namespace')
466
471
467 # use our plugin for doctesting. It will remove the standard doctest plugin
472 # use our plugin for doctesting. It will remove the standard doctest plugin
468 # if it finds it enabled
473 # if it finds it enabled
469 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
474 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
470 # We need a global ipython running in this process
475 # We need a global ipython running in this process
471 globalipapp.start_ipython()
476 globalipapp.start_ipython()
472 # Now nose can run
477 # Now nose can run
473 TestProgram(argv=argv, addplugins=plugins)
478 TestProgram(argv=argv, addplugins=plugins)
474
479
475
480
476 def run_iptestall():
481 def run_iptestall():
477 """Run the entire IPython test suite by calling nose and trial.
482 """Run the entire IPython test suite by calling nose and trial.
478
483
479 This function constructs :class:`IPTester` instances for all IPython
484 This function constructs :class:`IPTester` instances for all IPython
480 modules and package and then runs each of them. This causes the modules
485 modules and package and then runs each of them. This causes the modules
481 and packages of IPython to be tested each in their own subprocess using
486 and packages of IPython to be tested each in their own subprocess using
482 nose.
487 nose.
483 """
488 """
484
489
485 runners = make_runners()
490 runners = make_runners()
486
491
487 # Run the test runners in a temporary dir so we can nuke it when finished
492 # Run the test runners in a temporary dir so we can nuke it when finished
488 # to clean up any junk files left over by accident. This also makes it
493 # to clean up any junk files left over by accident. This also makes it
489 # robust against being run in non-writeable directories by mistake, as the
494 # robust against being run in non-writeable directories by mistake, as the
490 # temp dir will always be user-writeable.
495 # temp dir will always be user-writeable.
491 curdir = os.getcwdu()
496 curdir = os.getcwdu()
492 testdir = tempfile.gettempdir()
497 testdir = tempfile.gettempdir()
493 os.chdir(testdir)
498 os.chdir(testdir)
494
499
495 # Run all test runners, tracking execution time
500 # Run all test runners, tracking execution time
496 failed = []
501 failed = []
497 t_start = time.time()
502 t_start = time.time()
498 try:
503 try:
499 for (name, runner) in runners:
504 for (name, runner) in runners:
500 print '*'*70
505 print '*'*70
501 print 'IPython test group:',name
506 print 'IPython test group:',name
502 res = runner.run()
507 res = runner.run()
503 if res:
508 if res:
504 failed.append( (name, runner) )
509 failed.append( (name, runner) )
505 finally:
510 finally:
506 os.chdir(curdir)
511 os.chdir(curdir)
507 t_end = time.time()
512 t_end = time.time()
508 t_tests = t_end - t_start
513 t_tests = t_end - t_start
509 nrunners = len(runners)
514 nrunners = len(runners)
510 nfail = len(failed)
515 nfail = len(failed)
511 # summarize results
516 # summarize results
512 print
517 print
513 print '*'*70
518 print '*'*70
514 print 'Test suite completed for system with the following information:'
519 print 'Test suite completed for system with the following information:'
515 print report()
520 print report()
516 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
521 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
517 print
522 print
518 print 'Status:'
523 print 'Status:'
519 if not failed:
524 if not failed:
520 print 'OK'
525 print 'OK'
521 else:
526 else:
522 # If anything went wrong, point out what command to rerun manually to
527 # If anything went wrong, point out what command to rerun manually to
523 # see the actual errors and individual summary
528 # see the actual errors and individual summary
524 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
529 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
525 for name, failed_runner in failed:
530 for name, failed_runner in failed:
526 print '-'*40
531 print '-'*40
527 print 'Runner failed:',name
532 print 'Runner failed:',name
528 print 'You may wish to rerun this one individually, with:'
533 print 'You may wish to rerun this one individually, with:'
529 print ' '.join(failed_runner.call_args)
534 print ' '.join(failed_runner.call_args)
530 print
535 print
531 # Ensure that our exit code indicates failure
536 # Ensure that our exit code indicates failure
532 sys.exit(1)
537 sys.exit(1)
533
538
534
539
535 def main():
540 def main():
536 for arg in sys.argv[1:]:
541 for arg in sys.argv[1:]:
537 if arg.startswith('IPython'):
542 if arg.startswith('IPython'):
538 # This is in-process
543 # This is in-process
539 run_iptest()
544 run_iptest()
540 else:
545 else:
541 # This starts subprocesses
546 # This starts subprocesses
542 run_iptestall()
547 run_iptestall()
543
548
544
549
545 if __name__ == '__main__':
550 if __name__ == '__main__':
546 main()
551 main()
General Comments 0
You need to be logged in to leave comments. Login now