##// END OF EJS Templates
Fixing small bug in iptest. Can now be run as "iptest".
Brian Granger -
Show More
@@ -1,297 +1,300 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) or trial recursively. This
8 calling this script (with different arguments) or trial recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 For now, this script requires that both nose and twisted are installed. This
15 For now, this script requires that both nose and twisted are installed. This
16 will change in the future.
16 will change in the future.
17 """
17 """
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Module imports
20 # Module imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import os.path as path
24 import os.path as path
25 import sys
25 import sys
26 import subprocess
26 import subprocess
27 import time
27 import time
28 import warnings
28 import warnings
29
29
30 import nose.plugins.builtin
30 import nose.plugins.builtin
31 from nose.core import TestProgram
31 from nose.core import TestProgram
32
32
33 from IPython.platutils import find_cmd
33 from IPython.platutils import find_cmd
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
35
35
36 pjoin = path.join
36 pjoin = path.join
37
37
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39 # Logic for skipping doctests
39 # Logic for skipping doctests
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41
41
42 def test_for(mod):
42 def test_for(mod):
43 """Test to see if mod is importable."""
43 """Test to see if mod is importable."""
44 try:
44 try:
45 __import__(mod)
45 __import__(mod)
46 except ImportError:
46 except ImportError:
47 return False
47 return False
48 else:
48 else:
49 return True
49 return True
50
50
51 have_curses = test_for('_curses')
51 have_curses = test_for('_curses')
52 have_wx = test_for('wx')
52 have_wx = test_for('wx')
53 have_zi = test_for('zope.interface')
53 have_zi = test_for('zope.interface')
54 have_twisted = test_for('twisted')
54 have_twisted = test_for('twisted')
55 have_foolscap = test_for('foolscap')
55 have_foolscap = test_for('foolscap')
56 have_objc = test_for('objc')
56 have_objc = test_for('objc')
57 have_pexpect = test_for('pexpect')
57 have_pexpect = test_for('pexpect')
58
58
59 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
59 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
60 # testing problems. We should strive to minimize the number of skipped
60 # testing problems. We should strive to minimize the number of skipped
61 # modules, since this means untested code. As the testing machinery
61 # modules, since this means untested code. As the testing machinery
62 # solidifies, this list should eventually become empty.
62 # solidifies, this list should eventually become empty.
63 EXCLUDE = [pjoin('IPython', 'external'),
63 EXCLUDE = [pjoin('IPython', 'external'),
64 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
64 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
65 pjoin('IPython_doctest_plugin'),
65 pjoin('IPython_doctest_plugin'),
66 pjoin('IPython', 'Gnuplot'),
66 pjoin('IPython', 'Gnuplot'),
67 pjoin('IPython', 'Extensions', 'ipy_'),
67 pjoin('IPython', 'Extensions', 'ipy_'),
68 pjoin('IPython', 'Extensions', 'clearcmd'),
68 pjoin('IPython', 'Extensions', 'clearcmd'),
69 pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
69 pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
70 pjoin('IPython', 'Extensions', 'scitedirector'),
70 pjoin('IPython', 'Extensions', 'scitedirector'),
71 pjoin('IPython', 'Extensions', 'numeric_formats'),
71 pjoin('IPython', 'Extensions', 'numeric_formats'),
72 pjoin('IPython', 'testing', 'attic'),
72 pjoin('IPython', 'testing', 'attic'),
73 pjoin('IPython', 'testing', 'tutils'),
73 pjoin('IPython', 'testing', 'tutils'),
74 pjoin('IPython', 'testing', 'tools'),
74 pjoin('IPython', 'testing', 'tools'),
75 pjoin('IPython', 'testing', 'mkdoctests')
75 pjoin('IPython', 'testing', 'mkdoctests')
76 ]
76 ]
77
77
78 if not have_wx:
78 if not have_wx:
79 EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
79 EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
80 EXCLUDE.append(pjoin('IPython', 'gui'))
80 EXCLUDE.append(pjoin('IPython', 'gui'))
81 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
81 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
82
82
83 if not have_objc:
83 if not have_objc:
84 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
84 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
85
85
86 if not have_curses:
86 if not have_curses:
87 EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
87 EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
88
88
89 if not sys.platform == 'win32':
89 if not sys.platform == 'win32':
90 EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
90 EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
91
91
92 # These have to be skipped on win32 because the use echo, rm, cd, etc.
92 # These have to be skipped on win32 because the use echo, rm, cd, etc.
93 # See ticket https://bugs.launchpad.net/bugs/366982
93 # See ticket https://bugs.launchpad.net/bugs/366982
94 if sys.platform == 'win32':
94 if sys.platform == 'win32':
95 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
95 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
96 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
96 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
97
97
98 if not os.name == 'posix':
98 if not os.name == 'posix':
99 EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
99 EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
100
100
101 if not have_pexpect:
101 if not have_pexpect:
102 EXCLUDE.append(pjoin('IPython', 'irunner'))
102 EXCLUDE.append(pjoin('IPython', 'irunner'))
103
103
104 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
104 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
105 if sys.platform == 'win32':
105 if sys.platform == 'win32':
106 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
106 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
107
107
108
108
109 #-----------------------------------------------------------------------------
109 #-----------------------------------------------------------------------------
110 # Functions and classes
110 # Functions and classes
111 #-----------------------------------------------------------------------------
111 #-----------------------------------------------------------------------------
112
112
113 def run_iptest():
113 def run_iptest():
114 """Run the IPython test suite using nose.
114 """Run the IPython test suite using nose.
115
115
116 This function is called when this script is **not** called with the form
116 This function is called when this script is **not** called with the form
117 `iptest all`. It simply calls nose with appropriate command line flags
117 `iptest all`. It simply calls nose with appropriate command line flags
118 and accepts all of the standard nose arguments.
118 and accepts all of the standard nose arguments.
119 """
119 """
120
120
121 warnings.filterwarnings('ignore',
121 warnings.filterwarnings('ignore',
122 'This will be removed soon. Use IPython.testing.util instead')
122 'This will be removed soon. Use IPython.testing.util instead')
123
123
124 argv = sys.argv + [
124 argv = sys.argv + [
125 # Loading ipdoctest causes problems with Twisted.
125 # Loading ipdoctest causes problems with Twisted.
126 # I am removing this as a temporary fix to get the
126 # I am removing this as a temporary fix to get the
127 # test suite back into working shape. Our nose
127 # test suite back into working shape. Our nose
128 # plugin needs to be gone through with a fine
128 # plugin needs to be gone through with a fine
129 # toothed comb to find what is causing the problem.
129 # toothed comb to find what is causing the problem.
130 '--with-ipdoctest',
130 '--with-ipdoctest',
131 '--ipdoctest-tests','--ipdoctest-extension=txt',
131 '--ipdoctest-tests','--ipdoctest-extension=txt',
132 '--detailed-errors',
132 '--detailed-errors',
133
133
134 # We add --exe because of setuptools' imbecility (it
134 # We add --exe because of setuptools' imbecility (it
135 # blindly does chmod +x on ALL files). Nose does the
135 # blindly does chmod +x on ALL files). Nose does the
136 # right thing and it tries to avoid executables,
136 # right thing and it tries to avoid executables,
137 # setuptools unfortunately forces our hand here. This
137 # setuptools unfortunately forces our hand here. This
138 # has been discussed on the distutils list and the
138 # has been discussed on the distutils list and the
139 # setuptools devs refuse to fix this problem!
139 # setuptools devs refuse to fix this problem!
140 '--exe',
140 '--exe',
141 ]
141 ]
142
142
143 # Detect if any tests were required by explicitly calling an IPython
143 # Detect if any tests were required by explicitly calling an IPython
144 # submodule or giving a specific path
144 # submodule or giving a specific path
145 has_tests = False
145 has_tests = False
146 for arg in sys.argv:
146 for arg in sys.argv:
147 if 'IPython' in arg or arg.endswith('.py') or \
147 if 'IPython' in arg or arg.endswith('.py') or \
148 (':' in arg and '.py' in arg):
148 (':' in arg and '.py' in arg):
149 has_tests = True
149 has_tests = True
150 break
150 break
151
151
152 # If nothing was specifically requested, test full IPython
152 # If nothing was specifically requested, test full IPython
153 if not has_tests:
153 if not has_tests:
154 argv.append('IPython')
154 argv.append('IPython')
155
155
156 # Construct list of plugins, omitting the existing doctest plugin, which
156 # Construct list of plugins, omitting the existing doctest plugin, which
157 # ours replaces (and extends).
157 # ours replaces (and extends).
158 plugins = [IPythonDoctest(EXCLUDE)]
158 plugins = [IPythonDoctest(EXCLUDE)]
159 for p in nose.plugins.builtin.plugins:
159 for p in nose.plugins.builtin.plugins:
160 plug = p()
160 plug = p()
161 if plug.name == 'doctest':
161 if plug.name == 'doctest':
162 continue
162 continue
163
163
164 #print '*** adding plugin:',plug.name # dbg
164 #print '*** adding plugin:',plug.name # dbg
165 plugins.append(plug)
165 plugins.append(plug)
166
166
167 TestProgram(argv=argv,plugins=plugins)
167 TestProgram(argv=argv,plugins=plugins)
168
168
169
169
170 class IPTester(object):
170 class IPTester(object):
171 """Call that calls iptest or trial in a subprocess.
171 """Call that calls iptest or trial in a subprocess.
172 """
172 """
173 def __init__(self,runner='iptest',params=None):
173 def __init__(self,runner='iptest',params=None):
174 """ """
174 """ """
175 if runner == 'iptest':
175 if runner == 'iptest':
176 self.runner = ['iptest','-v']
176 self.runner = ['iptest','-v']
177 else:
177 else:
178 self.runner = [find_cmd('trial')]
178 self.runner = [find_cmd('trial')]
179 if params is None:
179 if params is None:
180 params = []
180 params = []
181 if isinstance(params,str):
181 if isinstance(params,str):
182 params = [params]
182 params = [params]
183 self.params = params
183 self.params = params
184
184
185 # Assemble call
185 # Assemble call
186 self.call_args = self.runner+self.params
186 self.call_args = self.runner+self.params
187
187
188 def run(self):
188 def run(self):
189 """Run the stored commands"""
189 """Run the stored commands"""
190 return subprocess.call(self.call_args)
190 return subprocess.call(self.call_args)
191
191
192
192
193 def make_runners():
193 def make_runners():
194 """Define the modules and packages that need to be tested.
194 """Define the modules and packages that need to be tested.
195 """
195 """
196
196
197 # This omits additional top-level modules that should not be doctested.
197 # This omits additional top-level modules that should not be doctested.
198 # XXX: Shell.py is also ommited because of a bug in the skip_doctest
198 # XXX: Shell.py is also ommited because of a bug in the skip_doctest
199 # decorator. See ticket https://bugs.launchpad.net/bugs/366209
199 # decorator. See ticket https://bugs.launchpad.net/bugs/366209
200 top_mod = \
200 top_mod = \
201 ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
201 ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
202 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
202 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
203 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
203 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
204 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
204 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
205 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
205 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
206 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
206 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
207 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
207 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
208 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
208 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
209 'shellglobals.py', 'strdispatch.py', 'twshell.py',
209 'shellglobals.py', 'strdispatch.py', 'twshell.py',
210 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
210 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
211 # See note above for why this is skipped
211 # See note above for why this is skipped
212 # 'Shell.py',
212 # 'Shell.py',
213 'winconsole.py']
213 'winconsole.py']
214
214
215 if have_pexpect:
215 if have_pexpect:
216 top_mod.append('irunner.py')
216 top_mod.append('irunner.py')
217
217
218 if sys.platform == 'win32':
218 if sys.platform == 'win32':
219 top_mod.append('platutils_win32.py')
219 top_mod.append('platutils_win32.py')
220 elif os.name == 'posix':
220 elif os.name == 'posix':
221 top_mod.append('platutils_posix.py')
221 top_mod.append('platutils_posix.py')
222 else:
222 else:
223 top_mod.append('platutils_dummy.py')
223 top_mod.append('platutils_dummy.py')
224
224
225 # These are tested by nose, so skip IPython.kernel
225 # These are tested by nose, so skip IPython.kernel
226 top_pack = ['config','Extensions','frontend',
226 top_pack = ['config','Extensions','frontend',
227 'testing','tests','tools','UserConfig']
227 'testing','tests','tools','UserConfig']
228
228
229 if have_wx:
229 if have_wx:
230 top_pack.append('gui')
230 top_pack.append('gui')
231
231
232 modules = ['IPython.%s' % m[:-3] for m in top_mod ]
232 modules = ['IPython.%s' % m[:-3] for m in top_mod ]
233 packages = ['IPython.%s' % m for m in top_pack ]
233 packages = ['IPython.%s' % m for m in top_pack ]
234
234
235 # Make runners
235 # Make runners
236 runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
236 runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
237
237
238 # Test IPython.kernel using trial if twisted is installed
238 # Test IPython.kernel using trial if twisted is installed
239 if have_zi and have_twisted and have_foolscap:
239 if have_zi and have_twisted and have_foolscap:
240 runners['trial'] = IPTester('trial',['IPython'])
240 runners['trial'] = IPTester('trial',['IPython'])
241
241
242 runners['modules'] = IPTester(params=modules)
242 runners['modules'] = IPTester(params=modules)
243
243
244 return runners
244 return runners
245
245
246
246
247 def run_iptestall():
247 def run_iptestall():
248 """Run the entire IPython test suite by calling nose and trial.
248 """Run the entire IPython test suite by calling nose and trial.
249
249
250 This function constructs :class:`IPTester` instances for all IPython
250 This function constructs :class:`IPTester` instances for all IPython
251 modules and package and then runs each of them. This causes the modules
251 modules and package and then runs each of them. This causes the modules
252 and packages of IPython to be tested each in their own subprocess using
252 and packages of IPython to be tested each in their own subprocess using
253 nose or twisted.trial appropriately.
253 nose or twisted.trial appropriately.
254 """
254 """
255 runners = make_runners()
255 runners = make_runners()
256 # Run all test runners, tracking execution time
256 # Run all test runners, tracking execution time
257 failed = {}
257 failed = {}
258 t_start = time.time()
258 t_start = time.time()
259 for name,runner in runners.iteritems():
259 for name,runner in runners.iteritems():
260 print '*'*77
260 print '*'*77
261 print 'IPython test set:',name
261 print 'IPython test set:',name
262 res = runner.run()
262 res = runner.run()
263 if res:
263 if res:
264 failed[name] = res
264 failed[name] = res
265 t_end = time.time()
265 t_end = time.time()
266 t_tests = t_end - t_start
266 t_tests = t_end - t_start
267 nrunners = len(runners)
267 nrunners = len(runners)
268 nfail = len(failed)
268 nfail = len(failed)
269 # summarize results
269 # summarize results
270 print
270 print
271 print '*'*77
271 print '*'*77
272 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
272 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
273 print
273 print
274 if not failed:
274 if not failed:
275 print 'OK'
275 print 'OK'
276 else:
276 else:
277 # If anything went wrong, point out what command to rerun manually to
277 # If anything went wrong, point out what command to rerun manually to
278 # see the actual errors and individual summary
278 # see the actual errors and individual summary
279 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
279 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
280 for name in failed:
280 for name in failed:
281 failed_runner = runners[name]
281 failed_runner = runners[name]
282 print '-'*40
282 print '-'*40
283 print 'Runner failed:',name
283 print 'Runner failed:',name
284 print 'You may wish to rerun this one individually, with:'
284 print 'You may wish to rerun this one individually, with:'
285 print ' '.join(failed_runner.call_args)
285 print ' '.join(failed_runner.call_args)
286 print
286 print
287
287
288
288
289 def main():
289 def main():
290 if sys.argv[1] == 'all':
290 if len(sys.argv) == 1:
291 run_iptestall()
291 run_iptestall()
292 else:
292 else:
293 run_iptest()
293 if sys.argv[1] == 'all':
294 run_iptestall()
295 else:
296 run_iptest()
294
297
295
298
296 if __name__ == '__main__':
299 if __name__ == '__main__':
297 main() No newline at end of file
300 main()
General Comments 0
You need to be logged in to leave comments. Login now