##// END OF EJS Templates
Small fixes for wx-dependent tests and include clearcmd....
Fernando Perez -
Show More
@@ -1,300 +1,305 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) or trial recursively. This
8 calling this script (with different arguments) or trial recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 For now, this script requires that both nose and twisted are installed. This
15 For now, this script requires that both nose and twisted are installed. This
16 will change in the future.
16 will change in the future.
17 """
17 """
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Module imports
20 # Module imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import os.path as path
24 import os.path as path
25 import sys
25 import sys
26 import subprocess
26 import subprocess
27 import time
27 import time
28 import warnings
28 import warnings
29
29
30 import nose.plugins.builtin
30 import nose.plugins.builtin
31 from nose.core import TestProgram
31 from nose.core import TestProgram
32
32
33 from IPython.platutils import find_cmd
33 from IPython.platutils import find_cmd
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
35
35
36 pjoin = path.join
36 pjoin = path.join
37
37
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39 # Logic for skipping doctests
39 # Logic for skipping doctests
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41
41
42 def test_for(mod):
42 def test_for(mod):
43 """Test to see if mod is importable."""
43 """Test to see if mod is importable."""
44 try:
44 try:
45 __import__(mod)
45 __import__(mod)
46 except ImportError:
46 except ImportError:
47 return False
47 return False
48 else:
48 else:
49 return True
49 return True
50
50
51 have_curses = test_for('_curses')
51 have_curses = test_for('_curses')
52 have_wx = test_for('wx')
52 have_wx = test_for('wx')
53 have_wx_aui = test_for('wx.aui')
53 have_zi = test_for('zope.interface')
54 have_zi = test_for('zope.interface')
54 have_twisted = test_for('twisted')
55 have_twisted = test_for('twisted')
55 have_foolscap = test_for('foolscap')
56 have_foolscap = test_for('foolscap')
56 have_objc = test_for('objc')
57 have_objc = test_for('objc')
57 have_pexpect = test_for('pexpect')
58 have_pexpect = test_for('pexpect')
58
59
59 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
60 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
60 # testing problems. We should strive to minimize the number of skipped
61 # testing problems. We should strive to minimize the number of skipped
61 # modules, since this means untested code. As the testing machinery
62 # modules, since this means untested code. As the testing machinery
62 # solidifies, this list should eventually become empty.
63 # solidifies, this list should eventually become empty.
63 EXCLUDE = [pjoin('IPython', 'external'),
64 EXCLUDE = [pjoin('IPython', 'external'),
64 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
65 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
65 pjoin('IPython_doctest_plugin'),
66 pjoin('IPython_doctest_plugin'),
66 pjoin('IPython', 'Gnuplot'),
67 pjoin('IPython', 'Gnuplot'),
67 pjoin('IPython', 'Extensions', 'ipy_'),
68 pjoin('IPython', 'Extensions', 'ipy_'),
68 pjoin('IPython', 'Extensions', 'clearcmd'),
69 pjoin('IPython', 'Extensions', 'PhysicalQInput'),
69 pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
70 pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
71 pjoin('IPython', 'Extensions', 'InterpreterPasteInput'),
70 pjoin('IPython', 'Extensions', 'scitedirector'),
72 pjoin('IPython', 'Extensions', 'scitedirector'),
71 pjoin('IPython', 'Extensions', 'numeric_formats'),
73 pjoin('IPython', 'Extensions', 'numeric_formats'),
72 pjoin('IPython', 'testing', 'attic'),
74 pjoin('IPython', 'testing', 'attic'),
73 pjoin('IPython', 'testing', 'tutils'),
75 pjoin('IPython', 'testing', 'tutils'),
74 pjoin('IPython', 'testing', 'tools'),
76 pjoin('IPython', 'testing', 'tools'),
75 pjoin('IPython', 'testing', 'mkdoctests')
77 pjoin('IPython', 'testing', 'mkdoctests'),
76 ]
78 ]
77
79
78 if not have_wx:
80 if not have_wx:
79 EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
81 EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
80 EXCLUDE.append(pjoin('IPython', 'gui'))
82 EXCLUDE.append(pjoin('IPython', 'gui'))
81 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
83 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
82
84
85 if not have_wx_aui:
86 EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
87
83 if not have_objc:
88 if not have_objc:
84 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
89 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
85
90
86 if not have_curses:
91 if not have_curses:
87 EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
92 EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
88
93
89 if not sys.platform == 'win32':
94 if not sys.platform == 'win32':
90 EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
95 EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
91
96
92 # These have to be skipped on win32 because the use echo, rm, cd, etc.
97 # These have to be skipped on win32 because the use echo, rm, cd, etc.
93 # See ticket https://bugs.launchpad.net/bugs/366982
98 # See ticket https://bugs.launchpad.net/bugs/366982
94 if sys.platform == 'win32':
99 if sys.platform == 'win32':
95 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
100 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
96 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
101 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
97
102
98 if not os.name == 'posix':
103 if not os.name == 'posix':
99 EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
104 EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
100
105
101 if not have_pexpect:
106 if not have_pexpect:
102 EXCLUDE.append(pjoin('IPython', 'irunner'))
107 EXCLUDE.append(pjoin('IPython', 'irunner'))
103
108
104 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
109 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
105 if sys.platform == 'win32':
110 if sys.platform == 'win32':
106 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
111 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
107
112
108
113
109 #-----------------------------------------------------------------------------
114 #-----------------------------------------------------------------------------
110 # Functions and classes
115 # Functions and classes
111 #-----------------------------------------------------------------------------
116 #-----------------------------------------------------------------------------
112
117
113 def run_iptest():
118 def run_iptest():
114 """Run the IPython test suite using nose.
119 """Run the IPython test suite using nose.
115
120
116 This function is called when this script is **not** called with the form
121 This function is called when this script is **not** called with the form
117 `iptest all`. It simply calls nose with appropriate command line flags
122 `iptest all`. It simply calls nose with appropriate command line flags
118 and accepts all of the standard nose arguments.
123 and accepts all of the standard nose arguments.
119 """
124 """
120
125
121 warnings.filterwarnings('ignore',
126 warnings.filterwarnings('ignore',
122 'This will be removed soon. Use IPython.testing.util instead')
127 'This will be removed soon. Use IPython.testing.util instead')
123
128
124 argv = sys.argv + [
129 argv = sys.argv + [
125 # Loading ipdoctest causes problems with Twisted.
130 # Loading ipdoctest causes problems with Twisted.
126 # I am removing this as a temporary fix to get the
131 # I am removing this as a temporary fix to get the
127 # test suite back into working shape. Our nose
132 # test suite back into working shape. Our nose
128 # plugin needs to be gone through with a fine
133 # plugin needs to be gone through with a fine
129 # toothed comb to find what is causing the problem.
134 # toothed comb to find what is causing the problem.
130 '--with-ipdoctest',
135 '--with-ipdoctest',
131 '--ipdoctest-tests','--ipdoctest-extension=txt',
136 '--ipdoctest-tests','--ipdoctest-extension=txt',
132 '--detailed-errors',
137 '--detailed-errors',
133
138
134 # We add --exe because of setuptools' imbecility (it
139 # We add --exe because of setuptools' imbecility (it
135 # blindly does chmod +x on ALL files). Nose does the
140 # blindly does chmod +x on ALL files). Nose does the
136 # right thing and it tries to avoid executables,
141 # right thing and it tries to avoid executables,
137 # setuptools unfortunately forces our hand here. This
142 # setuptools unfortunately forces our hand here. This
138 # has been discussed on the distutils list and the
143 # has been discussed on the distutils list and the
139 # setuptools devs refuse to fix this problem!
144 # setuptools devs refuse to fix this problem!
140 '--exe',
145 '--exe',
141 ]
146 ]
142
147
143 # Detect if any tests were required by explicitly calling an IPython
148 # Detect if any tests were required by explicitly calling an IPython
144 # submodule or giving a specific path
149 # submodule or giving a specific path
145 has_tests = False
150 has_tests = False
146 for arg in sys.argv:
151 for arg in sys.argv:
147 if 'IPython' in arg or arg.endswith('.py') or \
152 if 'IPython' in arg or arg.endswith('.py') or \
148 (':' in arg and '.py' in arg):
153 (':' in arg and '.py' in arg):
149 has_tests = True
154 has_tests = True
150 break
155 break
151
156
152 # If nothing was specifically requested, test full IPython
157 # If nothing was specifically requested, test full IPython
153 if not has_tests:
158 if not has_tests:
154 argv.append('IPython')
159 argv.append('IPython')
155
160
156 # Construct list of plugins, omitting the existing doctest plugin, which
161 # Construct list of plugins, omitting the existing doctest plugin, which
157 # ours replaces (and extends).
162 # ours replaces (and extends).
158 plugins = [IPythonDoctest(EXCLUDE)]
163 plugins = [IPythonDoctest(EXCLUDE)]
159 for p in nose.plugins.builtin.plugins:
164 for p in nose.plugins.builtin.plugins:
160 plug = p()
165 plug = p()
161 if plug.name == 'doctest':
166 if plug.name == 'doctest':
162 continue
167 continue
163
168
164 #print '*** adding plugin:',plug.name # dbg
169 #print '*** adding plugin:',plug.name # dbg
165 plugins.append(plug)
170 plugins.append(plug)
166
171
167 TestProgram(argv=argv,plugins=plugins)
172 TestProgram(argv=argv,plugins=plugins)
168
173
169
174
170 class IPTester(object):
175 class IPTester(object):
171 """Call that calls iptest or trial in a subprocess.
176 """Call that calls iptest or trial in a subprocess.
172 """
177 """
173 def __init__(self,runner='iptest',params=None):
178 def __init__(self,runner='iptest',params=None):
174 """ """
179 """ """
175 if runner == 'iptest':
180 if runner == 'iptest':
176 self.runner = ['iptest','-v']
181 self.runner = ['iptest','-v']
177 else:
182 else:
178 self.runner = [find_cmd('trial')]
183 self.runner = [find_cmd('trial')]
179 if params is None:
184 if params is None:
180 params = []
185 params = []
181 if isinstance(params,str):
186 if isinstance(params,str):
182 params = [params]
187 params = [params]
183 self.params = params
188 self.params = params
184
189
185 # Assemble call
190 # Assemble call
186 self.call_args = self.runner+self.params
191 self.call_args = self.runner+self.params
187
192
188 def run(self):
193 def run(self):
189 """Run the stored commands"""
194 """Run the stored commands"""
190 return subprocess.call(self.call_args)
195 return subprocess.call(self.call_args)
191
196
192
197
193 def make_runners():
198 def make_runners():
194 """Define the modules and packages that need to be tested.
199 """Define the modules and packages that need to be tested.
195 """
200 """
196
201
197 # This omits additional top-level modules that should not be doctested.
202 # This omits additional top-level modules that should not be doctested.
198 # XXX: Shell.py is also ommited because of a bug in the skip_doctest
203 # XXX: Shell.py is also ommited because of a bug in the skip_doctest
199 # decorator. See ticket https://bugs.launchpad.net/bugs/366209
204 # decorator. See ticket https://bugs.launchpad.net/bugs/366209
200 top_mod = \
205 top_mod = \
201 ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
206 ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
202 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
207 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
203 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
208 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
204 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
209 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
205 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
210 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
206 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
211 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
207 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
212 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
208 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
213 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
209 'shellglobals.py', 'strdispatch.py', 'twshell.py',
214 'shellglobals.py', 'strdispatch.py', 'twshell.py',
210 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
215 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
211 # See note above for why this is skipped
216 # See note above for why this is skipped
212 # 'Shell.py',
217 # 'Shell.py',
213 'winconsole.py']
218 'winconsole.py']
214
219
215 if have_pexpect:
220 if have_pexpect:
216 top_mod.append('irunner.py')
221 top_mod.append('irunner.py')
217
222
218 if sys.platform == 'win32':
223 if sys.platform == 'win32':
219 top_mod.append('platutils_win32.py')
224 top_mod.append('platutils_win32.py')
220 elif os.name == 'posix':
225 elif os.name == 'posix':
221 top_mod.append('platutils_posix.py')
226 top_mod.append('platutils_posix.py')
222 else:
227 else:
223 top_mod.append('platutils_dummy.py')
228 top_mod.append('platutils_dummy.py')
224
229
225 # These are tested by nose, so skip IPython.kernel
230 # These are tested by nose, so skip IPython.kernel
226 top_pack = ['config','Extensions','frontend',
231 top_pack = ['config','Extensions','frontend',
227 'testing','tests','tools','UserConfig']
232 'testing','tests','tools','UserConfig']
228
233
229 if have_wx:
234 if have_wx:
230 top_pack.append('gui')
235 top_pack.append('gui')
231
236
232 modules = ['IPython.%s' % m[:-3] for m in top_mod ]
237 modules = ['IPython.%s' % m[:-3] for m in top_mod ]
233 packages = ['IPython.%s' % m for m in top_pack ]
238 packages = ['IPython.%s' % m for m in top_pack ]
234
239
235 # Make runners
240 # Make runners
236 runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
241 runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
237
242
238 # Test IPython.kernel using trial if twisted is installed
243 # Test IPython.kernel using trial if twisted is installed
239 if have_zi and have_twisted and have_foolscap:
244 if have_zi and have_twisted and have_foolscap:
240 runners['trial'] = IPTester('trial',['IPython'])
245 runners['trial'] = IPTester('trial',['IPython'])
241
246
242 runners['modules'] = IPTester(params=modules)
247 runners['modules'] = IPTester(params=modules)
243
248
244 return runners
249 return runners
245
250
246
251
247 def run_iptestall():
252 def run_iptestall():
248 """Run the entire IPython test suite by calling nose and trial.
253 """Run the entire IPython test suite by calling nose and trial.
249
254
250 This function constructs :class:`IPTester` instances for all IPython
255 This function constructs :class:`IPTester` instances for all IPython
251 modules and package and then runs each of them. This causes the modules
256 modules and package and then runs each of them. This causes the modules
252 and packages of IPython to be tested each in their own subprocess using
257 and packages of IPython to be tested each in their own subprocess using
253 nose or twisted.trial appropriately.
258 nose or twisted.trial appropriately.
254 """
259 """
255 runners = make_runners()
260 runners = make_runners()
256 # Run all test runners, tracking execution time
261 # Run all test runners, tracking execution time
257 failed = {}
262 failed = {}
258 t_start = time.time()
263 t_start = time.time()
259 for name,runner in runners.iteritems():
264 for name,runner in runners.iteritems():
260 print '*'*77
265 print '*'*77
261 print 'IPython test set:',name
266 print 'IPython test group:',name
262 res = runner.run()
267 res = runner.run()
263 if res:
268 if res:
264 failed[name] = res
269 failed[name] = res
265 t_end = time.time()
270 t_end = time.time()
266 t_tests = t_end - t_start
271 t_tests = t_end - t_start
267 nrunners = len(runners)
272 nrunners = len(runners)
268 nfail = len(failed)
273 nfail = len(failed)
269 # summarize results
274 # summarize results
270 print
275 print
271 print '*'*77
276 print '*'*77
272 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
277 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
273 print
278 print
274 if not failed:
279 if not failed:
275 print 'OK'
280 print 'OK'
276 else:
281 else:
277 # If anything went wrong, point out what command to rerun manually to
282 # If anything went wrong, point out what command to rerun manually to
278 # see the actual errors and individual summary
283 # see the actual errors and individual summary
279 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
284 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
280 for name in failed:
285 for name in failed:
281 failed_runner = runners[name]
286 failed_runner = runners[name]
282 print '-'*40
287 print '-'*40
283 print 'Runner failed:',name
288 print 'Runner failed:',name
284 print 'You may wish to rerun this one individually, with:'
289 print 'You may wish to rerun this one individually, with:'
285 print ' '.join(failed_runner.call_args)
290 print ' '.join(failed_runner.call_args)
286 print
291 print
287
292
288
293
289 def main():
294 def main():
290 if len(sys.argv) == 1:
295 if len(sys.argv) == 1:
291 run_iptestall()
296 run_iptestall()
292 else:
297 else:
293 if sys.argv[1] == 'all':
298 if sys.argv[1] == 'all':
294 run_iptestall()
299 run_iptestall()
295 else:
300 else:
296 run_iptest()
301 run_iptest()
297
302
298
303
299 if __name__ == '__main__':
304 if __name__ == '__main__':
300 main() No newline at end of file
305 main()
General Comments 0
You need to be logged in to leave comments. Login now