##// END OF EJS Templates
Updated iptest to skip inputhook*.py files for doctesting.
Brian Granger -
Show More
@@ -1,279 +1,286 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) or trial recursively. This
8 calling this script (with different arguments) or trial recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 For now, this script requires that both nose and twisted are installed. This
15 For now, this script requires that both nose and twisted are installed. This
16 will change in the future.
16 will change in the future.
17 """
17 """
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Module imports
20 # Module imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import os.path as path
24 import os.path as path
25 import sys
25 import sys
26 import subprocess
26 import subprocess
27 import time
27 import time
28 import warnings
28 import warnings
29
29
30 import nose.plugins.builtin
30 import nose.plugins.builtin
31 from nose.core import TestProgram
31 from nose.core import TestProgram
32
32
33 from IPython.utils.platutils import find_cmd
33 from IPython.utils.platutils import find_cmd
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
34 from IPython.testing.plugin.ipdoctest import IPythonDoctest
35
35
36 pjoin = path.join
36 pjoin = path.join
37
37
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39 # Logic for skipping doctests
39 # Logic for skipping doctests
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41
41
42 def test_for(mod):
42 def test_for(mod):
43 """Test to see if mod is importable."""
43 """Test to see if mod is importable."""
44 try:
44 try:
45 __import__(mod)
45 __import__(mod)
46 except ImportError:
46 except ImportError:
47 return False
47 return False
48 else:
48 else:
49 return True
49 return True
50
50
51 have_curses = test_for('_curses')
51 have_curses = test_for('_curses')
52 have_wx = test_for('wx')
52 have_wx = test_for('wx')
53 have_zi = test_for('zope.interface')
53 have_zi = test_for('zope.interface')
54 have_twisted = test_for('twisted')
54 have_twisted = test_for('twisted')
55 have_foolscap = test_for('foolscap')
55 have_foolscap = test_for('foolscap')
56 have_objc = test_for('objc')
56 have_objc = test_for('objc')
57 have_pexpect = test_for('pexpect')
57 have_pexpect = test_for('pexpect')
58 have_gtk = test_for('gtk')
59 have_gobject = test_for('gobject')
58
60
59
61
60 def make_exclude():
62 def make_exclude():
61
63
62 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
64 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
63 # testing problems. We should strive to minimize the number of skipped
65 # testing problems. We should strive to minimize the number of skipped
64 # modules, since this means untested code. As the testing machinery
66 # modules, since this means untested code. As the testing machinery
65 # solidifies, this list should eventually become empty.
67 # solidifies, this list should eventually become empty.
66 EXCLUDE = [pjoin('IPython', 'external'),
68 EXCLUDE = [pjoin('IPython', 'external'),
67 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
69 pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
68 pjoin('IPython_doctest_plugin'),
70 pjoin('IPython_doctest_plugin'),
69 pjoin('IPython', 'extensions', 'ipy_'),
71 pjoin('IPython', 'extensions', 'ipy_'),
70 pjoin('IPython', 'extensions', 'clearcmd'),
72 pjoin('IPython', 'extensions', 'clearcmd'),
71 pjoin('IPython', 'extensions', 'PhysicalQInteractive'),
73 pjoin('IPython', 'extensions', 'PhysicalQInteractive'),
72 pjoin('IPython', 'extensions', 'scitedirector'),
74 pjoin('IPython', 'extensions', 'scitedirector'),
73 pjoin('IPython', 'extensions', 'numeric_formats'),
75 pjoin('IPython', 'extensions', 'numeric_formats'),
74 pjoin('IPython', 'testing', 'attic'),
76 pjoin('IPython', 'testing', 'attic'),
75 pjoin('IPython', 'testing', 'tools'),
77 pjoin('IPython', 'testing', 'tools'),
76 pjoin('IPython', 'testing', 'mkdoctests')
78 pjoin('IPython', 'testing', 'mkdoctests'),
79 pjoin('IPython', 'lib', 'inputhook')
77 ]
80 ]
78
81
79 if not have_wx:
82 if not have_wx:
80 EXCLUDE.append(pjoin('IPython', 'extensions', 'igrid'))
83 EXCLUDE.append(pjoin('IPython', 'extensions', 'igrid'))
81 EXCLUDE.append(pjoin('IPython', 'gui'))
84 EXCLUDE.append(pjoin('IPython', 'gui'))
82 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
85 EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
86 EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookwx'))
87
88 if not have_gtk or not have_gobject:
89 EXCLUDE.append(pjoin('IPython', 'lib', 'inputhookgtk'))
83
90
84 if not have_objc:
91 if not have_objc:
85 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
92 EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
86
93
87 if not have_curses:
94 if not have_curses:
88 EXCLUDE.append(pjoin('IPython', 'extensions', 'ibrowse'))
95 EXCLUDE.append(pjoin('IPython', 'extensions', 'ibrowse'))
89
96
90 if not sys.platform == 'win32':
97 if not sys.platform == 'win32':
91 EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
98 EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
92
99
93 # These have to be skipped on win32 because the use echo, rm, cd, etc.
100 # These have to be skipped on win32 because the use echo, rm, cd, etc.
94 # See ticket https://bugs.launchpad.net/bugs/366982
101 # See ticket https://bugs.launchpad.net/bugs/366982
95 if sys.platform == 'win32':
102 if sys.platform == 'win32':
96 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
103 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
97 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
104 EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
98
105
99 if not os.name == 'posix':
106 if not os.name == 'posix':
100 EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
107 EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
101
108
102 if not have_pexpect:
109 if not have_pexpect:
103 EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
110 EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
104
111
105 # Skip shell always because of a bug in FakeModule.
112 # Skip shell always because of a bug in FakeModule.
106 EXCLUDE.append(pjoin('IPython', 'core', 'shell'))
113 EXCLUDE.append(pjoin('IPython', 'core', 'shell'))
107
114
108 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
115 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
109 if sys.platform == 'win32':
116 if sys.platform == 'win32':
110 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
117 EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
111
118
112 return EXCLUDE
119 return EXCLUDE
113
120
114 #-----------------------------------------------------------------------------
121 #-----------------------------------------------------------------------------
115 # Functions and classes
122 # Functions and classes
116 #-----------------------------------------------------------------------------
123 #-----------------------------------------------------------------------------
117
124
118 def run_iptest():
125 def run_iptest():
119 """Run the IPython test suite using nose.
126 """Run the IPython test suite using nose.
120
127
121 This function is called when this script is **not** called with the form
128 This function is called when this script is **not** called with the form
122 `iptest all`. It simply calls nose with appropriate command line flags
129 `iptest all`. It simply calls nose with appropriate command line flags
123 and accepts all of the standard nose arguments.
130 and accepts all of the standard nose arguments.
124 """
131 """
125
132
126 warnings.filterwarnings('ignore',
133 warnings.filterwarnings('ignore',
127 'This will be removed soon. Use IPython.testing.util instead')
134 'This will be removed soon. Use IPython.testing.util instead')
128
135
129 argv = sys.argv + [
136 argv = sys.argv + [
130 # Loading ipdoctest causes problems with Twisted.
137 # Loading ipdoctest causes problems with Twisted.
131 # I am removing this as a temporary fix to get the
138 # I am removing this as a temporary fix to get the
132 # test suite back into working shape. Our nose
139 # test suite back into working shape. Our nose
133 # plugin needs to be gone through with a fine
140 # plugin needs to be gone through with a fine
134 # toothed comb to find what is causing the problem.
141 # toothed comb to find what is causing the problem.
135 '--with-ipdoctest',
142 '--with-ipdoctest',
136 '--ipdoctest-tests','--ipdoctest-extension=txt',
143 '--ipdoctest-tests','--ipdoctest-extension=txt',
137 '--detailed-errors',
144 '--detailed-errors',
138
145
139 # We add --exe because of setuptools' imbecility (it
146 # We add --exe because of setuptools' imbecility (it
140 # blindly does chmod +x on ALL files). Nose does the
147 # blindly does chmod +x on ALL files). Nose does the
141 # right thing and it tries to avoid executables,
148 # right thing and it tries to avoid executables,
142 # setuptools unfortunately forces our hand here. This
149 # setuptools unfortunately forces our hand here. This
143 # has been discussed on the distutils list and the
150 # has been discussed on the distutils list and the
144 # setuptools devs refuse to fix this problem!
151 # setuptools devs refuse to fix this problem!
145 '--exe',
152 '--exe',
146 ]
153 ]
147
154
148 # Detect if any tests were required by explicitly calling an IPython
155 # Detect if any tests were required by explicitly calling an IPython
149 # submodule or giving a specific path
156 # submodule or giving a specific path
150 has_tests = False
157 has_tests = False
151 for arg in sys.argv:
158 for arg in sys.argv:
152 if 'IPython' in arg or arg.endswith('.py') or \
159 if 'IPython' in arg or arg.endswith('.py') or \
153 (':' in arg and '.py' in arg):
160 (':' in arg and '.py' in arg):
154 has_tests = True
161 has_tests = True
155 break
162 break
156
163
157 # If nothing was specifically requested, test full IPython
164 # If nothing was specifically requested, test full IPython
158 if not has_tests:
165 if not has_tests:
159 argv.append('IPython')
166 argv.append('IPython')
160
167
161 # Construct list of plugins, omitting the existing doctest plugin, which
168 # Construct list of plugins, omitting the existing doctest plugin, which
162 # ours replaces (and extends).
169 # ours replaces (and extends).
163 EXCLUDE = make_exclude()
170 EXCLUDE = make_exclude()
164 plugins = [IPythonDoctest(EXCLUDE)]
171 plugins = [IPythonDoctest(EXCLUDE)]
165 for p in nose.plugins.builtin.plugins:
172 for p in nose.plugins.builtin.plugins:
166 plug = p()
173 plug = p()
167 if plug.name == 'doctest':
174 if plug.name == 'doctest':
168 continue
175 continue
169 plugins.append(plug)
176 plugins.append(plug)
170
177
171 TestProgram(argv=argv,plugins=plugins)
178 TestProgram(argv=argv,plugins=plugins)
172
179
173
180
174 class IPTester(object):
181 class IPTester(object):
175 """Call that calls iptest or trial in a subprocess.
182 """Call that calls iptest or trial in a subprocess.
176 """
183 """
177 def __init__(self,runner='iptest',params=None):
184 def __init__(self,runner='iptest',params=None):
178 """ """
185 """ """
179 if runner == 'iptest':
186 if runner == 'iptest':
180 self.runner = ['iptest','-v']
187 self.runner = ['iptest','-v']
181 else:
188 else:
182 self.runner = [find_cmd('trial')]
189 self.runner = [find_cmd('trial')]
183 if params is None:
190 if params is None:
184 params = []
191 params = []
185 if isinstance(params,str):
192 if isinstance(params,str):
186 params = [params]
193 params = [params]
187 self.params = params
194 self.params = params
188
195
189 # Assemble call
196 # Assemble call
190 self.call_args = self.runner+self.params
197 self.call_args = self.runner+self.params
191
198
192 def run(self):
199 def run(self):
193 """Run the stored commands"""
200 """Run the stored commands"""
194 return subprocess.call(self.call_args)
201 return subprocess.call(self.call_args)
195
202
196
203
197 def make_runners():
204 def make_runners():
198 """Define the top-level packages that need to be tested.
205 """Define the top-level packages that need to be tested.
199 """
206 """
200
207
201 nose_packages = ['config', 'core', 'extensions',
208 nose_packages = ['config', 'core', 'extensions',
202 'frontend', 'lib', 'quarantine',
209 'frontend', 'lib', 'quarantine',
203 'scripts', 'testing', 'utils']
210 'scripts', 'testing', 'utils']
204 trial_packages = ['kernel']
211 trial_packages = ['kernel']
205
212
206 if have_wx:
213 if have_wx:
207 nose_packages.append('gui')
214 nose_packages.append('gui')
208
215
209 nose_packages = ['IPython.%s' % m for m in nose_packages ]
216 nose_packages = ['IPython.%s' % m for m in nose_packages ]
210 trial_packages = ['IPython.%s' % m for m in trial_packages ]
217 trial_packages = ['IPython.%s' % m for m in trial_packages ]
211
218
212 # Make runners
219 # Make runners
213 runners = dict()
220 runners = dict()
214
221
215 nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages]))
222 nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages]))
216 if have_zi and have_twisted and have_foolscap:
223 if have_zi and have_twisted and have_foolscap:
217 trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages]))
224 trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages]))
218 runners.update(nose_runners)
225 runners.update(nose_runners)
219 runners.update(trial_runners)
226 runners.update(trial_runners)
220
227
221 return runners
228 return runners
222
229
223
230
224 def run_iptestall():
231 def run_iptestall():
225 """Run the entire IPython test suite by calling nose and trial.
232 """Run the entire IPython test suite by calling nose and trial.
226
233
227 This function constructs :class:`IPTester` instances for all IPython
234 This function constructs :class:`IPTester` instances for all IPython
228 modules and package and then runs each of them. This causes the modules
235 modules and package and then runs each of them. This causes the modules
229 and packages of IPython to be tested each in their own subprocess using
236 and packages of IPython to be tested each in their own subprocess using
230 nose or twisted.trial appropriately.
237 nose or twisted.trial appropriately.
231 """
238 """
232
239
233 runners = make_runners()
240 runners = make_runners()
234
241
235 # Run all test runners, tracking execution time
242 # Run all test runners, tracking execution time
236 failed = {}
243 failed = {}
237 t_start = time.time()
244 t_start = time.time()
238 for name,runner in runners.iteritems():
245 for name,runner in runners.iteritems():
239 print '*'*77
246 print '*'*77
240 print 'IPython test set:', name
247 print 'IPython test set:', name
241 res = runner.run()
248 res = runner.run()
242 if res:
249 if res:
243 failed[name] = res
250 failed[name] = res
244 t_end = time.time()
251 t_end = time.time()
245 t_tests = t_end - t_start
252 t_tests = t_end - t_start
246 nrunners = len(runners)
253 nrunners = len(runners)
247 nfail = len(failed)
254 nfail = len(failed)
248 # summarize results
255 # summarize results
249 print
256 print
250 print '*'*77
257 print '*'*77
251 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
258 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
252 print
259 print
253 if not failed:
260 if not failed:
254 print 'OK'
261 print 'OK'
255 else:
262 else:
256 # If anything went wrong, point out what command to rerun manually to
263 # If anything went wrong, point out what command to rerun manually to
257 # see the actual errors and individual summary
264 # see the actual errors and individual summary
258 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
265 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
259 for name in failed:
266 for name in failed:
260 failed_runner = runners[name]
267 failed_runner = runners[name]
261 print '-'*40
268 print '-'*40
262 print 'Runner failed:',name
269 print 'Runner failed:',name
263 print 'You may wish to rerun this one individually, with:'
270 print 'You may wish to rerun this one individually, with:'
264 print ' '.join(failed_runner.call_args)
271 print ' '.join(failed_runner.call_args)
265 print
272 print
266
273
267
274
268 def main():
275 def main():
269 if len(sys.argv) == 1:
276 if len(sys.argv) == 1:
270 run_iptestall()
277 run_iptestall()
271 else:
278 else:
272 if sys.argv[1] == 'all':
279 if sys.argv[1] == 'all':
273 run_iptestall()
280 run_iptestall()
274 else:
281 else:
275 run_iptest()
282 run_iptest()
276
283
277
284
278 if __name__ == '__main__':
285 if __name__ == '__main__':
279 main() No newline at end of file
286 main()
General Comments 0
You need to be logged in to leave comments. Login now