##// END OF EJS Templates
Add testing function check_pairs to check input/output pairs against a function, and produce useful failure messages.
Thomas Kluyver -
Show More
@@ -1,283 +1,309 b''
1 """Generic testing tools that do NOT depend on Twisted.
1 """Generic testing tools that do NOT depend on Twisted.
2
2
3 In particular, this module exposes a set of top-level assert* functions that
3 In particular, this module exposes a set of top-level assert* functions that
4 can be used in place of nose.tools.assert* in method generators (the ones in
4 can be used in place of nose.tools.assert* in method generators (the ones in
5 nose can not, at least as of nose 0.10.4).
5 nose can not, at least as of nose 0.10.4).
6
6
7 Note: our testing package contains testing.util, which does depend on Twisted
7 Note: our testing package contains testing.util, which does depend on Twisted
8 and provides utilities for tests that manage Deferreds. All testing support
8 and provides utilities for tests that manage Deferreds. All testing support
9 tools that only depend on nose, IPython or the standard library should go here
9 tools that only depend on nose, IPython or the standard library should go here
10 instead.
10 instead.
11
11
12
12
13 Authors
13 Authors
14 -------
14 -------
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
16 """
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Copyright (C) 2009 The IPython Development Team
21 # Copyright (C) 2009 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 import os
31 import os
32 import re
32 import re
33 import sys
33 import sys
34
34
35 try:
35 try:
36 # These tools are used by parts of the runtime, so we make the nose
36 # These tools are used by parts of the runtime, so we make the nose
37 # dependency optional at this point. Nose is a hard dependency to run the
37 # dependency optional at this point. Nose is a hard dependency to run the
38 # test suite, but NOT to use ipython itself.
38 # test suite, but NOT to use ipython itself.
39 import nose.tools as nt
39 import nose.tools as nt
40 has_nose = True
40 has_nose = True
41 except ImportError:
41 except ImportError:
42 has_nose = False
42 has_nose = False
43
43
44 from IPython.config.loader import Config
44 from IPython.config.loader import Config
45 from IPython.utils.process import find_cmd, getoutputerror
45 from IPython.utils.process import find_cmd, getoutputerror
46 from IPython.utils.text import list_strings
46 from IPython.utils.text import list_strings
47 from IPython.utils.io import temp_pyfile
47 from IPython.utils.io import temp_pyfile
48
48
49 from . import decorators as dec
49 from . import decorators as dec
50 from . import skipdoctest
50 from . import skipdoctest
51
51
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53 # Globals
53 # Globals
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55
55
56 # Make a bunch of nose.tools assert wrappers that can be used in test
56 # Make a bunch of nose.tools assert wrappers that can be used in test
57 # generators. This will expose an assert* function for each one in nose.tools.
57 # generators. This will expose an assert* function for each one in nose.tools.
58
58
59 _tpl = """
59 _tpl = """
60 def %(name)s(*a,**kw):
60 def %(name)s(*a,**kw):
61 return nt.%(name)s(*a,**kw)
61 return nt.%(name)s(*a,**kw)
62 """
62 """
63
63
64 if has_nose:
64 if has_nose:
65 for _x in [a for a in dir(nt) if a.startswith('assert')]:
65 for _x in [a for a in dir(nt) if a.startswith('assert')]:
66 exec _tpl % dict(name=_x)
66 exec _tpl % dict(name=_x)
67
67
68 #-----------------------------------------------------------------------------
68 #-----------------------------------------------------------------------------
69 # Functions and classes
69 # Functions and classes
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71
71
72 # The docstring for full_path doctests differently on win32 (different path
72 # The docstring for full_path doctests differently on win32 (different path
73 # separator) so just skip the doctest there. The example remains informative.
73 # separator) so just skip the doctest there. The example remains informative.
74 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
74 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
75
75
76 @doctest_deco
76 @doctest_deco
77 def full_path(startPath,files):
77 def full_path(startPath,files):
78 """Make full paths for all the listed files, based on startPath.
78 """Make full paths for all the listed files, based on startPath.
79
79
80 Only the base part of startPath is kept, since this routine is typically
80 Only the base part of startPath is kept, since this routine is typically
81 used with a script's __file__ variable as startPath. The base of startPath
81 used with a script's __file__ variable as startPath. The base of startPath
82 is then prepended to all the listed files, forming the output list.
82 is then prepended to all the listed files, forming the output list.
83
83
84 Parameters
84 Parameters
85 ----------
85 ----------
86 startPath : string
86 startPath : string
87 Initial path to use as the base for the results. This path is split
87 Initial path to use as the base for the results. This path is split
88 using os.path.split() and only its first component is kept.
88 using os.path.split() and only its first component is kept.
89
89
90 files : string or list
90 files : string or list
91 One or more files.
91 One or more files.
92
92
93 Examples
93 Examples
94 --------
94 --------
95
95
96 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
96 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
97 ['/foo/a.txt', '/foo/b.txt']
97 ['/foo/a.txt', '/foo/b.txt']
98
98
99 >>> full_path('/foo',['a.txt','b.txt'])
99 >>> full_path('/foo',['a.txt','b.txt'])
100 ['/a.txt', '/b.txt']
100 ['/a.txt', '/b.txt']
101
101
102 If a single file is given, the output is still a list:
102 If a single file is given, the output is still a list:
103 >>> full_path('/foo','a.txt')
103 >>> full_path('/foo','a.txt')
104 ['/a.txt']
104 ['/a.txt']
105 """
105 """
106
106
107 files = list_strings(files)
107 files = list_strings(files)
108 base = os.path.split(startPath)[0]
108 base = os.path.split(startPath)[0]
109 return [ os.path.join(base,f) for f in files ]
109 return [ os.path.join(base,f) for f in files ]
110
110
111
111
112 def parse_test_output(txt):
112 def parse_test_output(txt):
113 """Parse the output of a test run and return errors, failures.
113 """Parse the output of a test run and return errors, failures.
114
114
115 Parameters
115 Parameters
116 ----------
116 ----------
117 txt : str
117 txt : str
118 Text output of a test run, assumed to contain a line of one of the
118 Text output of a test run, assumed to contain a line of one of the
119 following forms::
119 following forms::
120 'FAILED (errors=1)'
120 'FAILED (errors=1)'
121 'FAILED (failures=1)'
121 'FAILED (failures=1)'
122 'FAILED (errors=1, failures=1)'
122 'FAILED (errors=1, failures=1)'
123
123
124 Returns
124 Returns
125 -------
125 -------
126 nerr, nfail: number of errors and failures.
126 nerr, nfail: number of errors and failures.
127 """
127 """
128
128
129 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
129 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
130 if err_m:
130 if err_m:
131 nerr = int(err_m.group(1))
131 nerr = int(err_m.group(1))
132 nfail = 0
132 nfail = 0
133 return nerr, nfail
133 return nerr, nfail
134
134
135 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
135 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
136 if fail_m:
136 if fail_m:
137 nerr = 0
137 nerr = 0
138 nfail = int(fail_m.group(1))
138 nfail = int(fail_m.group(1))
139 return nerr, nfail
139 return nerr, nfail
140
140
141 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
141 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
142 re.MULTILINE)
142 re.MULTILINE)
143 if both_m:
143 if both_m:
144 nerr = int(both_m.group(1))
144 nerr = int(both_m.group(1))
145 nfail = int(both_m.group(2))
145 nfail = int(both_m.group(2))
146 return nerr, nfail
146 return nerr, nfail
147
147
148 # If the input didn't match any of these forms, assume no error/failures
148 # If the input didn't match any of these forms, assume no error/failures
149 return 0, 0
149 return 0, 0
150
150
151
151
152 # So nose doesn't think this is a test
152 # So nose doesn't think this is a test
153 parse_test_output.__test__ = False
153 parse_test_output.__test__ = False
154
154
155
155
156 def default_argv():
156 def default_argv():
157 """Return a valid default argv for creating testing instances of ipython"""
157 """Return a valid default argv for creating testing instances of ipython"""
158
158
159 return ['--quick', # so no config file is loaded
159 return ['--quick', # so no config file is loaded
160 # Other defaults to minimize side effects on stdout
160 # Other defaults to minimize side effects on stdout
161 'colors=NoColor', '--no-term-title','--no-banner',
161 'colors=NoColor', '--no-term-title','--no-banner',
162 'autocall=0']
162 'autocall=0']
163
163
164
164
165 def default_config():
165 def default_config():
166 """Return a config object with good defaults for testing."""
166 """Return a config object with good defaults for testing."""
167 config = Config()
167 config = Config()
168 config.TerminalInteractiveShell.colors = 'NoColor'
168 config.TerminalInteractiveShell.colors = 'NoColor'
169 config.TerminalTerminalInteractiveShell.term_title = False,
169 config.TerminalTerminalInteractiveShell.term_title = False,
170 config.TerminalInteractiveShell.autocall = 0
170 config.TerminalInteractiveShell.autocall = 0
171 config.HistoryManager.hist_file = u'test_hist.sqlite'
171 config.HistoryManager.hist_file = u'test_hist.sqlite'
172 config.HistoryManager.db_cache_size = 10000
172 config.HistoryManager.db_cache_size = 10000
173 return config
173 return config
174
174
175
175
176 def ipexec(fname, options=None):
176 def ipexec(fname, options=None):
177 """Utility to call 'ipython filename'.
177 """Utility to call 'ipython filename'.
178
178
179 Starts IPython witha minimal and safe configuration to make startup as fast
179 Starts IPython witha minimal and safe configuration to make startup as fast
180 as possible.
180 as possible.
181
181
182 Note that this starts IPython in a subprocess!
182 Note that this starts IPython in a subprocess!
183
183
184 Parameters
184 Parameters
185 ----------
185 ----------
186 fname : str
186 fname : str
187 Name of file to be executed (should have .py or .ipy extension).
187 Name of file to be executed (should have .py or .ipy extension).
188
188
189 options : optional, list
189 options : optional, list
190 Extra command-line flags to be passed to IPython.
190 Extra command-line flags to be passed to IPython.
191
191
192 Returns
192 Returns
193 -------
193 -------
194 (stdout, stderr) of ipython subprocess.
194 (stdout, stderr) of ipython subprocess.
195 """
195 """
196 if options is None: options = []
196 if options is None: options = []
197
197
198 # For these subprocess calls, eliminate all prompt printing so we only see
198 # For these subprocess calls, eliminate all prompt printing so we only see
199 # output from script execution
199 # output from script execution
200 prompt_opts = [ 'InteractiveShell.prompt_in1=""',
200 prompt_opts = [ 'InteractiveShell.prompt_in1=""',
201 'InteractiveShell.prompt_in2=""',
201 'InteractiveShell.prompt_in2=""',
202 'InteractiveShell.prompt_out=""'
202 'InteractiveShell.prompt_out=""'
203 ]
203 ]
204 cmdargs = ' '.join(default_argv() + prompt_opts + options)
204 cmdargs = ' '.join(default_argv() + prompt_opts + options)
205
205
206 _ip = get_ipython()
206 _ip = get_ipython()
207 test_dir = os.path.dirname(__file__)
207 test_dir = os.path.dirname(__file__)
208
208
209 ipython_cmd = find_cmd('ipython')
209 ipython_cmd = find_cmd('ipython')
210 # Absolute path for filename
210 # Absolute path for filename
211 full_fname = os.path.join(test_dir, fname)
211 full_fname = os.path.join(test_dir, fname)
212 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
212 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
213 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
213 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
214 return getoutputerror(full_cmd)
214 return getoutputerror(full_cmd)
215
215
216
216
217 def ipexec_validate(fname, expected_out, expected_err='',
217 def ipexec_validate(fname, expected_out, expected_err='',
218 options=None):
218 options=None):
219 """Utility to call 'ipython filename' and validate output/error.
219 """Utility to call 'ipython filename' and validate output/error.
220
220
221 This function raises an AssertionError if the validation fails.
221 This function raises an AssertionError if the validation fails.
222
222
223 Note that this starts IPython in a subprocess!
223 Note that this starts IPython in a subprocess!
224
224
225 Parameters
225 Parameters
226 ----------
226 ----------
227 fname : str
227 fname : str
228 Name of the file to be executed (should have .py or .ipy extension).
228 Name of the file to be executed (should have .py or .ipy extension).
229
229
230 expected_out : str
230 expected_out : str
231 Expected stdout of the process.
231 Expected stdout of the process.
232
232
233 expected_err : optional, str
233 expected_err : optional, str
234 Expected stderr of the process.
234 Expected stderr of the process.
235
235
236 options : optional, list
236 options : optional, list
237 Extra command-line flags to be passed to IPython.
237 Extra command-line flags to be passed to IPython.
238
238
239 Returns
239 Returns
240 -------
240 -------
241 None
241 None
242 """
242 """
243
243
244 import nose.tools as nt
244 import nose.tools as nt
245
245
246 out, err = ipexec(fname)
246 out, err = ipexec(fname)
247 #print 'OUT', out # dbg
247 #print 'OUT', out # dbg
248 #print 'ERR', err # dbg
248 #print 'ERR', err # dbg
249 # If there are any errors, we must check those befor stdout, as they may be
249 # If there are any errors, we must check those befor stdout, as they may be
250 # more informative than simply having an empty stdout.
250 # more informative than simply having an empty stdout.
251 if err:
251 if err:
252 if expected_err:
252 if expected_err:
253 nt.assert_equals(err.strip(), expected_err.strip())
253 nt.assert_equals(err.strip(), expected_err.strip())
254 else:
254 else:
255 raise ValueError('Running file %r produced error: %r' %
255 raise ValueError('Running file %r produced error: %r' %
256 (fname, err))
256 (fname, err))
257 # If no errors or output on stderr was expected, match stdout
257 # If no errors or output on stderr was expected, match stdout
258 nt.assert_equals(out.strip(), expected_out.strip())
258 nt.assert_equals(out.strip(), expected_out.strip())
259
259
260
260
261 class TempFileMixin(object):
261 class TempFileMixin(object):
262 """Utility class to create temporary Python/IPython files.
262 """Utility class to create temporary Python/IPython files.
263
263
264 Meant as a mixin class for test cases."""
264 Meant as a mixin class for test cases."""
265
265
266 def mktmp(self, src, ext='.py'):
266 def mktmp(self, src, ext='.py'):
267 """Make a valid python temp file."""
267 """Make a valid python temp file."""
268 fname, f = temp_pyfile(src, ext)
268 fname, f = temp_pyfile(src, ext)
269 self.tmpfile = f
269 self.tmpfile = f
270 self.fname = fname
270 self.fname = fname
271
271
272 def tearDown(self):
272 def tearDown(self):
273 if hasattr(self, 'tmpfile'):
273 if hasattr(self, 'tmpfile'):
274 # If the tmpfile wasn't made because of skipped tests, like in
274 # If the tmpfile wasn't made because of skipped tests, like in
275 # win32, there's nothing to cleanup.
275 # win32, there's nothing to cleanup.
276 self.tmpfile.close()
276 self.tmpfile.close()
277 try:
277 try:
278 os.unlink(self.fname)
278 os.unlink(self.fname)
279 except:
279 except:
280 # On Windows, even though we close the file, we still can't
280 # On Windows, even though we close the file, we still can't
281 # delete it. I have no clue why
281 # delete it. I have no clue why
282 pass
282 pass
283
283
284 pair_fail_msg = ("Testing function {0}\n\n"
285 "In:\n"
286 " {1!r}\n"
287 "Expected:\n"
288 " {2!r}\n"
289 "Got:\n"
290 " {3!r}\n")
291 def check_pairs(func, pairs):
292 """Utility function for the common case of checking a function with a
293 sequence of input/output pairs.
294
295 Parameters
296 ----------
297 func : callable
298 The function to be tested. Should accept a single argument.
299 pairs : iterable
300 A list of (input, expected_output) tuples.
301
302 Returns
303 -------
304 None. Raises an AssertionError if any output does not match the expected
305 value.
306 """
307 for inp, expected in pairs:
308 out = func(inp)
309 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
General Comments 0
You need to be logged in to leave comments. Login now