##// END OF EJS Templates
more general fix for #662...
MinRK -
Show More
@@ -1,402 +1,398 b''
1 """Generic testing tools that do NOT depend on Twisted.
1 """Generic testing tools that do NOT depend on Twisted.
2
2
3 In particular, this module exposes a set of top-level assert* functions that
3 In particular, this module exposes a set of top-level assert* functions that
4 can be used in place of nose.tools.assert* in method generators (the ones in
4 can be used in place of nose.tools.assert* in method generators (the ones in
5 nose can not, at least as of nose 0.10.4).
5 nose can not, at least as of nose 0.10.4).
6
6
7 Note: our testing package contains testing.util, which does depend on Twisted
7 Note: our testing package contains testing.util, which does depend on Twisted
8 and provides utilities for tests that manage Deferreds. All testing support
8 and provides utilities for tests that manage Deferreds. All testing support
9 tools that only depend on nose, IPython or the standard library should go here
9 tools that only depend on nose, IPython or the standard library should go here
10 instead.
10 instead.
11
11
12
12
13 Authors
13 Authors
14 -------
14 -------
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
16 """
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Copyright (C) 2009-2011 The IPython Development Team
21 # Copyright (C) 2009-2011 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 import os
31 import os
32 import re
32 import re
33 import sys
33 import sys
34 import tempfile
34 import tempfile
35
35
36 from contextlib import contextmanager
36 from contextlib import contextmanager
37 from io import StringIO
37 from io import StringIO
38
38
39 try:
39 try:
40 # These tools are used by parts of the runtime, so we make the nose
40 # These tools are used by parts of the runtime, so we make the nose
41 # dependency optional at this point. Nose is a hard dependency to run the
41 # dependency optional at this point. Nose is a hard dependency to run the
42 # test suite, but NOT to use ipython itself.
42 # test suite, but NOT to use ipython itself.
43 import nose.tools as nt
43 import nose.tools as nt
44 has_nose = True
44 has_nose = True
45 except ImportError:
45 except ImportError:
46 has_nose = False
46 has_nose = False
47
47
48 from IPython.config.loader import Config
48 from IPython.config.loader import Config
49 from IPython.utils.process import find_cmd, getoutputerror
49 from IPython.utils.process import find_cmd, getoutputerror
50 from IPython.utils.text import list_strings, getdefaultencoding
50 from IPython.utils.text import list_strings, getdefaultencoding
51 from IPython.utils.io import temp_pyfile, Tee
51 from IPython.utils.io import temp_pyfile, Tee
52 from IPython.utils import py3compat
52 from IPython.utils import py3compat
53
53
54 from . import decorators as dec
54 from . import decorators as dec
55 from . import skipdoctest
55 from . import skipdoctest
56
56
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58 # Globals
58 # Globals
59 #-----------------------------------------------------------------------------
59 #-----------------------------------------------------------------------------
60
60
61 # Make a bunch of nose.tools assert wrappers that can be used in test
61 # Make a bunch of nose.tools assert wrappers that can be used in test
62 # generators. This will expose an assert* function for each one in nose.tools.
62 # generators. This will expose an assert* function for each one in nose.tools.
63
63
64 _tpl = """
64 _tpl = """
65 def %(name)s(*a,**kw):
65 def %(name)s(*a,**kw):
66 return nt.%(name)s(*a,**kw)
66 return nt.%(name)s(*a,**kw)
67 """
67 """
68
68
69 if has_nose:
69 if has_nose:
70 for _x in [a for a in dir(nt) if a.startswith('assert')]:
70 for _x in [a for a in dir(nt) if a.startswith('assert')]:
71 exec _tpl % dict(name=_x)
71 exec _tpl % dict(name=_x)
72
72
73 #-----------------------------------------------------------------------------
73 #-----------------------------------------------------------------------------
74 # Functions and classes
74 # Functions and classes
75 #-----------------------------------------------------------------------------
75 #-----------------------------------------------------------------------------
76
76
77 # The docstring for full_path doctests differently on win32 (different path
77 # The docstring for full_path doctests differently on win32 (different path
78 # separator) so just skip the doctest there. The example remains informative.
78 # separator) so just skip the doctest there. The example remains informative.
79 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
79 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
80
80
81 @doctest_deco
81 @doctest_deco
82 def full_path(startPath,files):
82 def full_path(startPath,files):
83 """Make full paths for all the listed files, based on startPath.
83 """Make full paths for all the listed files, based on startPath.
84
84
85 Only the base part of startPath is kept, since this routine is typically
85 Only the base part of startPath is kept, since this routine is typically
86 used with a script's __file__ variable as startPath. The base of startPath
86 used with a script's __file__ variable as startPath. The base of startPath
87 is then prepended to all the listed files, forming the output list.
87 is then prepended to all the listed files, forming the output list.
88
88
89 Parameters
89 Parameters
90 ----------
90 ----------
91 startPath : string
91 startPath : string
92 Initial path to use as the base for the results. This path is split
92 Initial path to use as the base for the results. This path is split
93 using os.path.split() and only its first component is kept.
93 using os.path.split() and only its first component is kept.
94
94
95 files : string or list
95 files : string or list
96 One or more files.
96 One or more files.
97
97
98 Examples
98 Examples
99 --------
99 --------
100
100
101 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
101 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
102 ['/foo/a.txt', '/foo/b.txt']
102 ['/foo/a.txt', '/foo/b.txt']
103
103
104 >>> full_path('/foo',['a.txt','b.txt'])
104 >>> full_path('/foo',['a.txt','b.txt'])
105 ['/a.txt', '/b.txt']
105 ['/a.txt', '/b.txt']
106
106
107 If a single file is given, the output is still a list:
107 If a single file is given, the output is still a list:
108 >>> full_path('/foo','a.txt')
108 >>> full_path('/foo','a.txt')
109 ['/a.txt']
109 ['/a.txt']
110 """
110 """
111
111
112 files = list_strings(files)
112 files = list_strings(files)
113 base = os.path.split(startPath)[0]
113 base = os.path.split(startPath)[0]
114 return [ os.path.join(base,f) for f in files ]
114 return [ os.path.join(base,f) for f in files ]
115
115
116
116
117 def parse_test_output(txt):
117 def parse_test_output(txt):
118 """Parse the output of a test run and return errors, failures.
118 """Parse the output of a test run and return errors, failures.
119
119
120 Parameters
120 Parameters
121 ----------
121 ----------
122 txt : str
122 txt : str
123 Text output of a test run, assumed to contain a line of one of the
123 Text output of a test run, assumed to contain a line of one of the
124 following forms::
124 following forms::
125 'FAILED (errors=1)'
125 'FAILED (errors=1)'
126 'FAILED (failures=1)'
126 'FAILED (failures=1)'
127 'FAILED (errors=1, failures=1)'
127 'FAILED (errors=1, failures=1)'
128
128
129 Returns
129 Returns
130 -------
130 -------
131 nerr, nfail: number of errors and failures.
131 nerr, nfail: number of errors and failures.
132 """
132 """
133
133
134 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
134 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
135 if err_m:
135 if err_m:
136 nerr = int(err_m.group(1))
136 nerr = int(err_m.group(1))
137 nfail = 0
137 nfail = 0
138 return nerr, nfail
138 return nerr, nfail
139
139
140 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
140 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
141 if fail_m:
141 if fail_m:
142 nerr = 0
142 nerr = 0
143 nfail = int(fail_m.group(1))
143 nfail = int(fail_m.group(1))
144 return nerr, nfail
144 return nerr, nfail
145
145
146 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
146 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
147 re.MULTILINE)
147 re.MULTILINE)
148 if both_m:
148 if both_m:
149 nerr = int(both_m.group(1))
149 nerr = int(both_m.group(1))
150 nfail = int(both_m.group(2))
150 nfail = int(both_m.group(2))
151 return nerr, nfail
151 return nerr, nfail
152
152
153 # If the input didn't match any of these forms, assume no error/failures
153 # If the input didn't match any of these forms, assume no error/failures
154 return 0, 0
154 return 0, 0
155
155
156
156
157 # So nose doesn't think this is a test
157 # So nose doesn't think this is a test
158 parse_test_output.__test__ = False
158 parse_test_output.__test__ = False
159
159
160
160
161 def default_argv():
161 def default_argv():
162 """Return a valid default argv for creating testing instances of ipython"""
162 """Return a valid default argv for creating testing instances of ipython"""
163
163
164 return ['--quick', # so no config file is loaded
164 return ['--quick', # so no config file is loaded
165 # Other defaults to minimize side effects on stdout
165 # Other defaults to minimize side effects on stdout
166 '--colors=NoColor', '--no-term-title','--no-banner',
166 '--colors=NoColor', '--no-term-title','--no-banner',
167 '--autocall=0']
167 '--autocall=0']
168
168
169
169
170 def default_config():
170 def default_config():
171 """Return a config object with good defaults for testing."""
171 """Return a config object with good defaults for testing."""
172 config = Config()
172 config = Config()
173 config.TerminalInteractiveShell.colors = 'NoColor'
173 config.TerminalInteractiveShell.colors = 'NoColor'
174 config.TerminalTerminalInteractiveShell.term_title = False,
174 config.TerminalTerminalInteractiveShell.term_title = False,
175 config.TerminalInteractiveShell.autocall = 0
175 config.TerminalInteractiveShell.autocall = 0
176 config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
176 config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
177 config.HistoryManager.db_cache_size = 10000
177 config.HistoryManager.db_cache_size = 10000
178 return config
178 return config
179
179
180
180
181 def ipexec(fname, options=None):
181 def ipexec(fname, options=None):
182 """Utility to call 'ipython filename'.
182 """Utility to call 'ipython filename'.
183
183
184 Starts IPython witha minimal and safe configuration to make startup as fast
184 Starts IPython witha minimal and safe configuration to make startup as fast
185 as possible.
185 as possible.
186
186
187 Note that this starts IPython in a subprocess!
187 Note that this starts IPython in a subprocess!
188
188
189 Parameters
189 Parameters
190 ----------
190 ----------
191 fname : str
191 fname : str
192 Name of file to be executed (should have .py or .ipy extension).
192 Name of file to be executed (should have .py or .ipy extension).
193
193
194 options : optional, list
194 options : optional, list
195 Extra command-line flags to be passed to IPython.
195 Extra command-line flags to be passed to IPython.
196
196
197 Returns
197 Returns
198 -------
198 -------
199 (stdout, stderr) of ipython subprocess.
199 (stdout, stderr) of ipython subprocess.
200 """
200 """
201 if options is None: options = []
201 if options is None: options = []
202
202
203 # For these subprocess calls, eliminate all prompt printing so we only see
203 # For these subprocess calls, eliminate all prompt printing so we only see
204 # output from script execution
204 # output from script execution
205 prompt_opts = [ '--PromptManager.in_template=""',
205 prompt_opts = [ '--PromptManager.in_template=""',
206 '--PromptManager.in2_template=""',
206 '--PromptManager.in2_template=""',
207 '--PromptManager.out_template=""'
207 '--PromptManager.out_template=""'
208 ]
208 ]
209 cmdargs = ' '.join(default_argv() + prompt_opts + options)
209 cmdargs = ' '.join(default_argv() + prompt_opts + options)
210
210
211 _ip = get_ipython()
211 _ip = get_ipython()
212 test_dir = os.path.dirname(__file__)
212 test_dir = os.path.dirname(__file__)
213
213
214 ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
214 ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
215 # Absolute path for filename
215 # Absolute path for filename
216 full_fname = os.path.join(test_dir, fname)
216 full_fname = os.path.join(test_dir, fname)
217 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
217 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
218 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
218 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
219 out = getoutputerror(full_cmd)
219 out = getoutputerror(full_cmd)
220 # `import readline` causes 'ESC[?1034h' to be the first output sometimes,
220 # `import readline` causes 'ESC[?1034h' to be output sometimes,
221 # so strip that off the front of the first line if it is found
221 # so strip that out before doing comparisons
222 if out:
222 if out:
223 first = out[0]
223 out,err = out
224 m = re.match(r'\x1b\[[^h]+h', first)
224 out = re.sub(r'\x1b\[[^h]+h', '', out)
225 if m:
225 out = out,err
226 # strip initial readline escape
227 out = list(out)
228 out[0] = first[len(m.group()):]
229 out = tuple(out)
230 return out
226 return out
231
227
232
228
233 def ipexec_validate(fname, expected_out, expected_err='',
229 def ipexec_validate(fname, expected_out, expected_err='',
234 options=None):
230 options=None):
235 """Utility to call 'ipython filename' and validate output/error.
231 """Utility to call 'ipython filename' and validate output/error.
236
232
237 This function raises an AssertionError if the validation fails.
233 This function raises an AssertionError if the validation fails.
238
234
239 Note that this starts IPython in a subprocess!
235 Note that this starts IPython in a subprocess!
240
236
241 Parameters
237 Parameters
242 ----------
238 ----------
243 fname : str
239 fname : str
244 Name of the file to be executed (should have .py or .ipy extension).
240 Name of the file to be executed (should have .py or .ipy extension).
245
241
246 expected_out : str
242 expected_out : str
247 Expected stdout of the process.
243 Expected stdout of the process.
248
244
249 expected_err : optional, str
245 expected_err : optional, str
250 Expected stderr of the process.
246 Expected stderr of the process.
251
247
252 options : optional, list
248 options : optional, list
253 Extra command-line flags to be passed to IPython.
249 Extra command-line flags to be passed to IPython.
254
250
255 Returns
251 Returns
256 -------
252 -------
257 None
253 None
258 """
254 """
259
255
260 import nose.tools as nt
256 import nose.tools as nt
261
257
262 out, err = ipexec(fname, options)
258 out, err = ipexec(fname, options)
263 #print 'OUT', out # dbg
259 #print 'OUT', out # dbg
264 #print 'ERR', err # dbg
260 #print 'ERR', err # dbg
265 # If there are any errors, we must check those befor stdout, as they may be
261 # If there are any errors, we must check those befor stdout, as they may be
266 # more informative than simply having an empty stdout.
262 # more informative than simply having an empty stdout.
267 if err:
263 if err:
268 if expected_err:
264 if expected_err:
269 nt.assert_equals(err.strip(), expected_err.strip())
265 nt.assert_equals(err.strip(), expected_err.strip())
270 else:
266 else:
271 raise ValueError('Running file %r produced error: %r' %
267 raise ValueError('Running file %r produced error: %r' %
272 (fname, err))
268 (fname, err))
273 # If no errors or output on stderr was expected, match stdout
269 # If no errors or output on stderr was expected, match stdout
274 nt.assert_equals(out.strip(), expected_out.strip())
270 nt.assert_equals(out.strip(), expected_out.strip())
275
271
276
272
277 class TempFileMixin(object):
273 class TempFileMixin(object):
278 """Utility class to create temporary Python/IPython files.
274 """Utility class to create temporary Python/IPython files.
279
275
280 Meant as a mixin class for test cases."""
276 Meant as a mixin class for test cases."""
281
277
282 def mktmp(self, src, ext='.py'):
278 def mktmp(self, src, ext='.py'):
283 """Make a valid python temp file."""
279 """Make a valid python temp file."""
284 fname, f = temp_pyfile(src, ext)
280 fname, f = temp_pyfile(src, ext)
285 self.tmpfile = f
281 self.tmpfile = f
286 self.fname = fname
282 self.fname = fname
287
283
288 def tearDown(self):
284 def tearDown(self):
289 if hasattr(self, 'tmpfile'):
285 if hasattr(self, 'tmpfile'):
290 # If the tmpfile wasn't made because of skipped tests, like in
286 # If the tmpfile wasn't made because of skipped tests, like in
291 # win32, there's nothing to cleanup.
287 # win32, there's nothing to cleanup.
292 self.tmpfile.close()
288 self.tmpfile.close()
293 try:
289 try:
294 os.unlink(self.fname)
290 os.unlink(self.fname)
295 except:
291 except:
296 # On Windows, even though we close the file, we still can't
292 # On Windows, even though we close the file, we still can't
297 # delete it. I have no clue why
293 # delete it. I have no clue why
298 pass
294 pass
299
295
300 pair_fail_msg = ("Testing {0}\n\n"
296 pair_fail_msg = ("Testing {0}\n\n"
301 "In:\n"
297 "In:\n"
302 " {1!r}\n"
298 " {1!r}\n"
303 "Expected:\n"
299 "Expected:\n"
304 " {2!r}\n"
300 " {2!r}\n"
305 "Got:\n"
301 "Got:\n"
306 " {3!r}\n")
302 " {3!r}\n")
307 def check_pairs(func, pairs):
303 def check_pairs(func, pairs):
308 """Utility function for the common case of checking a function with a
304 """Utility function for the common case of checking a function with a
309 sequence of input/output pairs.
305 sequence of input/output pairs.
310
306
311 Parameters
307 Parameters
312 ----------
308 ----------
313 func : callable
309 func : callable
314 The function to be tested. Should accept a single argument.
310 The function to be tested. Should accept a single argument.
315 pairs : iterable
311 pairs : iterable
316 A list of (input, expected_output) tuples.
312 A list of (input, expected_output) tuples.
317
313
318 Returns
314 Returns
319 -------
315 -------
320 None. Raises an AssertionError if any output does not match the expected
316 None. Raises an AssertionError if any output does not match the expected
321 value.
317 value.
322 """
318 """
323 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
319 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
324 for inp, expected in pairs:
320 for inp, expected in pairs:
325 out = func(inp)
321 out = func(inp)
326 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
322 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
327
323
328
324
329 if py3compat.PY3:
325 if py3compat.PY3:
330 MyStringIO = StringIO
326 MyStringIO = StringIO
331 else:
327 else:
332 # In Python 2, stdout/stderr can have either bytes or unicode written to them,
328 # In Python 2, stdout/stderr can have either bytes or unicode written to them,
333 # so we need a class that can handle both.
329 # so we need a class that can handle both.
334 class MyStringIO(StringIO):
330 class MyStringIO(StringIO):
335 def write(self, s):
331 def write(self, s):
336 s = py3compat.cast_unicode(s, encoding=getdefaultencoding())
332 s = py3compat.cast_unicode(s, encoding=getdefaultencoding())
337 super(MyStringIO, self).write(s)
333 super(MyStringIO, self).write(s)
338
334
339 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
335 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
340 {2!r}"""
336 {2!r}"""
341
337
342 class AssertPrints(object):
338 class AssertPrints(object):
343 """Context manager for testing that code prints certain text.
339 """Context manager for testing that code prints certain text.
344
340
345 Examples
341 Examples
346 --------
342 --------
347 >>> with AssertPrints("abc", suppress=False):
343 >>> with AssertPrints("abc", suppress=False):
348 ... print "abcd"
344 ... print "abcd"
349 ... print "def"
345 ... print "def"
350 ...
346 ...
351 abcd
347 abcd
352 def
348 def
353 """
349 """
354 def __init__(self, s, channel='stdout', suppress=True):
350 def __init__(self, s, channel='stdout', suppress=True):
355 self.s = s
351 self.s = s
356 self.channel = channel
352 self.channel = channel
357 self.suppress = suppress
353 self.suppress = suppress
358
354
359 def __enter__(self):
355 def __enter__(self):
360 self.orig_stream = getattr(sys, self.channel)
356 self.orig_stream = getattr(sys, self.channel)
361 self.buffer = MyStringIO()
357 self.buffer = MyStringIO()
362 self.tee = Tee(self.buffer, channel=self.channel)
358 self.tee = Tee(self.buffer, channel=self.channel)
363 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
359 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
364
360
365 def __exit__(self, etype, value, traceback):
361 def __exit__(self, etype, value, traceback):
366 self.tee.flush()
362 self.tee.flush()
367 setattr(sys, self.channel, self.orig_stream)
363 setattr(sys, self.channel, self.orig_stream)
368 printed = self.buffer.getvalue()
364 printed = self.buffer.getvalue()
369 assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
365 assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
370 return False
366 return False
371
367
372 class AssertNotPrints(AssertPrints):
368 class AssertNotPrints(AssertPrints):
373 """Context manager for checking that certain output *isn't* produced.
369 """Context manager for checking that certain output *isn't* produced.
374
370
375 Counterpart of AssertPrints"""
371 Counterpart of AssertPrints"""
376 def __exit__(self, etype, value, traceback):
372 def __exit__(self, etype, value, traceback):
377 self.tee.flush()
373 self.tee.flush()
378 setattr(sys, self.channel, self.orig_stream)
374 setattr(sys, self.channel, self.orig_stream)
379 printed = self.buffer.getvalue()
375 printed = self.buffer.getvalue()
380 assert self.s not in printed, notprinted_msg.format(self.s, self.channel, printed)
376 assert self.s not in printed, notprinted_msg.format(self.s, self.channel, printed)
381 return False
377 return False
382
378
383 @contextmanager
379 @contextmanager
384 def mute_warn():
380 def mute_warn():
385 from IPython.utils import warn
381 from IPython.utils import warn
386 save_warn = warn.warn
382 save_warn = warn.warn
387 warn.warn = lambda *a, **kw: None
383 warn.warn = lambda *a, **kw: None
388 try:
384 try:
389 yield
385 yield
390 finally:
386 finally:
391 warn.warn = save_warn
387 warn.warn = save_warn
392
388
393 @contextmanager
389 @contextmanager
394 def make_tempfile(name):
390 def make_tempfile(name):
395 """ Create an empty, named, temporary file for the duration of the context.
391 """ Create an empty, named, temporary file for the duration of the context.
396 """
392 """
397 f = open(name, 'w')
393 f = open(name, 'w')
398 f.close()
394 f.close()
399 try:
395 try:
400 yield
396 yield
401 finally:
397 finally:
402 os.unlink(name)
398 os.unlink(name)
General Comments 0
You need to be logged in to leave comments. Login now