##// END OF EJS Templates
getoutput wants a string, not a command list
MinRK -
r11353:2792933e
parent child
Show More
@@ -1,392 +1,393
1 """Generic testing tools.
1 """Generic testing tools.
2
2
3 Authors
3 Authors
4 -------
4 -------
5 - Fernando Perez <Fernando.Perez@berkeley.edu>
5 - Fernando Perez <Fernando.Perez@berkeley.edu>
6 """
6 """
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2009-2011 The IPython Development Team
11 # Copyright (C) 2009-2011 The IPython Development Team
12 #
12 #
13 # Distributed under the terms of the BSD License. The full license is in
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
14 # the file COPYING, distributed as part of this software.
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 import os
21 import os
22 import pipes
22 import re
23 import re
23 import sys
24 import sys
24 import tempfile
25 import tempfile
25
26
26 from contextlib import contextmanager
27 from contextlib import contextmanager
27 from io import StringIO
28 from io import StringIO
28
29
29 try:
30 try:
30 # These tools are used by parts of the runtime, so we make the nose
31 # These tools are used by parts of the runtime, so we make the nose
31 # dependency optional at this point. Nose is a hard dependency to run the
32 # dependency optional at this point. Nose is a hard dependency to run the
32 # test suite, but NOT to use ipython itself.
33 # test suite, but NOT to use ipython itself.
33 import nose.tools as nt
34 import nose.tools as nt
34 has_nose = True
35 has_nose = True
35 except ImportError:
36 except ImportError:
36 has_nose = False
37 has_nose = False
37
38
38 from IPython.config.loader import Config
39 from IPython.config.loader import Config
39 from IPython.utils.process import find_cmd, getoutputerror
40 from IPython.utils.process import find_cmd, getoutputerror
40 from IPython.utils.text import list_strings
41 from IPython.utils.text import list_strings
41 from IPython.utils.io import temp_pyfile, Tee
42 from IPython.utils.io import temp_pyfile, Tee
42 from IPython.utils import py3compat
43 from IPython.utils import py3compat
43 from IPython.utils.encoding import DEFAULT_ENCODING
44 from IPython.utils.encoding import DEFAULT_ENCODING
44
45
45 from . import decorators as dec
46 from . import decorators as dec
46 from . import skipdoctest
47 from . import skipdoctest
47
48
48 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
49 # Functions and classes
50 # Functions and classes
50 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
51
52
52 # The docstring for full_path doctests differently on win32 (different path
53 # The docstring for full_path doctests differently on win32 (different path
53 # separator) so just skip the doctest there. The example remains informative.
54 # separator) so just skip the doctest there. The example remains informative.
54 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
55 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
55
56
56 @doctest_deco
57 @doctest_deco
57 def full_path(startPath,files):
58 def full_path(startPath,files):
58 """Make full paths for all the listed files, based on startPath.
59 """Make full paths for all the listed files, based on startPath.
59
60
60 Only the base part of startPath is kept, since this routine is typically
61 Only the base part of startPath is kept, since this routine is typically
61 used with a script's __file__ variable as startPath. The base of startPath
62 used with a script's __file__ variable as startPath. The base of startPath
62 is then prepended to all the listed files, forming the output list.
63 is then prepended to all the listed files, forming the output list.
63
64
64 Parameters
65 Parameters
65 ----------
66 ----------
66 startPath : string
67 startPath : string
67 Initial path to use as the base for the results. This path is split
68 Initial path to use as the base for the results. This path is split
68 using os.path.split() and only its first component is kept.
69 using os.path.split() and only its first component is kept.
69
70
70 files : string or list
71 files : string or list
71 One or more files.
72 One or more files.
72
73
73 Examples
74 Examples
74 --------
75 --------
75
76
76 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
77 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
77 ['/foo/a.txt', '/foo/b.txt']
78 ['/foo/a.txt', '/foo/b.txt']
78
79
79 >>> full_path('/foo',['a.txt','b.txt'])
80 >>> full_path('/foo',['a.txt','b.txt'])
80 ['/a.txt', '/b.txt']
81 ['/a.txt', '/b.txt']
81
82
82 If a single file is given, the output is still a list:
83 If a single file is given, the output is still a list:
83 >>> full_path('/foo','a.txt')
84 >>> full_path('/foo','a.txt')
84 ['/a.txt']
85 ['/a.txt']
85 """
86 """
86
87
87 files = list_strings(files)
88 files = list_strings(files)
88 base = os.path.split(startPath)[0]
89 base = os.path.split(startPath)[0]
89 return [ os.path.join(base,f) for f in files ]
90 return [ os.path.join(base,f) for f in files ]
90
91
91
92
92 def parse_test_output(txt):
93 def parse_test_output(txt):
93 """Parse the output of a test run and return errors, failures.
94 """Parse the output of a test run and return errors, failures.
94
95
95 Parameters
96 Parameters
96 ----------
97 ----------
97 txt : str
98 txt : str
98 Text output of a test run, assumed to contain a line of one of the
99 Text output of a test run, assumed to contain a line of one of the
99 following forms::
100 following forms::
100
101
101 'FAILED (errors=1)'
102 'FAILED (errors=1)'
102 'FAILED (failures=1)'
103 'FAILED (failures=1)'
103 'FAILED (errors=1, failures=1)'
104 'FAILED (errors=1, failures=1)'
104
105
105 Returns
106 Returns
106 -------
107 -------
107 nerr, nfail: number of errors and failures.
108 nerr, nfail: number of errors and failures.
108 """
109 """
109
110
110 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
111 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
111 if err_m:
112 if err_m:
112 nerr = int(err_m.group(1))
113 nerr = int(err_m.group(1))
113 nfail = 0
114 nfail = 0
114 return nerr, nfail
115 return nerr, nfail
115
116
116 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
117 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
117 if fail_m:
118 if fail_m:
118 nerr = 0
119 nerr = 0
119 nfail = int(fail_m.group(1))
120 nfail = int(fail_m.group(1))
120 return nerr, nfail
121 return nerr, nfail
121
122
122 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
123 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
123 re.MULTILINE)
124 re.MULTILINE)
124 if both_m:
125 if both_m:
125 nerr = int(both_m.group(1))
126 nerr = int(both_m.group(1))
126 nfail = int(both_m.group(2))
127 nfail = int(both_m.group(2))
127 return nerr, nfail
128 return nerr, nfail
128
129
129 # If the input didn't match any of these forms, assume no error/failures
130 # If the input didn't match any of these forms, assume no error/failures
130 return 0, 0
131 return 0, 0
131
132
132
133
133 # So nose doesn't think this is a test
134 # So nose doesn't think this is a test
134 parse_test_output.__test__ = False
135 parse_test_output.__test__ = False
135
136
136
137
137 def default_argv():
138 def default_argv():
138 """Return a valid default argv for creating testing instances of ipython"""
139 """Return a valid default argv for creating testing instances of ipython"""
139
140
140 return ['--quick', # so no config file is loaded
141 return ['--quick', # so no config file is loaded
141 # Other defaults to minimize side effects on stdout
142 # Other defaults to minimize side effects on stdout
142 '--colors=NoColor', '--no-term-title','--no-banner',
143 '--colors=NoColor', '--no-term-title','--no-banner',
143 '--autocall=0']
144 '--autocall=0']
144
145
145
146
146 def default_config():
147 def default_config():
147 """Return a config object with good defaults for testing."""
148 """Return a config object with good defaults for testing."""
148 config = Config()
149 config = Config()
149 config.TerminalInteractiveShell.colors = 'NoColor'
150 config.TerminalInteractiveShell.colors = 'NoColor'
150 config.TerminalTerminalInteractiveShell.term_title = False,
151 config.TerminalTerminalInteractiveShell.term_title = False,
151 config.TerminalInteractiveShell.autocall = 0
152 config.TerminalInteractiveShell.autocall = 0
152 config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
153 config.HistoryManager.hist_file = tempfile.mktemp(u'test_hist.sqlite')
153 config.HistoryManager.db_cache_size = 10000
154 config.HistoryManager.db_cache_size = 10000
154 return config
155 return config
155
156
156
157
157 def ipexec(fname, options=None):
158 def ipexec(fname, options=None):
158 """Utility to call 'ipython filename'.
159 """Utility to call 'ipython filename'.
159
160
160 Starts IPython witha minimal and safe configuration to make startup as fast
161 Starts IPython with a minimal and safe configuration to make startup as fast
161 as possible.
162 as possible.
162
163
163 Note that this starts IPython in a subprocess!
164 Note that this starts IPython in a subprocess!
164
165
165 Parameters
166 Parameters
166 ----------
167 ----------
167 fname : str
168 fname : str
168 Name of file to be executed (should have .py or .ipy extension).
169 Name of file to be executed (should have .py or .ipy extension).
169
170
170 options : optional, list
171 options : optional, list
171 Extra command-line flags to be passed to IPython.
172 Extra command-line flags to be passed to IPython.
172
173
173 Returns
174 Returns
174 -------
175 -------
175 (stdout, stderr) of ipython subprocess.
176 (stdout, stderr) of ipython subprocess.
176 """
177 """
177 if options is None: options = []
178 if options is None: options = []
178
179
179 # For these subprocess calls, eliminate all prompt printing so we only see
180 # For these subprocess calls, eliminate all prompt printing so we only see
180 # output from script execution
181 # output from script execution
181 prompt_opts = [ '--PromptManager.in_template=""',
182 prompt_opts = [ '--PromptManager.in_template=""',
182 '--PromptManager.in2_template=""',
183 '--PromptManager.in2_template=""',
183 '--PromptManager.out_template=""'
184 '--PromptManager.out_template=""'
184 ]
185 ]
185 cmdargs = ' '.join(default_argv() + prompt_opts + options)
186 cmdargs = ' '.join(default_argv() + prompt_opts + options)
186
187
187 _ip = get_ipython()
188 _ip = get_ipython()
188 test_dir = os.path.dirname(__file__)
189 test_dir = os.path.dirname(__file__)
189
190
190 ipython_cmd = [sys.executable, '-m', 'IPython']
191 ipython_cmd = pipes.quote(sys.executable) + " -m IPython"
191 # Absolute path for filename
192 # Absolute path for filename
192 full_fname = os.path.join(test_dir, fname)
193 full_fname = os.path.join(test_dir, fname)
193 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
194 full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
194 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
195 #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
195 out, err = getoutputerror(full_cmd)
196 out, err = getoutputerror(full_cmd)
196 # `import readline` causes 'ESC[?1034h' to be output sometimes,
197 # `import readline` causes 'ESC[?1034h' to be output sometimes,
197 # so strip that out before doing comparisons
198 # so strip that out before doing comparisons
198 if out:
199 if out:
199 out = re.sub(r'\x1b\[[^h]+h', '', out)
200 out = re.sub(r'\x1b\[[^h]+h', '', out)
200 return out, err
201 return out, err
201
202
202
203
203 def ipexec_validate(fname, expected_out, expected_err='',
204 def ipexec_validate(fname, expected_out, expected_err='',
204 options=None):
205 options=None):
205 """Utility to call 'ipython filename' and validate output/error.
206 """Utility to call 'ipython filename' and validate output/error.
206
207
207 This function raises an AssertionError if the validation fails.
208 This function raises an AssertionError if the validation fails.
208
209
209 Note that this starts IPython in a subprocess!
210 Note that this starts IPython in a subprocess!
210
211
211 Parameters
212 Parameters
212 ----------
213 ----------
213 fname : str
214 fname : str
214 Name of the file to be executed (should have .py or .ipy extension).
215 Name of the file to be executed (should have .py or .ipy extension).
215
216
216 expected_out : str
217 expected_out : str
217 Expected stdout of the process.
218 Expected stdout of the process.
218
219
219 expected_err : optional, str
220 expected_err : optional, str
220 Expected stderr of the process.
221 Expected stderr of the process.
221
222
222 options : optional, list
223 options : optional, list
223 Extra command-line flags to be passed to IPython.
224 Extra command-line flags to be passed to IPython.
224
225
225 Returns
226 Returns
226 -------
227 -------
227 None
228 None
228 """
229 """
229
230
230 import nose.tools as nt
231 import nose.tools as nt
231
232
232 out, err = ipexec(fname, options)
233 out, err = ipexec(fname, options)
233 #print 'OUT', out # dbg
234 #print 'OUT', out # dbg
234 #print 'ERR', err # dbg
235 #print 'ERR', err # dbg
235 # If there are any errors, we must check those befor stdout, as they may be
236 # If there are any errors, we must check those befor stdout, as they may be
236 # more informative than simply having an empty stdout.
237 # more informative than simply having an empty stdout.
237 if err:
238 if err:
238 if expected_err:
239 if expected_err:
239 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
240 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
240 else:
241 else:
241 raise ValueError('Running file %r produced error: %r' %
242 raise ValueError('Running file %r produced error: %r' %
242 (fname, err))
243 (fname, err))
243 # If no errors or output on stderr was expected, match stdout
244 # If no errors or output on stderr was expected, match stdout
244 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
245 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
245
246
246
247
247 class TempFileMixin(object):
248 class TempFileMixin(object):
248 """Utility class to create temporary Python/IPython files.
249 """Utility class to create temporary Python/IPython files.
249
250
250 Meant as a mixin class for test cases."""
251 Meant as a mixin class for test cases."""
251
252
252 def mktmp(self, src, ext='.py'):
253 def mktmp(self, src, ext='.py'):
253 """Make a valid python temp file."""
254 """Make a valid python temp file."""
254 fname, f = temp_pyfile(src, ext)
255 fname, f = temp_pyfile(src, ext)
255 self.tmpfile = f
256 self.tmpfile = f
256 self.fname = fname
257 self.fname = fname
257
258
258 def tearDown(self):
259 def tearDown(self):
259 if hasattr(self, 'tmpfile'):
260 if hasattr(self, 'tmpfile'):
260 # If the tmpfile wasn't made because of skipped tests, like in
261 # If the tmpfile wasn't made because of skipped tests, like in
261 # win32, there's nothing to cleanup.
262 # win32, there's nothing to cleanup.
262 self.tmpfile.close()
263 self.tmpfile.close()
263 try:
264 try:
264 os.unlink(self.fname)
265 os.unlink(self.fname)
265 except:
266 except:
266 # On Windows, even though we close the file, we still can't
267 # On Windows, even though we close the file, we still can't
267 # delete it. I have no clue why
268 # delete it. I have no clue why
268 pass
269 pass
269
270
270 pair_fail_msg = ("Testing {0}\n\n"
271 pair_fail_msg = ("Testing {0}\n\n"
271 "In:\n"
272 "In:\n"
272 " {1!r}\n"
273 " {1!r}\n"
273 "Expected:\n"
274 "Expected:\n"
274 " {2!r}\n"
275 " {2!r}\n"
275 "Got:\n"
276 "Got:\n"
276 " {3!r}\n")
277 " {3!r}\n")
277 def check_pairs(func, pairs):
278 def check_pairs(func, pairs):
278 """Utility function for the common case of checking a function with a
279 """Utility function for the common case of checking a function with a
279 sequence of input/output pairs.
280 sequence of input/output pairs.
280
281
281 Parameters
282 Parameters
282 ----------
283 ----------
283 func : callable
284 func : callable
284 The function to be tested. Should accept a single argument.
285 The function to be tested. Should accept a single argument.
285 pairs : iterable
286 pairs : iterable
286 A list of (input, expected_output) tuples.
287 A list of (input, expected_output) tuples.
287
288
288 Returns
289 Returns
289 -------
290 -------
290 None. Raises an AssertionError if any output does not match the expected
291 None. Raises an AssertionError if any output does not match the expected
291 value.
292 value.
292 """
293 """
293 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
294 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
294 for inp, expected in pairs:
295 for inp, expected in pairs:
295 out = func(inp)
296 out = func(inp)
296 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
297 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
297
298
298
299
299 if py3compat.PY3:
300 if py3compat.PY3:
300 MyStringIO = StringIO
301 MyStringIO = StringIO
301 else:
302 else:
302 # In Python 2, stdout/stderr can have either bytes or unicode written to them,
303 # In Python 2, stdout/stderr can have either bytes or unicode written to them,
303 # so we need a class that can handle both.
304 # so we need a class that can handle both.
304 class MyStringIO(StringIO):
305 class MyStringIO(StringIO):
305 def write(self, s):
306 def write(self, s):
306 s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
307 s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
307 super(MyStringIO, self).write(s)
308 super(MyStringIO, self).write(s)
308
309
309 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
310 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
310 -------
311 -------
311 {2!s}
312 {2!s}
312 -------
313 -------
313 """
314 """
314
315
315 class AssertPrints(object):
316 class AssertPrints(object):
316 """Context manager for testing that code prints certain text.
317 """Context manager for testing that code prints certain text.
317
318
318 Examples
319 Examples
319 --------
320 --------
320 >>> with AssertPrints("abc", suppress=False):
321 >>> with AssertPrints("abc", suppress=False):
321 ... print "abcd"
322 ... print "abcd"
322 ... print "def"
323 ... print "def"
323 ...
324 ...
324 abcd
325 abcd
325 def
326 def
326 """
327 """
327 def __init__(self, s, channel='stdout', suppress=True):
328 def __init__(self, s, channel='stdout', suppress=True):
328 self.s = s
329 self.s = s
329 self.channel = channel
330 self.channel = channel
330 self.suppress = suppress
331 self.suppress = suppress
331
332
332 def __enter__(self):
333 def __enter__(self):
333 self.orig_stream = getattr(sys, self.channel)
334 self.orig_stream = getattr(sys, self.channel)
334 self.buffer = MyStringIO()
335 self.buffer = MyStringIO()
335 self.tee = Tee(self.buffer, channel=self.channel)
336 self.tee = Tee(self.buffer, channel=self.channel)
336 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
337 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
337
338
338 def __exit__(self, etype, value, traceback):
339 def __exit__(self, etype, value, traceback):
339 self.tee.flush()
340 self.tee.flush()
340 setattr(sys, self.channel, self.orig_stream)
341 setattr(sys, self.channel, self.orig_stream)
341 printed = self.buffer.getvalue()
342 printed = self.buffer.getvalue()
342 assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
343 assert self.s in printed, notprinted_msg.format(self.s, self.channel, printed)
343 return False
344 return False
344
345
345 printed_msg = """Found {0!r} in printed output (on {1}):
346 printed_msg = """Found {0!r} in printed output (on {1}):
346 -------
347 -------
347 {2!s}
348 {2!s}
348 -------
349 -------
349 """
350 """
350
351
351 class AssertNotPrints(AssertPrints):
352 class AssertNotPrints(AssertPrints):
352 """Context manager for checking that certain output *isn't* produced.
353 """Context manager for checking that certain output *isn't* produced.
353
354
354 Counterpart of AssertPrints"""
355 Counterpart of AssertPrints"""
355 def __exit__(self, etype, value, traceback):
356 def __exit__(self, etype, value, traceback):
356 self.tee.flush()
357 self.tee.flush()
357 setattr(sys, self.channel, self.orig_stream)
358 setattr(sys, self.channel, self.orig_stream)
358 printed = self.buffer.getvalue()
359 printed = self.buffer.getvalue()
359 assert self.s not in printed, printed_msg.format(self.s, self.channel, printed)
360 assert self.s not in printed, printed_msg.format(self.s, self.channel, printed)
360 return False
361 return False
361
362
362 @contextmanager
363 @contextmanager
363 def mute_warn():
364 def mute_warn():
364 from IPython.utils import warn
365 from IPython.utils import warn
365 save_warn = warn.warn
366 save_warn = warn.warn
366 warn.warn = lambda *a, **kw: None
367 warn.warn = lambda *a, **kw: None
367 try:
368 try:
368 yield
369 yield
369 finally:
370 finally:
370 warn.warn = save_warn
371 warn.warn = save_warn
371
372
372 @contextmanager
373 @contextmanager
373 def make_tempfile(name):
374 def make_tempfile(name):
374 """ Create an empty, named, temporary file for the duration of the context.
375 """ Create an empty, named, temporary file for the duration of the context.
375 """
376 """
376 f = open(name, 'w')
377 f = open(name, 'w')
377 f.close()
378 f.close()
378 try:
379 try:
379 yield
380 yield
380 finally:
381 finally:
381 os.unlink(name)
382 os.unlink(name)
382
383
383
384
384 @contextmanager
385 @contextmanager
385 def monkeypatch(obj, name, attr):
386 def monkeypatch(obj, name, attr):
386 """
387 """
387 Context manager to replace attribute named `name` in `obj` with `attr`.
388 Context manager to replace attribute named `name` in `obj` with `attr`.
388 """
389 """
389 orig = getattr(obj, name)
390 orig = getattr(obj, name)
390 setattr(obj, name, attr)
391 setattr(obj, name, attr)
391 yield
392 yield
392 setattr(obj, name, orig)
393 setattr(obj, name, orig)
General Comments 0
You need to be logged in to leave comments. Login now