tools.py
476 lines
| 13.8 KiB
| text/x-python
|
PythonLexer
|
r6421 | """Generic testing tools. | ||
|
r1955 | |||
Authors | ||||
------- | ||||
- Fernando Perez <Fernando.Perez@berkeley.edu> | ||||
""" | ||||
|
r2498 | |||
|
r22983 | # Copyright (c) IPython Development Team. | ||
# Distributed under the terms of the Modified BSD License. | ||||
|
r1955 | |||
import os | ||||
|
r26129 | from pathlib import Path | ||
|
r2353 | import re | ||
|
r1955 | import sys | ||
|
r4486 | import tempfile | ||
|
r25113 | import unittest | ||
|
r1955 | |||
|
r4214 | from contextlib import contextmanager | ||
|
r4901 | from io import StringIO | ||
|
r11475 | from subprocess import Popen, PIPE | ||
|
r22983 | from unittest.mock import patch | ||
|
r4214 | |||
|
r21253 | from traitlets.config.loader import Config | ||
|
r12354 | from IPython.utils.process import get_output_error_code | ||
|
r6653 | from IPython.utils.text import list_strings | ||
|
r4901 | from IPython.utils.io import temp_pyfile, Tee | ||
from IPython.utils import py3compat | ||||
|
r1955 | |||
|
r2461 | from . import decorators as dec | ||
|
r3905 | from . import skipdoctest | ||
|
r2461 | |||
|
r1955 | |||
|
r2461 | # The docstring for full_path doctests differently on win32 (different path | ||
# separator) so just skip the doctest there. The example remains informative. | ||||
|
r3905 | doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco | ||
|
r1982 | |||
|
r2461 | @doctest_deco | ||
|
r1955 | def full_path(startPath,files): | ||
"""Make full paths for all the listed files, based on startPath. | ||||
Only the base part of startPath is kept, since this routine is typically | ||||
|
r13592 | used with a script's ``__file__`` variable as startPath. The base of startPath | ||
|
r1955 | is then prepended to all the listed files, forming the output list. | ||
|
r1973 | Parameters | ||
---------- | ||||
|
r13592 | startPath : string | ||
Initial path to use as the base for the results. This path is split | ||||
|
r1955 | using os.path.split() and only its first component is kept. | ||
|
r13592 | files : string or list | ||
One or more files. | ||||
|
r1955 | |||
|
r1973 | Examples | ||
-------- | ||||
|
r1955 | |||
>>> full_path('/foo/bar.py',['a.txt','b.txt']) | ||||
['/foo/a.txt', '/foo/b.txt'] | ||||
>>> full_path('/foo',['a.txt','b.txt']) | ||||
['/a.txt', '/b.txt'] | ||||
|
r13592 | If a single file is given, the output is still a list:: | ||
>>> full_path('/foo','a.txt') | ||||
['/a.txt'] | ||||
|
r1955 | """ | ||
|
r2498 | files = list_strings(files) | ||
|
r1955 | base = os.path.split(startPath)[0] | ||
return [ os.path.join(base,f) for f in files ] | ||||
|
r2353 | |||
def parse_test_output(txt): | ||||
"""Parse the output of a test run and return errors, failures. | ||||
Parameters | ||||
---------- | ||||
txt : str | ||||
Text output of a test run, assumed to contain a line of one of the | ||||
following forms:: | ||||
|
r24067 | |||
|
r2353 | 'FAILED (errors=1)' | ||
'FAILED (failures=1)' | ||||
'FAILED (errors=1, failures=1)' | ||||
Returns | ||||
------- | ||||
|
r13592 | nerr, nfail | ||
number of errors and failures. | ||||
|
r2353 | """ | ||
err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE) | ||||
if err_m: | ||||
nerr = int(err_m.group(1)) | ||||
nfail = 0 | ||||
return nerr, nfail | ||||
|
r4872 | |||
|
r2353 | fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE) | ||
if fail_m: | ||||
nerr = 0 | ||||
nfail = int(fail_m.group(1)) | ||||
return nerr, nfail | ||||
both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt, | ||||
re.MULTILINE) | ||||
if both_m: | ||||
nerr = int(both_m.group(1)) | ||||
nfail = int(both_m.group(2)) | ||||
return nerr, nfail | ||||
|
r4872 | |||
|
r2353 | # If the input didn't match any of these forms, assume no error/failures | ||
return 0, 0 | ||||
|
r2414 | |||
|
r2353 | # So nose doesn't think this is a test | ||
parse_test_output.__test__ = False | ||||
|
r2414 | |||
def default_argv(): | ||||
"""Return a valid default argv for creating testing instances of ipython""" | ||||
|
r2477 | return ['--quick', # so no config file is loaded | ||
# Other defaults to minimize side effects on stdout | ||||
|
r4197 | '--colors=NoColor', '--no-term-title','--no-banner', | ||
'--autocall=0'] | ||||
|
r2499 | |||
def default_config(): | ||||
"""Return a config object with good defaults for testing.""" | ||||
config = Config() | ||||
|
r2761 | config.TerminalInteractiveShell.colors = 'NoColor' | ||
config.TerminalTerminalInteractiveShell.term_title = False, | ||||
config.TerminalInteractiveShell.autocall = 0 | ||||
|
r15372 | f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False) | ||
|
r26129 | config.HistoryManager.hist_file = Path(f.name) | ||
|
r15372 | f.close() | ||
|
r3712 | config.HistoryManager.db_cache_size = 10000 | ||
|
r2499 | return config | ||
|
r2414 | |||
|
r11861 | def get_ipython_cmd(as_string=False): | ||
""" | ||||
Return appropriate IPython command line name. By default, this will return | ||||
a list that can be used with subprocess.Popen, for example, but passing | ||||
`as_string=True` allows for returning the IPython command as a string. | ||||
Parameters | ||||
---------- | ||||
as_string: bool | ||||
Flag to allow to return the command as a string. | ||||
""" | ||||
|
r12151 | ipython_cmd = [sys.executable, "-m", "IPython"] | ||
|
r11861 | |||
if as_string: | ||||
ipython_cmd = " ".join(ipython_cmd) | ||||
return ipython_cmd | ||||
|
r18526 | def ipexec(fname, options=None, commands=()): | ||
|
r2414 | """Utility to call 'ipython filename'. | ||
|
r11353 | Starts IPython with a minimal and safe configuration to make startup as fast | ||
|
r2414 | as possible. | ||
Note that this starts IPython in a subprocess! | ||||
Parameters | ||||
---------- | ||||
|
r27014 | fname : str, Path | ||
|
r2414 | Name of file to be executed (should have .py or .ipy extension). | ||
|
r2415 | options : optional, list | ||
Extra command-line flags to be passed to IPython. | ||||
|
r18526 | commands : optional, list | ||
Commands to send in on stdin | ||||
|
r2414 | Returns | ||
------- | ||||
|
r24067 | ``(stdout, stderr)`` of ipython subprocess. | ||
|
r2415 | """ | ||
|
r27243 | __tracebackhide__ = True | ||
if options is None: | ||||
options = [] | ||||
|
r4872 | |||
|
r22427 | cmdargs = default_argv() + options | ||
|
r4872 | |||
|
r2414 | test_dir = os.path.dirname(__file__) | ||
|
r11861 | |||
ipython_cmd = get_ipython_cmd() | ||||
|
r2446 | # Absolute path for filename | ||
|
r2507 | full_fname = os.path.join(test_dir, fname) | ||
|
r25932 | full_cmd = ipython_cmd + cmdargs + ['--', full_fname] | ||
|
r15466 | env = os.environ.copy() | ||
|
r21000 | # FIXME: ignore all warnings in ipexec while we have shims | ||
# should we keep suppressing warnings here, even after removing shims? | ||||
env['PYTHONWARNINGS'] = 'ignore' | ||||
# env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr | ||||
|
r26939 | # Prevent coloring under PyCharm ("\x1b[0m" at the end of the stdout) | ||
env.pop("PYCHARM_HOSTED", None) | ||||
|
r18946 | for k, v in env.items(): | ||
# Debug a bizarre failure we've seen on Windows: | ||||
# TypeError: environment can only contain strings | ||||
if not isinstance(v, str): | ||||
|
r18948 | print(k, v) | ||
|
r18526 | p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env) | ||
|
r24010 | out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None) | ||
out, err = py3compat.decode(out), py3compat.decode(err) | ||||
|
r6190 | # `import readline` causes 'ESC[?1034h' to be output sometimes, | ||
# so strip that out before doing comparisons | ||||
|
r4470 | if out: | ||
|
r6190 | out = re.sub(r'\x1b\[[^h]+h', '', out) | ||
|
r6221 | return out, err | ||
|
r2414 | |||
|
r2494 | def ipexec_validate(fname, expected_out, expected_err='', | ||
|
r18526 | options=None, commands=()): | ||
|
r2414 | """Utility to call 'ipython filename' and validate output/error. | ||
This function raises an AssertionError if the validation fails. | ||||
Note that this starts IPython in a subprocess! | ||||
Parameters | ||||
---------- | ||||
|
r27014 | fname : str, Path | ||
|
r2414 | Name of the file to be executed (should have .py or .ipy extension). | ||
expected_out : str | ||||
Expected stdout of the process. | ||||
|
r2415 | expected_err : optional, str | ||
Expected stderr of the process. | ||||
options : optional, list | ||||
Extra command-line flags to be passed to IPython. | ||||
|
r2414 | Returns | ||
------- | ||||
None | ||||
""" | ||||
|
r27243 | __tracebackhide__ = True | ||
|
r2414 | |||
|
r18526 | out, err = ipexec(fname, options, commands) | ||
|
r28756 | # print('OUT', out) # dbg | ||
# print('ERR', err) # dbg | ||||
|
r24132 | # If there are any errors, we must check those before stdout, as they may be | ||
|
r2494 | # more informative than simply having an empty stdout. | ||
if err: | ||||
if expected_err: | ||||
|
r27243 | assert "\n".join(err.strip().splitlines()) == "\n".join( | ||
expected_err.strip().splitlines() | ||||
) | ||||
|
r2494 | else: | ||
raise ValueError('Running file %r produced error: %r' % | ||||
(fname, err)) | ||||
# If no errors or output on stderr was expected, match stdout | ||||
|
r27243 | assert "\n".join(out.strip().splitlines()) == "\n".join( | ||
expected_out.strip().splitlines() | ||||
) | ||||
|
r2415 | |||
|
r25113 | class TempFileMixin(unittest.TestCase): | ||
|
r2415 | """Utility class to create temporary Python/IPython files. | ||
Meant as a mixin class for test cases.""" | ||||
|
r4872 | |||
|
r2415 | def mktmp(self, src, ext='.py'): | ||
"""Make a valid python temp file.""" | ||||
|
r25106 | fname = temp_pyfile(src, ext) | ||
|
r24341 | if not hasattr(self, 'tmps'): | ||
self.tmps=[] | ||||
|
r25106 | self.tmps.append(fname) | ||
|
r2415 | self.fname = fname | ||
|
r2907 | def tearDown(self): | ||
|
r24341 | # If the tmpfile wasn't made because of skipped tests, like in | ||
# win32, there's nothing to cleanup. | ||||
if hasattr(self, 'tmps'): | ||||
|
r25106 | for fname in self.tmps: | ||
|
r24341 | # If the tmpfile wasn't made because of skipped tests, like in | ||
# win32, there's nothing to cleanup. | ||||
try: | ||||
os.unlink(fname) | ||||
except: | ||||
# On Windows, even though we close the file, we still can't | ||||
# delete it. I have no clue why | ||||
pass | ||||
|
r2415 | |||
|
r22379 | def __enter__(self): | ||
return self | ||||
def __exit__(self, exc_type, exc_value, traceback): | ||||
self.tearDown() | ||||
|
r4746 | pair_fail_msg = ("Testing {0}\n\n" | ||
|
r4079 | "In:\n" | ||
" {1!r}\n" | ||||
"Expected:\n" | ||||
" {2!r}\n" | ||||
"Got:\n" | ||||
" {3!r}\n") | ||||
def check_pairs(func, pairs): | ||||
|
r4872 | """Utility function for the common case of checking a function with a | ||
|
r4079 | sequence of input/output pairs. | ||
|
r4872 | |||
|
r4079 | Parameters | ||
---------- | ||||
func : callable | ||||
The function to be tested. Should accept a single argument. | ||||
pairs : iterable | ||||
A list of (input, expected_output) tuples. | ||||
|
r4872 | |||
|
r4079 | Returns | ||
------- | ||||
None. Raises an AssertionError if any output does not match the expected | ||||
value. | ||||
""" | ||||
|
r27243 | __tracebackhide__ = True | ||
|
r4746 | name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>")) | ||
|
r4079 | for inp, expected in pairs: | ||
out = func(inp) | ||||
|
r4746 | assert out == expected, pair_fail_msg.format(name, inp, expected, out) | ||
|
r4214 | |||
|
r4920 | |||
|
r23094 | MyStringIO = StringIO | ||
|
r4901 | |||
|
r15219 | _re_type = type(re.compile(r'')) | ||
|
r4901 | notprinted_msg = """Did not find {0!r} in printed output (on {1}): | ||
|
r8099 | ------- | ||
{2!s} | ||||
------- | ||||
""" | ||||
|
r4920 | |||
|
r4901 | class AssertPrints(object): | ||
"""Context manager for testing that code prints certain text. | ||||
|
r24067 | |||
|
r4901 | Examples | ||
-------- | ||||
|
r4904 | >>> with AssertPrints("abc", suppress=False): | ||
|
r13393 | ... print("abcd") | ||
... print("def") | ||||
|
r24067 | ... | ||
|
r4901 | abcd | ||
def | ||||
""" | ||||
|
r4904 | def __init__(self, s, channel='stdout', suppress=True): | ||
|
r4901 | self.s = s | ||
|
r23037 | if isinstance(self.s, (str, _re_type)): | ||
|
r12543 | self.s = [self.s] | ||
|
r4901 | self.channel = channel | ||
|
r4904 | self.suppress = suppress | ||
|
r24067 | |||
|
r4901 | def __enter__(self): | ||
self.orig_stream = getattr(sys, self.channel) | ||||
self.buffer = MyStringIO() | ||||
self.tee = Tee(self.buffer, channel=self.channel) | ||||
|
r4904 | setattr(sys, self.channel, self.buffer if self.suppress else self.tee) | ||
|
r24067 | |||
|
r4901 | def __exit__(self, etype, value, traceback): | ||
|
r27243 | __tracebackhide__ = True | ||
|
r17801 | try: | ||
if value is not None: | ||||
# If an error was raised, don't check anything else | ||||
return False | ||||
self.tee.flush() | ||||
setattr(sys, self.channel, self.orig_stream) | ||||
printed = self.buffer.getvalue() | ||||
for s in self.s: | ||||
if isinstance(s, _re_type): | ||||
assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed) | ||||
else: | ||||
assert s in printed, notprinted_msg.format(s, self.channel, printed) | ||||
|
r12950 | return False | ||
|
r17801 | finally: | ||
self.tee.close() | ||||
|
r8099 | |||
printed_msg = """Found {0!r} in printed output (on {1}): | ||||
------- | ||||
{2!s} | ||||
------- | ||||
""" | ||||
|
r4904 | class AssertNotPrints(AssertPrints): | ||
"""Context manager for checking that certain output *isn't* produced. | ||||
|
r24067 | |||
|
r4904 | Counterpart of AssertPrints""" | ||
def __exit__(self, etype, value, traceback): | ||||
|
r27243 | __tracebackhide__ = True | ||
|
r17801 | try: | ||
if value is not None: | ||||
# If an error was raised, don't check anything else | ||||
self.tee.close() | ||||
return False | ||||
self.tee.flush() | ||||
setattr(sys, self.channel, self.orig_stream) | ||||
printed = self.buffer.getvalue() | ||||
for s in self.s: | ||||
if isinstance(s, _re_type): | ||||
assert not s.search(printed),printed_msg.format( | ||||
s.pattern, self.channel, printed) | ||||
else: | ||||
assert s not in printed, printed_msg.format( | ||||
s, self.channel, printed) | ||||
|
r12950 | return False | ||
|
r17801 | finally: | ||
self.tee.close() | ||||
|
r4901 | |||
|
r4214 | @contextmanager | ||
def mute_warn(): | ||||
from IPython.utils import warn | ||||
save_warn = warn.warn | ||||
warn.warn = lambda *a, **kw: None | ||||
try: | ||||
yield | ||||
finally: | ||||
|
r4688 | warn.warn = save_warn | ||
@contextmanager | ||||
def make_tempfile(name): | ||||
|
r27495 | """Create an empty, named, temporary file for the duration of the context.""" | ||
open(name, "w", encoding="utf-8").close() | ||||
|
r4688 | try: | ||
yield | ||||
finally: | ||||
os.unlink(name) | ||||
|
r7858 | |||
|
r22983 | def fake_input(inputs): | ||
"""Temporarily replace the input() function to return the given values | ||||
Use as a context manager: | ||||
with fake_input(['result1', 'result2']): | ||||
... | ||||
Values are returned in order. If input() is called again after the last value | ||||
was used, EOFError is raised. | ||||
""" | ||||
it = iter(inputs) | ||||
def mock_input(prompt=''): | ||||
try: | ||||
return next(it) | ||||
|
r25833 | except StopIteration as e: | ||
raise EOFError('No more inputs given') from e | ||||
|
r22983 | |||
return patch('builtins.input', mock_input) | ||||
|
r7858 | |||
|
r12354 | def help_output_test(subcommand=''): | ||
"""test that `ipython [subcommand] -h` works""" | ||||
|
r13739 | cmd = get_ipython_cmd() + [subcommand, '-h'] | ||
|
r12354 | out, err, rc = get_output_error_code(cmd) | ||
|
r27041 | assert rc == 0, err | ||
assert "Traceback" not in err | ||||
assert "Options" in out | ||||
assert "--help-all" in out | ||||
|
r12354 | return out, err | ||
def help_all_output_test(subcommand=''): | ||||
"""test that `ipython [subcommand] --help-all` works""" | ||||
|
r13739 | cmd = get_ipython_cmd() + [subcommand, '--help-all'] | ||
|
r12354 | out, err, rc = get_output_error_code(cmd) | ||
|
r27041 | assert rc == 0, err | ||
assert "Traceback" not in err | ||||
assert "Options" in out | ||||
assert "Class" in out | ||||
|
r12354 | return out, err | ||