run-tests.py
471 lines
| 14.4 KiB
| text/x-python
|
PythonLexer
/ tests / run-tests.py
|
r2110 | #!/usr/bin/env python | ||
# | ||||
# run-tests.py - Run a set of tests on Mercurial | ||||
# | ||||
# Copyright 2006 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms | ||||
# of the GNU General Public License, incorporated herein by reference. | ||||
|
r2571 | import difflib | ||
import errno | ||||
import optparse | ||||
import os | ||||
import popen2 | ||||
import re | ||||
import shutil | ||||
import signal | ||||
import sys | ||||
|
r2110 | import tempfile | ||
|
r2571 | import time | ||
|
r2110 | |||
|
r4881 | # hghave reserved exit code to skip test | ||
SKIPPED_STATUS = 80 | ||||
|
r4365 | required_tools = ["python", "diff", "grep", "unzip", "gunzip", "bunzip2", "sed"] | ||
|
r2110 | |||
|
r2571 | parser = optparse.OptionParser("%prog [options] [tests]") | ||
|
r2144 | parser.add_option("-v", "--verbose", action="store_true", | ||
help="output verbose messages") | ||||
|
r2571 | parser.add_option("-t", "--timeout", type="int", | ||
|
r2672 | help="kill errant tests after TIMEOUT seconds") | ||
|
r2144 | parser.add_option("-c", "--cover", action="store_true", | ||
help="print a test coverage report") | ||||
parser.add_option("-s", "--cover_stdlib", action="store_true", | ||||
help="print a test coverage report inc. standard libraries") | ||||
parser.add_option("-C", "--annotate", action="store_true", | ||||
help="output files annotated with coverage") | ||||
|
r3300 | parser.add_option("-r", "--retest", action="store_true", | ||
help="retest failed tests") | ||||
|
r3301 | parser.add_option("-f", "--first", action="store_true", | ||
help="exit on the first test failure") | ||||
|
r3625 | parser.add_option("-R", "--restart", action="store_true", | ||
help="restart at last error") | ||||
|
r3626 | parser.add_option("-i", "--interactive", action="store_true", | ||
help="prompt to accept changed output") | ||||
|
r3300 | |||
|
r2576 | parser.set_defaults(timeout=180) | ||
|
r2110 | (options, args) = parser.parse_args() | ||
verbose = options.verbose | ||||
|
r2144 | coverage = options.cover or options.cover_stdlib or options.annotate | ||
|
r4319 | python = sys.executable | ||
|
r2110 | |||
def vlog(*msg): | ||||
if verbose: | ||||
for m in msg: | ||||
print m, | ||||
|
r2247 | def splitnewlines(text): | ||
'''like str.splitlines, but only split on newlines. | ||||
keep line endings.''' | ||||
i = 0 | ||||
lines = [] | ||||
while True: | ||||
n = text.find('\n', i) | ||||
if n == -1: | ||||
last = text[i:] | ||||
if last: | ||||
lines.append(last) | ||||
return lines | ||||
lines.append(text[i:n+1]) | ||||
i = n + 1 | ||||
|
r4881 | def extract_missing_features(lines): | ||
'''Extract missing/unknown features log lines as a list''' | ||||
missing = [] | ||||
for line in lines: | ||||
if not line.startswith('hghave: '): | ||||
continue | ||||
line = line.splitlines()[0] | ||||
missing.append(line[8:]) | ||||
return missing | ||||
|
r2110 | def show_diff(expected, output): | ||
for line in difflib.unified_diff(expected, output, | ||||
|
r2409 | "Expected output", "Test output"): | ||
|
r2247 | sys.stdout.write(line) | ||
|
r2110 | |||
def find_program(program): | ||||
"""Search PATH for a executable program""" | ||||
for p in os.environ.get('PATH', os.defpath).split(os.pathsep): | ||||
name = os.path.join(p, program) | ||||
if os.access(name, os.X_OK): | ||||
return name | ||||
return None | ||||
|
r2133 | def check_required_tools(): | ||
# Before we go any further, check for pre-requisite tools | ||||
# stuff from coreutils (cat, rm, etc) are not tested | ||||
for p in required_tools: | ||||
if os.name == 'nt': | ||||
p += '.exe' | ||||
found = find_program(p) | ||||
if found: | ||||
vlog("# Found prerequisite", p, "at", found) | ||||
else: | ||||
print "WARNING: Did not find prerequisite tool: "+p | ||||
|
r2110 | |||
def cleanup_exit(): | ||||
if verbose: | ||||
print "# Cleaning up HGTMP", HGTMP | ||||
shutil.rmtree(HGTMP, True) | ||||
|
r2570 | def use_correct_python(): | ||
# some tests run python interpreter. they must use same | ||||
# interpreter we use or bad things will happen. | ||||
exedir, exename = os.path.split(sys.executable) | ||||
if exename == 'python': | ||||
path = find_program('python') | ||||
if os.path.dirname(path) == exedir: | ||||
return | ||||
vlog('# Making python executable in test path use correct Python') | ||||
my_python = os.path.join(BINDIR, 'python') | ||||
try: | ||||
os.symlink(sys.executable, my_python) | ||||
except AttributeError: | ||||
# windows fallback | ||||
shutil.copyfile(sys.executable, my_python) | ||||
shutil.copymode(sys.executable, my_python) | ||||
|
r3223 | |||
|
r2133 | def install_hg(): | ||
|
r4319 | global python | ||
|
r2133 | vlog("# Performing temporary installation of HG") | ||
installerrs = os.path.join("tests", "install.err") | ||||
|
r2110 | |||
|
r5267 | # Run installer in hg root | ||
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..')) | ||||
|
r2183 | cmd = ('%s setup.py clean --all' | ||
|
r5189 | ' install --force --home="%s" --install-lib="%s"' | ||
' --install-scripts="%s" >%s 2>&1' | ||||
% (sys.executable, INST, PYTHONDIR, BINDIR, installerrs)) | ||||
|
r2133 | vlog("# Running", cmd) | ||
if os.system(cmd) == 0: | ||||
if not verbose: | ||||
os.remove(installerrs) | ||||
else: | ||||
f = open(installerrs) | ||||
for line in f: | ||||
print line, | ||||
f.close() | ||||
sys.exit(1) | ||||
os.chdir(TESTDIR) | ||||
|
r2110 | |||
|
r2133 | os.environ["PATH"] = "%s%s%s" % (BINDIR, os.pathsep, os.environ["PATH"]) | ||
|
r5251 | |||
pythonpath = os.environ.get("PYTHONPATH") | ||||
if pythonpath: | ||||
pythonpath = PYTHONDIR + os.pathsep + pythonpath | ||||
else: | ||||
pythonpath = PYTHONDIR | ||||
os.environ["PYTHONPATH"] = pythonpath | ||||
|
r2110 | |||
|
r2570 | use_correct_python() | ||
|
r2144 | if coverage: | ||
vlog("# Installing coverage wrapper") | ||||
os.environ['COVERAGE_FILE'] = COVERAGE_FILE | ||||
if os.path.exists(COVERAGE_FILE): | ||||
os.unlink(COVERAGE_FILE) | ||||
# Create a wrapper script to invoke hg via coverage.py | ||||
|
r2146 | os.rename(os.path.join(BINDIR, "hg"), os.path.join(BINDIR, "_hg.py")) | ||
|
r2144 | f = open(os.path.join(BINDIR, 'hg'), 'w') | ||
f.write('#!' + sys.executable + '\n') | ||||
|
r4633 | f.write('import sys, os; os.execv(sys.executable, [sys.executable, ' | ||
'"%s", "-x", "%s"] + sys.argv[1:])\n' % | ||||
(os.path.join(TESTDIR, 'coverage.py'), | ||||
os.path.join(BINDIR, '_hg.py'))) | ||||
|
r2144 | f.close() | ||
os.chmod(os.path.join(BINDIR, 'hg'), 0700) | ||||
|
r4319 | python = '"%s" "%s" -x' % (sys.executable, | ||
os.path.join(TESTDIR,'coverage.py')) | ||||
|
r2144 | |||
def output_coverage(): | ||||
|
r2145 | vlog("# Producing coverage report") | ||
omit = [BINDIR, TESTDIR, PYTHONDIR] | ||||
if not options.cover_stdlib: | ||||
# Exclude as system paths (ignoring empty strings seen on win) | ||||
|
r2146 | omit += [x for x in sys.path if x != ''] | ||
|
r2145 | omit = ','.join(omit) | ||
os.chdir(PYTHONDIR) | ||||
|
r4318 | cmd = '"%s" "%s" -i -r "--omit=%s"' % ( | ||
|
r2145 | sys.executable, os.path.join(TESTDIR, 'coverage.py'), omit) | ||
vlog("# Running: "+cmd) | ||||
os.system(cmd) | ||||
if options.annotate: | ||||
adir = os.path.join(TESTDIR, 'annotated') | ||||
if not os.path.isdir(adir): | ||||
os.mkdir(adir) | ||||
|
r4318 | cmd = '"%s" "%s" -i -a "--directory=%s" "--omit=%s"' % ( | ||
|
r2145 | sys.executable, os.path.join(TESTDIR, 'coverage.py'), | ||
adir, omit) | ||||
|
r2144 | vlog("# Running: "+cmd) | ||
os.system(cmd) | ||||
|
r2571 | class Timeout(Exception): | ||
pass | ||||
def alarmed(signum, frame): | ||||
raise Timeout | ||||
|
r2247 | def run(cmd): | ||
|
r2110 | """Run command in a sub-process, capturing the output (stdout and stderr). | ||
Return the exist code, and output.""" | ||||
# TODO: Use subprocess.Popen if we're running on Python 2.4 | ||||
if os.name == 'nt': | ||||
tochild, fromchild = os.popen4(cmd) | ||||
tochild.close() | ||||
output = fromchild.read() | ||||
ret = fromchild.close() | ||||
if ret == None: | ||||
ret = 0 | ||||
else: | ||||
proc = popen2.Popen4(cmd) | ||||
|
r2571 | try: | ||
output = '' | ||||
proc.tochild.close() | ||||
output = proc.fromchild.read() | ||||
ret = proc.wait() | ||||
|
r4880 | if os.WIFEXITED(ret): | ||
ret = os.WEXITSTATUS(ret) | ||||
|
r2571 | except Timeout: | ||
vlog('# Process %d timed out - killing it' % proc.pid) | ||||
os.kill(proc.pid, signal.SIGTERM) | ||||
ret = proc.wait() | ||||
if ret == 0: | ||||
ret = signal.SIGTERM << 8 | ||||
|
r5078 | output += ("\n### Abort: timeout after %d seconds.\n" | ||
% options.timeout) | ||||
|
r2247 | return ret, splitnewlines(output) | ||
|
r2110 | |||
def run_one(test): | ||||
|
r2710 | '''tristate output: | ||
None -> skipped | ||||
True -> passed | ||||
False -> failed''' | ||||
|
r2110 | vlog("# Test", test) | ||
if not verbose: | ||||
sys.stdout.write('.') | ||||
sys.stdout.flush() | ||||
|
r2989 | # create a fresh hgrc | ||
hgrc = file(HGRCPATH, 'w+') | ||||
|
r4529 | hgrc.write('[ui]\n') | ||
hgrc.write('slash = True\n') | ||||
|
r2989 | hgrc.close() | ||
|
r2110 | err = os.path.join(TESTDIR, test+".err") | ||
ref = os.path.join(TESTDIR, test+".out") | ||||
|
r4320 | testpath = os.path.join(TESTDIR, test) | ||
|
r2110 | |||
if os.path.exists(err): | ||||
os.remove(err) # Remove any previous output files | ||||
# Make a tmp subdirectory to work in | ||||
tmpd = os.path.join(HGTMP, test) | ||||
os.mkdir(tmpd) | ||||
os.chdir(tmpd) | ||||
|
r4321 | try: | ||
tf = open(testpath) | ||||
firstline = tf.readline().rstrip() | ||||
tf.close() | ||||
except: | ||||
firstline = '' | ||||
|
r2710 | lctest = test.lower() | ||
|
r2110 | |||
|
r4321 | if lctest.endswith('.py') or firstline == '#!/usr/bin/env python': | ||
|
r4320 | cmd = '%s "%s"' % (python, testpath) | ||
|
r2710 | elif lctest.endswith('.bat'): | ||
# do not run batch scripts on non-windows | ||||
if os.name != 'nt': | ||||
print '\nSkipping %s: batch script' % test | ||||
return None | ||||
# To reliably get the error code from batch files on WinXP, | ||||
# the "cmd /c call" prefix is needed. Grrr | ||||
|
r4320 | cmd = 'cmd /c call "%s"' % testpath | ||
|
r2710 | else: | ||
# do not run shell scripts on windows | ||||
if os.name == 'nt': | ||||
print '\nSkipping %s: shell script' % test | ||||
return None | ||||
# do not try to run non-executable programs | ||||
|
r4320 | if not os.access(testpath, os.X_OK): | ||
|
r2710 | print '\nSkipping %s: not executable' % test | ||
return None | ||||
|
r4320 | cmd = '"%s"' % testpath | ||
|
r2110 | |||
|
r2571 | if options.timeout > 0: | ||
signal.alarm(options.timeout) | ||||
|
r2110 | vlog("# Running", cmd) | ||
ret, out = run(cmd) | ||||
vlog("# Ret was:", ret) | ||||
|
r2571 | if options.timeout > 0: | ||
signal.alarm(0) | ||||
|
r4881 | skipped = (ret == SKIPPED_STATUS) | ||
|
r2213 | diffret = 0 | ||
# If reference output file exists, check test output against it | ||||
if os.path.exists(ref): | ||||
f = open(ref, "r") | ||||
|
r2247 | ref_out = splitnewlines(f.read()) | ||
|
r2213 | f.close() | ||
|
r2246 | else: | ||
|
r2703 | ref_out = [] | ||
|
r4881 | if not skipped and out != ref_out: | ||
|
r2246 | diffret = 1 | ||
print "\nERROR: %s output changed" % (test) | ||||
show_diff(ref_out, out) | ||||
|
r4881 | if skipped: | ||
missing = extract_missing_features(out) | ||||
if not missing: | ||||
missing = ['irrelevant'] | ||||
print '\nSkipping %s: %s' % (test, missing[-1]) | ||||
elif ret: | ||||
|
r2110 | print "\nERROR: %s failed with error code %d" % (test, ret) | ||
|
r2213 | elif diffret: | ||
ret = diffret | ||||
|
r2110 | |||
|
r5081 | if ret != 0 and not skipped: | ||
|
r4881 | # Save errors to a file for diagnosis | ||
|
r2247 | f = open(err, "wb") | ||
|
r2110 | for line in out: | ||
f.write(line) | ||||
f.close() | ||||
|
r2571 | # Kill off any leftover daemon processes | ||
try: | ||||
fp = file(DAEMON_PIDS) | ||||
for line in fp: | ||||
try: | ||||
pid = int(line) | ||||
except ValueError: | ||||
continue | ||||
try: | ||||
os.kill(pid, 0) | ||||
vlog('# Killing daemon process %d' % pid) | ||||
os.kill(pid, signal.SIGTERM) | ||||
time.sleep(0.25) | ||||
os.kill(pid, 0) | ||||
vlog('# Daemon process %d is stuck - really killing it' % pid) | ||||
os.kill(pid, signal.SIGKILL) | ||||
except OSError, err: | ||||
if err.errno != errno.ESRCH: | ||||
raise | ||||
fp.close() | ||||
os.unlink(DAEMON_PIDS) | ||||
except IOError: | ||||
pass | ||||
|
r2110 | os.chdir(TESTDIR) | ||
shutil.rmtree(tmpd, True) | ||||
|
r4881 | if skipped: | ||
return None | ||||
|
r2110 | return ret == 0 | ||
|
r2133 | |||
os.umask(022) | ||||
check_required_tools() | ||||
# Reset some environment variables to well-known values so that | ||||
# the tests produce repeatable output. | ||||
os.environ['LANG'] = os.environ['LC_ALL'] = 'C' | ||||
os.environ['TZ'] = 'GMT' | ||||
TESTDIR = os.environ["TESTDIR"] = os.getcwd() | ||||
HGTMP = os.environ["HGTMP"] = tempfile.mkdtemp("", "hgtests.") | ||||
|
r2571 | DAEMON_PIDS = os.environ["DAEMON_PIDS"] = os.path.join(HGTMP, 'daemon.pids') | ||
|
r2989 | HGRCPATH = os.environ["HGRCPATH"] = os.path.join(HGTMP, '.hgrc') | ||
|
r2571 | |||
|
r4365 | os.environ["HGEDITOR"] = sys.executable + ' -c "import sys; sys.exit(0)"' | ||
|
r4387 | os.environ["HGMERGE"] = ('python "%s" -L my -L other' | ||
% os.path.join(TESTDIR, os.path.pardir, 'contrib', | ||||
'simplemerge')) | ||||
|
r4365 | os.environ["HGUSER"] = "test" | ||
os.environ["HGENCODING"] = "ascii" | ||||
os.environ["HGENCODINGMODE"] = "strict" | ||||
|
r2133 | vlog("# Using TESTDIR", TESTDIR) | ||
vlog("# Using HGTMP", HGTMP) | ||||
|
r2110 | |||
|
r2144 | INST = os.path.join(HGTMP, "install") | ||
BINDIR = os.path.join(INST, "bin") | ||||
PYTHONDIR = os.path.join(INST, "lib", "python") | ||||
COVERAGE_FILE = os.path.join(TESTDIR, ".coverage") | ||||
|
r2133 | try: | ||
|
r2258 | try: | ||
install_hg() | ||||
|
r2133 | |||
|
r2571 | if options.timeout > 0: | ||
try: | ||||
signal.signal(signal.SIGALRM, alarmed) | ||||
vlog('# Running tests with %d-second timeout' % | ||||
options.timeout) | ||||
except AttributeError: | ||||
print 'WARNING: cannot run tests with timeouts' | ||||
options.timeout = 0 | ||||
|
r3625 | tested = 0 | ||
|
r2258 | failed = 0 | ||
|
r2710 | skipped = 0 | ||
|
r2110 | |||
|
r2258 | if len(args) == 0: | ||
args = os.listdir(".") | ||||
|
r3624 | args.sort() | ||
|
r3625 | |||
tests = [] | ||||
|
r2258 | for test in args: | ||
|
r2702 | if (test.startswith("test-") and '~' not in test and | ||
|
r3223 | ('.' not in test or test.endswith('.py') or | ||
|
r2702 | test.endswith('.bat'))): | ||
|
r3625 | tests.append(test) | ||
if options.restart: | ||||
orig = list(tests) | ||||
while tests: | ||||
if os.path.exists(tests[0] + ".err"): | ||||
break | ||||
tests.pop(0) | ||||
if not tests: | ||||
print "running all tests" | ||||
tests = orig | ||||
|
r2133 | |||
|
r3625 | for test in tests: | ||
if options.retest and not os.path.exists(test + ".err"): | ||||
skipped += 1 | ||||
continue | ||||
ret = run_one(test) | ||||
if ret is None: | ||||
skipped += 1 | ||||
elif not ret: | ||||
|
r3626 | if options.interactive: | ||
print "Accept this change? [n] ", | ||||
answer = sys.stdin.readline().strip() | ||||
if answer.lower() in "y yes".split(): | ||||
os.rename(test + ".err", test + ".out") | ||||
tested += 1 | ||||
continue | ||||
|
r3625 | failed += 1 | ||
if options.first: | ||||
break | ||||
tested += 1 | ||||
print "\n# Ran %d tests, %d skipped, %d failed." % (tested, skipped, | ||||
|
r2710 | failed) | ||
|
r2258 | if coverage: | ||
output_coverage() | ||||
except KeyboardInterrupt: | ||||
failed = True | ||||
print "\ninterrupted!" | ||||
|
r2133 | finally: | ||
cleanup_exit() | ||||
|
r2110 | |||
if failed: | ||||
sys.exit(1) | ||||