test-verify-repo-operations.py
593 lines
| 18.2 KiB
| text/x-python
|
PythonLexer
/ tests / test-verify-repo-operations.py
|
r28255 | from __future__ import print_function, absolute_import | ||
"""Fuzz testing for operations against a Mercurial repository | ||||
This uses Hypothesis's stateful testing to generate random repository | ||||
operations and test Mercurial using them, both to see if there are any | ||||
unexpected errors and to compare different versions of it.""" | ||||
import os | ||||
import sys | ||||
# These tests require Hypothesis and pytz to be installed. | ||||
# Running 'pip install hypothesis pytz' will achieve that. | ||||
# Note: This won't work if you're running Python < 2.7. | ||||
try: | ||||
from hypothesis.extra.datetime import datetimes | ||||
except ImportError: | ||||
sys.stderr.write("skipped: hypothesis or pytz not installed" + os.linesep) | ||||
sys.exit(80) | ||||
# If you are running an old version of pip you may find that the enum34 | ||||
# backport is not installed automatically. If so 'pip install enum34' will | ||||
# fix this problem. | ||||
try: | ||||
import enum | ||||
assert enum # Silence pyflakes | ||||
except ImportError: | ||||
sys.stderr.write("skipped: enum34 not installed" + os.linesep) | ||||
sys.exit(80) | ||||
import binascii | ||||
from contextlib import contextmanager | ||||
import errno | ||||
import pipes | ||||
import shutil | ||||
import silenttestrunner | ||||
import subprocess | ||||
from hypothesis.errors import HypothesisException | ||||
|
r28258 | from hypothesis.stateful import ( | ||
rule, RuleBasedStateMachine, Bundle, precondition) | ||||
|
r28255 | from hypothesis import settings, note, strategies as st | ||
from hypothesis.configuration import set_hypothesis_home_dir | ||||
|
r28259 | from hypothesis.database import ExampleDatabase | ||
|
r28255 | |||
testdir = os.path.abspath(os.environ["TESTDIR"]) | ||||
# We store Hypothesis examples here rather in the temporary test directory | ||||
# so that when rerunning a failing test this always results in refinding the | ||||
# previous failure. This directory is in .hgignore and should not be checked in | ||||
# but is useful to have for development. | ||||
set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis")) | ||||
runtests = os.path.join(os.environ["RUNTESTDIR"], "run-tests.py") | ||||
testtmp = os.environ["TESTTMP"] | ||||
assert os.path.isdir(testtmp) | ||||
generatedtests = os.path.join(testdir, "hypothesis-generated") | ||||
try: | ||||
os.makedirs(generatedtests) | ||||
except OSError: | ||||
pass | ||||
# We write out generated .t files to a file in order to ease debugging and to | ||||
# give a starting point for turning failures Hypothesis finds into normal | ||||
# tests. In order to ensure that multiple copies of this test can be run in | ||||
# parallel we use atomic file create to ensure that we always get a unique | ||||
# name. | ||||
file_index = 0 | ||||
while True: | ||||
file_index += 1 | ||||
savefile = os.path.join(generatedtests, "test-generated-%d.t" % ( | ||||
file_index, | ||||
)) | ||||
try: | ||||
os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY)) | ||||
break | ||||
except OSError as e: | ||||
if e.errno != errno.EEXIST: | ||||
raise | ||||
assert os.path.exists(savefile) | ||||
hgrc = os.path.join(".hg", "hgrc") | ||||
filecharacters = ( | ||||
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" | ||||
"[]^_`;=@{}~ !#$%&'()+,-" | ||||
) | ||||
files = st.text(filecharacters, min_size=1).map(lambda x: x.strip()).filter( | ||||
bool).map(lambda s: s.encode('ascii')) | ||||
safetext = st.text(st.characters( | ||||
min_codepoint=1, max_codepoint=127, | ||||
blacklist_categories=('Cc', 'Cs')), min_size=1).map( | ||||
lambda s: s.encode('utf-8') | ||||
) | ||||
@contextmanager | ||||
def acceptableerrors(*args): | ||||
"""Sometimes we know an operation we're about to perform might fail, and | ||||
we're OK with some of the failures. In those cases this may be used as a | ||||
context manager and will swallow expected failures, as identified by | ||||
substrings of the error message Mercurial emits.""" | ||||
try: | ||||
yield | ||||
except subprocess.CalledProcessError as e: | ||||
if not any(a in e.output for a in args): | ||||
note(e.output) | ||||
raise | ||||
|
r28257 | reponames = st.text("abcdefghijklmnopqrstuvwxyz01234556789", min_size=1).map( | ||
lambda s: s.encode('ascii') | ||||
) | ||||
|
r28255 | class verifyingstatemachine(RuleBasedStateMachine): | ||
"""This defines the set of acceptable operations on a Mercurial repository | ||||
using Hypothesis's RuleBasedStateMachine. | ||||
The general concept is that we manage multiple repositories inside a | ||||
repos/ directory in our temporary test location. Some of these are freshly | ||||
inited, some are clones of the others. Our current working directory is | ||||
always inside one of these repositories while the tests are running. | ||||
Hypothesis then performs a series of operations against these repositories, | ||||
including hg commands, generating contents and editing the .hgrc file. | ||||
If these operations fail in unexpected ways or behave differently in | ||||
different configurations of Mercurial, the test will fail and a minimized | ||||
.t test file will be written to the hypothesis-generated directory to | ||||
exhibit that failure. | ||||
Operations are defined as methods with @rule() decorators. See the | ||||
Hypothesis documentation at | ||||
http://hypothesis.readthedocs.org/en/release/stateful.html for more | ||||
details.""" | ||||
# A bundle is a reusable collection of previously generated data which may | ||||
# be provided as arguments to future operations. | ||||
|
r28257 | repos = Bundle('repos') | ||
|
r28255 | paths = Bundle('paths') | ||
contents = Bundle('contents') | ||||
|
r28256 | branches = Bundle('branches') | ||
|
r28255 | committimes = Bundle('committimes') | ||
def __init__(self): | ||||
super(verifyingstatemachine, self).__init__() | ||||
|
r28257 | self.repodir = os.path.join(testtmp, "repos") | ||
|
r28255 | if os.path.exists(self.repodir): | ||
shutil.rmtree(self.repodir) | ||||
os.chdir(testtmp) | ||||
self.log = [] | ||||
self.failed = False | ||||
|
r28257 | self.mkdirp("repos") | ||
self.cd("repos") | ||||
self.mkdirp("repo1") | ||||
self.cd("repo1") | ||||
|
r28255 | self.hg("init") | ||
|
r28258 | self.extensions = {} | ||
self.all_extensions = set() | ||||
self.non_skippable_extensions = set() | ||||
|
r28255 | |||
def teardown(self): | ||||
"""On teardown we clean up after ourselves as usual, but we also | ||||
do some additional testing: We generate a .t file based on our test | ||||
run using run-test.py -i to get the correct output. | ||||
We then test it in a number of other configurations, verifying that | ||||
each passes the same test.""" | ||||
super(verifyingstatemachine, self).teardown() | ||||
try: | ||||
shutil.rmtree(self.repodir) | ||||
except OSError: | ||||
pass | ||||
ttest = os.linesep.join(" " + l for l in self.log) | ||||
os.chdir(testtmp) | ||||
path = os.path.join(testtmp, "test-generated.t") | ||||
with open(path, 'w') as o: | ||||
o.write(ttest + os.linesep) | ||||
with open(os.devnull, "w") as devnull: | ||||
rewriter = subprocess.Popen( | ||||
[runtests, "--local", "-i", path], stdin=subprocess.PIPE, | ||||
stdout=devnull, stderr=devnull, | ||||
) | ||||
rewriter.communicate("yes") | ||||
with open(path, 'r') as i: | ||||
ttest = i.read() | ||||
e = None | ||||
if not self.failed: | ||||
try: | ||||
|
r28258 | for ext in ( | ||
self.all_extensions - self.non_skippable_extensions | ||||
): | ||||
try: | ||||
os.environ["SKIP_EXTENSION"] = ext | ||||
output = subprocess.check_output([ | ||||
runtests, path, "--local", | ||||
], stderr=subprocess.STDOUT) | ||||
assert "Ran 1 test" in output, output | ||||
finally: | ||||
del os.environ["SKIP_EXTENSION"] | ||||
|
r28255 | output = subprocess.check_output([ | ||
runtests, path, "--local", "--pure" | ||||
], stderr=subprocess.STDOUT) | ||||
assert "Ran 1 test" in output, output | ||||
except subprocess.CalledProcessError as e: | ||||
note(e.output) | ||||
finally: | ||||
os.unlink(path) | ||||
try: | ||||
os.unlink(path + ".err") | ||||
except OSError: | ||||
pass | ||||
if self.failed or e is not None: | ||||
with open(savefile, "wb") as o: | ||||
o.write(ttest) | ||||
if e is not None: | ||||
raise e | ||||
def execute_step(self, step): | ||||
try: | ||||
return super(verifyingstatemachine, self).execute_step(step) | ||||
except (HypothesisException, KeyboardInterrupt): | ||||
raise | ||||
except Exception: | ||||
self.failed = True | ||||
raise | ||||
# Section: Basic commands. | ||||
def mkdirp(self, path): | ||||
if os.path.exists(path): | ||||
return | ||||
self.log.append( | ||||
"$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),)) | ||||
os.makedirs(path) | ||||
def cd(self, path): | ||||
path = os.path.relpath(path) | ||||
if path == ".": | ||||
return | ||||
os.chdir(path) | ||||
self.log.append("$ cd -- %s" % (pipes.quote(path),)) | ||||
def hg(self, *args): | ||||
self.command("hg", *args) | ||||
def command(self, *args): | ||||
self.log.append("$ " + ' '.join(map(pipes.quote, args))) | ||||
subprocess.check_output(args, stderr=subprocess.STDOUT) | ||||
# Section: Set up basic data | ||||
# This section has no side effects but generates data that we will want | ||||
# to use later. | ||||
@rule( | ||||
target=paths, | ||||
source=st.lists(files, min_size=1).map(lambda l: os.path.join(*l))) | ||||
def genpath(self, source): | ||||
return source | ||||
@rule( | ||||
target=committimes, | ||||
when=datetimes(min_year=1970, max_year=2038) | st.none()) | ||||
def gentime(self, when): | ||||
return when | ||||
@rule( | ||||
target=contents, | ||||
content=st.one_of( | ||||
st.binary(), | ||||
st.text().map(lambda x: x.encode('utf-8')) | ||||
)) | ||||
def gencontent(self, content): | ||||
return content | ||||
|
r28256 | @rule( | ||
target=branches, | ||||
name=safetext, | ||||
) | ||||
def genbranch(self, name): | ||||
return name | ||||
|
r28255 | @rule(target=paths, source=paths) | ||
def lowerpath(self, source): | ||||
return source.lower() | ||||
@rule(target=paths, source=paths) | ||||
def upperpath(self, source): | ||||
return source.upper() | ||||
# Section: Basic path operations | ||||
@rule(path=paths, content=contents) | ||||
def writecontent(self, path, content): | ||||
self.unadded_changes = True | ||||
if os.path.isdir(path): | ||||
return | ||||
parent = os.path.dirname(path) | ||||
if parent: | ||||
try: | ||||
self.mkdirp(parent) | ||||
except OSError: | ||||
# It may be the case that there is a regular file that has | ||||
# previously been created that has the same name as an ancestor | ||||
# of the current path. This will cause mkdirp to fail with this | ||||
# error. We just turn this into a no-op in that case. | ||||
return | ||||
with open(path, 'wb') as o: | ||||
o.write(content) | ||||
self.log.append(( | ||||
"$ python -c 'import binascii; " | ||||
"print(binascii.unhexlify(\"%s\"))' > %s") % ( | ||||
binascii.hexlify(content), | ||||
pipes.quote(path), | ||||
)) | ||||
@rule(path=paths) | ||||
def addpath(self, path): | ||||
if os.path.exists(path): | ||||
self.hg("add", "--", path) | ||||
@rule(path=paths) | ||||
def forgetpath(self, path): | ||||
if os.path.exists(path): | ||||
with acceptableerrors( | ||||
"file is already untracked", | ||||
): | ||||
self.hg("forget", "--", path) | ||||
@rule(s=st.none() | st.integers(0, 100)) | ||||
def addremove(self, s): | ||||
args = ["addremove"] | ||||
if s is not None: | ||||
args.extend(["-s", str(s)]) | ||||
self.hg(*args) | ||||
@rule(path=paths) | ||||
def removepath(self, path): | ||||
if os.path.exists(path): | ||||
with acceptableerrors( | ||||
'file is untracked', | ||||
'file has been marked for add', | ||||
'file is modified', | ||||
): | ||||
self.hg("remove", "--", path) | ||||
@rule( | ||||
message=safetext, | ||||
amend=st.booleans(), | ||||
when=committimes, | ||||
addremove=st.booleans(), | ||||
secret=st.booleans(), | ||||
close_branch=st.booleans(), | ||||
) | ||||
def maybecommit( | ||||
self, message, amend, when, addremove, secret, close_branch | ||||
): | ||||
command = ["commit"] | ||||
errors = ["nothing changed"] | ||||
if amend: | ||||
errors.append("cannot amend public changesets") | ||||
command.append("--amend") | ||||
command.append("-m" + pipes.quote(message)) | ||||
if secret: | ||||
command.append("--secret") | ||||
if close_branch: | ||||
command.append("--close-branch") | ||||
errors.append("can only close branch heads") | ||||
if addremove: | ||||
command.append("--addremove") | ||||
if when is not None: | ||||
if when.year == 1970: | ||||
errors.append('negative date value') | ||||
if when.year == 2038: | ||||
errors.append('exceeds 32 bits') | ||||
command.append("--date=%s" % ( | ||||
when.strftime('%Y-%m-%d %H:%M:%S %z'),)) | ||||
with acceptableerrors(*errors): | ||||
self.hg(*command) | ||||
|
r28257 | # Section: Repository management | ||
@property | ||||
def currentrepo(self): | ||||
return os.path.basename(os.getcwd()) | ||||
@rule( | ||||
target=repos, | ||||
source=repos, | ||||
name=reponames, | ||||
) | ||||
def clone(self, source, name): | ||||
if not os.path.exists(os.path.join("..", name)): | ||||
self.cd("..") | ||||
self.hg("clone", source, name) | ||||
self.cd(name) | ||||
return name | ||||
@rule( | ||||
target=repos, | ||||
name=reponames, | ||||
) | ||||
def fresh(self, name): | ||||
if not os.path.exists(os.path.join("..", name)): | ||||
self.cd("..") | ||||
self.mkdirp(name) | ||||
self.cd(name) | ||||
self.hg("init") | ||||
return name | ||||
@rule(name=repos) | ||||
def switch(self, name): | ||||
self.cd(os.path.join("..", name)) | ||||
assert self.currentrepo == name | ||||
assert os.path.exists(".hg") | ||||
@rule(target=repos) | ||||
def origin(self): | ||||
return "repo1" | ||||
@rule() | ||||
def pull(self, repo=repos): | ||||
with acceptableerrors( | ||||
"repository default not found", | ||||
"repository is unrelated", | ||||
): | ||||
self.hg("pull") | ||||
@rule(newbranch=st.booleans()) | ||||
def push(self, newbranch): | ||||
with acceptableerrors( | ||||
"default repository not configured", | ||||
"no changes found", | ||||
): | ||||
if newbranch: | ||||
self.hg("push", "--new-branch") | ||||
else: | ||||
with acceptableerrors( | ||||
"creates new branches" | ||||
): | ||||
self.hg("push") | ||||
|
r28255 | # Section: Simple side effect free "check" operations | ||
@rule() | ||||
def log(self): | ||||
self.hg("log") | ||||
@rule() | ||||
def verify(self): | ||||
self.hg("verify") | ||||
@rule() | ||||
def diff(self): | ||||
self.hg("diff", "--nodates") | ||||
@rule() | ||||
def status(self): | ||||
self.hg("status") | ||||
@rule() | ||||
def export(self): | ||||
self.hg("export") | ||||
|
r28256 | # Section: Branch management | ||
@rule() | ||||
def checkbranch(self): | ||||
self.hg("branch") | ||||
@rule(branch=branches) | ||||
def switchbranch(self, branch): | ||||
with acceptableerrors( | ||||
'cannot use an integer as a name', | ||||
'cannot be used in a name', | ||||
'a branch of the same name already exists', | ||||
'is reserved', | ||||
): | ||||
self.hg("branch", "--", branch) | ||||
@rule(branch=branches, clean=st.booleans()) | ||||
def update(self, branch, clean): | ||||
with acceptableerrors( | ||||
'unknown revision', | ||||
'parse error', | ||||
): | ||||
if clean: | ||||
self.hg("update", "-C", "--", branch) | ||||
else: | ||||
self.hg("update", "--", branch) | ||||
|
r28258 | # Section: Extension management | ||
def hasextension(self, extension): | ||||
repo = self.currentrepo | ||||
return repo in self.extensions and extension in self.extensions[repo] | ||||
def commandused(self, extension): | ||||
assert extension in self.all_extensions | ||||
self.non_skippable_extensions.add(extension) | ||||
@rule(extension=st.sampled_from(( | ||||
'shelve', 'mq', 'blackbox', | ||||
))) | ||||
def addextension(self, extension): | ||||
self.all_extensions.add(extension) | ||||
extensions = self.extensions.setdefault(self.currentrepo, set()) | ||||
if extension in extensions: | ||||
return | ||||
extensions.add(extension) | ||||
if not os.path.exists(hgrc): | ||||
self.command("touch", hgrc) | ||||
with open(hgrc, 'a') as o: | ||||
line = "[extensions]\n%s=\n" % (extension,) | ||||
o.write(line) | ||||
for l in line.splitlines(): | ||||
self.log.append(( | ||||
'$ if test "$SKIP_EXTENSION" != "%s" ; ' | ||||
'then echo %r >> %s; fi') % ( | ||||
extension, l, hgrc,)) | ||||
# Section: Commands from the shelve extension | ||||
@rule() | ||||
@precondition(lambda self: self.hasextension("shelve")) | ||||
def shelve(self): | ||||
self.commandused("shelve") | ||||
with acceptableerrors("nothing changed"): | ||||
self.hg("shelve") | ||||
@rule() | ||||
@precondition(lambda self: self.hasextension("shelve")) | ||||
def unshelve(self): | ||||
self.commandused("shelve") | ||||
with acceptableerrors("no shelved changes to apply"): | ||||
self.hg("unshelve") | ||||
|
r28259 | class writeonlydatabase(ExampleDatabase): | ||
def __init__(self, underlying): | ||||
super(ExampleDatabase, self).__init__() | ||||
self.underlying = underlying | ||||
def fetch(self, key): | ||||
return () | ||||
def save(self, key, value): | ||||
self.underlying.save(key, value) | ||||
def delete(self, key, value): | ||||
self.underlying.delete(key, value) | ||||
def close(self): | ||||
self.underlying.close() | ||||
|
r28255 | settings.register_profile( | ||
'default', settings( | ||||
timeout=300, | ||||
stateful_step_count=50, | ||||
max_examples=10, | ||||
) | ||||
) | ||||
settings.register_profile( | ||||
'fast', settings( | ||||
timeout=10, | ||||
stateful_step_count=20, | ||||
max_examples=5, | ||||
min_satisfying_examples=1, | ||||
max_shrinks=0, | ||||
) | ||||
) | ||||
|
r28259 | settings.register_profile( | ||
'continuous', settings( | ||||
timeout=-1, | ||||
stateful_step_count=1000, | ||||
max_examples=10 ** 8, | ||||
max_iterations=10 ** 8, | ||||
database=writeonlydatabase(settings.default.database) | ||||
) | ||||
) | ||||
|
r28255 | settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default')) | ||
verifyingtest = verifyingstatemachine.TestCase | ||||
verifyingtest.settings = settings.default | ||||
if __name__ == '__main__': | ||||
try: | ||||
silenttestrunner.main(__name__) | ||||
finally: | ||||
# So as to prevent proliferation of useless test files, if we never | ||||
# actually wrote a failing test we clean up after ourselves and delete | ||||
# the file for doing so that we owned. | ||||
if os.path.exists(savefile) and os.path.getsize(savefile) == 0: | ||||
os.unlink(savefile) | ||||