test-verify-repo-operations.py
640 lines
| 18.9 KiB
| text/x-python
|
PythonLexer
/ tests / test-verify-repo-operations.py
|
r28255 | from __future__ import print_function, absolute_import | ||
"""Fuzz testing for operations against a Mercurial repository | ||||
This uses Hypothesis's stateful testing to generate random repository | ||||
operations and test Mercurial using them, both to see if there are any | ||||
unexpected errors and to compare different versions of it.""" | ||||
import os | ||||
|
r28499 | import subprocess | ||
|
r28255 | import sys | ||
|
r28499 | # Only run if slow tests are allowed | ||
|
r43346 | if subprocess.call(['python', '%s/hghave' % os.environ['TESTDIR'], 'slow']): | ||
|
r28499 | sys.exit(80) | ||
|
r28255 | # These tests require Hypothesis and pytz to be installed. | ||
# Running 'pip install hypothesis pytz' will achieve that. | ||||
# Note: This won't work if you're running Python < 2.7. | ||||
try: | ||||
from hypothesis.extra.datetime import datetimes | ||||
except ImportError: | ||||
sys.stderr.write("skipped: hypothesis or pytz not installed" + os.linesep) | ||||
sys.exit(80) | ||||
# If you are running an old version of pip you may find that the enum34 | ||||
# backport is not installed automatically. If so 'pip install enum34' will | ||||
# fix this problem. | ||||
try: | ||||
import enum | ||||
|
r43346 | |||
|
r28255 | assert enum # Silence pyflakes | ||
except ImportError: | ||||
sys.stderr.write("skipped: enum34 not installed" + os.linesep) | ||||
sys.exit(80) | ||||
import binascii | ||||
from contextlib import contextmanager | ||||
import errno | ||||
import pipes | ||||
import shutil | ||||
import silenttestrunner | ||||
import subprocess | ||||
from hypothesis.errors import HypothesisException | ||||
|
r28258 | from hypothesis.stateful import ( | ||
|
r43346 | rule, | ||
RuleBasedStateMachine, | ||||
Bundle, | ||||
precondition, | ||||
) | ||||
|
r28255 | from hypothesis import settings, note, strategies as st | ||
from hypothesis.configuration import set_hypothesis_home_dir | ||||
|
r28259 | from hypothesis.database import ExampleDatabase | ||
|
r28255 | |||
testdir = os.path.abspath(os.environ["TESTDIR"]) | ||||
# We store Hypothesis examples here rather in the temporary test directory | ||||
# so that when rerunning a failing test this always results in refinding the | ||||
# previous failure. This directory is in .hgignore and should not be checked in | ||||
# but is useful to have for development. | ||||
set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis")) | ||||
runtests = os.path.join(os.environ["RUNTESTDIR"], "run-tests.py") | ||||
testtmp = os.environ["TESTTMP"] | ||||
assert os.path.isdir(testtmp) | ||||
generatedtests = os.path.join(testdir, "hypothesis-generated") | ||||
try: | ||||
os.makedirs(generatedtests) | ||||
except OSError: | ||||
pass | ||||
# We write out generated .t files to a file in order to ease debugging and to | ||||
# give a starting point for turning failures Hypothesis finds into normal | ||||
# tests. In order to ensure that multiple copies of this test can be run in | ||||
# parallel we use atomic file create to ensure that we always get a unique | ||||
# name. | ||||
file_index = 0 | ||||
while True: | ||||
file_index += 1 | ||||
|
r43346 | savefile = os.path.join( | ||
generatedtests, "test-generated-%d.t" % (file_index,) | ||||
) | ||||
|
r28255 | try: | ||
os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY)) | ||||
break | ||||
except OSError as e: | ||||
if e.errno != errno.EEXIST: | ||||
raise | ||||
assert os.path.exists(savefile) | ||||
hgrc = os.path.join(".hg", "hgrc") | ||||
filecharacters = ( | ||||
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" | ||||
"[]^_`;=@{}~ !#$%&'()+,-" | ||||
) | ||||
|
r43346 | files = ( | ||
st.text(filecharacters, min_size=1) | ||||
.map(lambda x: x.strip()) | ||||
.filter(bool) | ||||
.map(lambda s: s.encode('ascii')) | ||||
|
r28255 | ) | ||
|
r43346 | safetext = st.text( | ||
st.characters( | ||||
min_codepoint=1, max_codepoint=127, blacklist_categories=('Cc', 'Cs') | ||||
), | ||||
min_size=1, | ||||
).map(lambda s: s.encode('utf-8')) | ||||
|
r46554 | extensions = st.sampled_from( | ||
( | ||||
'shelve', | ||||
'mq', | ||||
'blackbox', | ||||
) | ||||
) | ||||
|
r28279 | |||
|
r43346 | |||
|
r28255 | @contextmanager | ||
def acceptableerrors(*args): | ||||
"""Sometimes we know an operation we're about to perform might fail, and | ||||
we're OK with some of the failures. In those cases this may be used as a | ||||
context manager and will swallow expected failures, as identified by | ||||
substrings of the error message Mercurial emits.""" | ||||
try: | ||||
yield | ||||
except subprocess.CalledProcessError as e: | ||||
if not any(a in e.output for a in args): | ||||
note(e.output) | ||||
raise | ||||
|
r43346 | |||
|
r28257 | reponames = st.text("abcdefghijklmnopqrstuvwxyz01234556789", min_size=1).map( | ||
lambda s: s.encode('ascii') | ||||
) | ||||
|
r43346 | |||
|
r28255 | class verifyingstatemachine(RuleBasedStateMachine): | ||
"""This defines the set of acceptable operations on a Mercurial repository | ||||
using Hypothesis's RuleBasedStateMachine. | ||||
The general concept is that we manage multiple repositories inside a | ||||
repos/ directory in our temporary test location. Some of these are freshly | ||||
inited, some are clones of the others. Our current working directory is | ||||
always inside one of these repositories while the tests are running. | ||||
Hypothesis then performs a series of operations against these repositories, | ||||
including hg commands, generating contents and editing the .hgrc file. | ||||
If these operations fail in unexpected ways or behave differently in | ||||
different configurations of Mercurial, the test will fail and a minimized | ||||
.t test file will be written to the hypothesis-generated directory to | ||||
exhibit that failure. | ||||
Operations are defined as methods with @rule() decorators. See the | ||||
Hypothesis documentation at | ||||
http://hypothesis.readthedocs.org/en/release/stateful.html for more | ||||
details.""" | ||||
# A bundle is a reusable collection of previously generated data which may | ||||
# be provided as arguments to future operations. | ||||
|
r28257 | repos = Bundle('repos') | ||
|
r28255 | paths = Bundle('paths') | ||
contents = Bundle('contents') | ||||
|
r28256 | branches = Bundle('branches') | ||
|
r28255 | committimes = Bundle('committimes') | ||
def __init__(self): | ||||
super(verifyingstatemachine, self).__init__() | ||||
|
r28257 | self.repodir = os.path.join(testtmp, "repos") | ||
|
r28255 | if os.path.exists(self.repodir): | ||
shutil.rmtree(self.repodir) | ||||
os.chdir(testtmp) | ||||
self.log = [] | ||||
self.failed = False | ||||
|
r28279 | self.configperrepo = {} | ||
self.all_extensions = set() | ||||
self.non_skippable_extensions = set() | ||||
|
r28255 | |||
|
r28257 | self.mkdirp("repos") | ||
self.cd("repos") | ||||
self.mkdirp("repo1") | ||||
self.cd("repo1") | ||||
|
r28255 | self.hg("init") | ||
def teardown(self): | ||||
"""On teardown we clean up after ourselves as usual, but we also | ||||
do some additional testing: We generate a .t file based on our test | ||||
run using run-test.py -i to get the correct output. | ||||
We then test it in a number of other configurations, verifying that | ||||
each passes the same test.""" | ||||
super(verifyingstatemachine, self).teardown() | ||||
try: | ||||
shutil.rmtree(self.repodir) | ||||
except OSError: | ||||
pass | ||||
ttest = os.linesep.join(" " + l for l in self.log) | ||||
os.chdir(testtmp) | ||||
path = os.path.join(testtmp, "test-generated.t") | ||||
with open(path, 'w') as o: | ||||
o.write(ttest + os.linesep) | ||||
with open(os.devnull, "w") as devnull: | ||||
rewriter = subprocess.Popen( | ||||
|
r43346 | [runtests, "--local", "-i", path], | ||
stdin=subprocess.PIPE, | ||||
stdout=devnull, | ||||
stderr=devnull, | ||||
|
r28255 | ) | ||
rewriter.communicate("yes") | ||||
with open(path, 'r') as i: | ||||
ttest = i.read() | ||||
e = None | ||||
if not self.failed: | ||||
try: | ||||
|
r43346 | output = subprocess.check_output( | ||
[runtests, path, "--local", "--pure"], | ||||
stderr=subprocess.STDOUT, | ||||
) | ||||
|
r28255 | assert "Ran 1 test" in output, output | ||
|
r43346 | for ext in self.all_extensions - self.non_skippable_extensions: | ||
tf = os.path.join( | ||||
testtmp, "test-generated-no-%s.t" % (ext,) | ||||
) | ||||
|
r28279 | with open(tf, 'w') as o: | ||
for l in ttest.splitlines(): | ||||
if l.startswith(" $ hg"): | ||||
l = l.replace( | ||||
|
r43346 | "--config %s=" % (extensionconfigkey(ext),), | ||
"", | ||||
) | ||||
|
r28279 | o.write(l + os.linesep) | ||
with open(tf, 'r') as r: | ||||
t = r.read() | ||||
assert ext not in t, t | ||||
|
r43346 | output = subprocess.check_output( | ||
|
r46554 | [ | ||
runtests, | ||||
tf, | ||||
"--local", | ||||
], | ||||
stderr=subprocess.STDOUT, | ||||
|
r43346 | ) | ||
|
r28279 | assert "Ran 1 test" in output, output | ||
|
r28255 | except subprocess.CalledProcessError as e: | ||
note(e.output) | ||||
if self.failed or e is not None: | ||||
with open(savefile, "wb") as o: | ||||
o.write(ttest) | ||||
if e is not None: | ||||
raise e | ||||
def execute_step(self, step): | ||||
try: | ||||
return super(verifyingstatemachine, self).execute_step(step) | ||||
except (HypothesisException, KeyboardInterrupt): | ||||
raise | ||||
except Exception: | ||||
self.failed = True | ||||
raise | ||||
# Section: Basic commands. | ||||
def mkdirp(self, path): | ||||
if os.path.exists(path): | ||||
return | ||||
self.log.append( | ||||
|
r43346 | "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),) | ||
) | ||||
|
r28255 | os.makedirs(path) | ||
def cd(self, path): | ||||
path = os.path.relpath(path) | ||||
if path == ".": | ||||
return | ||||
os.chdir(path) | ||||
self.log.append("$ cd -- %s" % (pipes.quote(path),)) | ||||
def hg(self, *args): | ||||
|
r28279 | extra_flags = [] | ||
for key, value in self.config.items(): | ||||
extra_flags.append("--config") | ||||
extra_flags.append("%s=%s" % (key, value)) | ||||
self.command("hg", *(tuple(extra_flags) + args)) | ||||
|
r28255 | |||
def command(self, *args): | ||||
self.log.append("$ " + ' '.join(map(pipes.quote, args))) | ||||
subprocess.check_output(args, stderr=subprocess.STDOUT) | ||||
# Section: Set up basic data | ||||
# This section has no side effects but generates data that we will want | ||||
# to use later. | ||||
@rule( | ||||
target=paths, | ||||
|
r43346 | source=st.lists(files, min_size=1).map(lambda l: os.path.join(*l)), | ||
) | ||||
|
r28255 | def genpath(self, source): | ||
return source | ||||
@rule( | ||||
target=committimes, | ||||
|
r43346 | when=datetimes(min_year=1970, max_year=2038) | st.none(), | ||
) | ||||
|
r28255 | def gentime(self, when): | ||
return when | ||||
@rule( | ||||
target=contents, | ||||
content=st.one_of( | ||||
|
r43346 | st.binary(), st.text().map(lambda x: x.encode('utf-8')) | ||
), | ||||
) | ||||
|
r28255 | def gencontent(self, content): | ||
return content | ||||
|
r28256 | @rule( | ||
|
r46554 | target=branches, | ||
name=safetext, | ||||
|
r28256 | ) | ||
def genbranch(self, name): | ||||
return name | ||||
|
r28255 | @rule(target=paths, source=paths) | ||
def lowerpath(self, source): | ||||
return source.lower() | ||||
@rule(target=paths, source=paths) | ||||
def upperpath(self, source): | ||||
return source.upper() | ||||
# Section: Basic path operations | ||||
@rule(path=paths, content=contents) | ||||
def writecontent(self, path, content): | ||||
self.unadded_changes = True | ||||
if os.path.isdir(path): | ||||
return | ||||
parent = os.path.dirname(path) | ||||
if parent: | ||||
try: | ||||
self.mkdirp(parent) | ||||
except OSError: | ||||
# It may be the case that there is a regular file that has | ||||
# previously been created that has the same name as an ancestor | ||||
# of the current path. This will cause mkdirp to fail with this | ||||
# error. We just turn this into a no-op in that case. | ||||
return | ||||
with open(path, 'wb') as o: | ||||
o.write(content) | ||||
|
r43346 | self.log.append( | ||
( | ||||
"$ python -c 'import binascii; " | ||||
"print(binascii.unhexlify(\"%s\"))' > %s" | ||||
) | ||||
|
r46554 | % ( | ||
binascii.hexlify(content), | ||||
pipes.quote(path), | ||||
) | ||||
|
r43346 | ) | ||
|
r28255 | |||
@rule(path=paths) | ||||
def addpath(self, path): | ||||
if os.path.exists(path): | ||||
self.hg("add", "--", path) | ||||
@rule(path=paths) | ||||
def forgetpath(self, path): | ||||
if os.path.exists(path): | ||||
|
r46554 | with acceptableerrors( | ||
"file is already untracked", | ||||
): | ||||
|
r28255 | self.hg("forget", "--", path) | ||
@rule(s=st.none() | st.integers(0, 100)) | ||||
def addremove(self, s): | ||||
args = ["addremove"] | ||||
if s is not None: | ||||
args.extend(["-s", str(s)]) | ||||
self.hg(*args) | ||||
@rule(path=paths) | ||||
def removepath(self, path): | ||||
if os.path.exists(path): | ||||
with acceptableerrors( | ||||
'file is untracked', | ||||
'file has been marked for add', | ||||
'file is modified', | ||||
): | ||||
self.hg("remove", "--", path) | ||||
@rule( | ||||
message=safetext, | ||||
amend=st.booleans(), | ||||
when=committimes, | ||||
addremove=st.booleans(), | ||||
secret=st.booleans(), | ||||
close_branch=st.booleans(), | ||||
) | ||||
def maybecommit( | ||||
self, message, amend, when, addremove, secret, close_branch | ||||
): | ||||
command = ["commit"] | ||||
errors = ["nothing changed"] | ||||
if amend: | ||||
errors.append("cannot amend public changesets") | ||||
command.append("--amend") | ||||
command.append("-m" + pipes.quote(message)) | ||||
if secret: | ||||
command.append("--secret") | ||||
if close_branch: | ||||
command.append("--close-branch") | ||||
errors.append("can only close branch heads") | ||||
if addremove: | ||||
command.append("--addremove") | ||||
if when is not None: | ||||
if when.year == 1970: | ||||
errors.append('negative date value') | ||||
if when.year == 2038: | ||||
errors.append('exceeds 32 bits') | ||||
|
r43346 | command.append( | ||
"--date=%s" % (when.strftime('%Y-%m-%d %H:%M:%S %z'),) | ||||
) | ||||
|
r28255 | |||
with acceptableerrors(*errors): | ||||
self.hg(*command) | ||||
|
r28257 | # Section: Repository management | ||
@property | ||||
def currentrepo(self): | ||||
return os.path.basename(os.getcwd()) | ||||
|
r28279 | @property | ||
def config(self): | ||||
return self.configperrepo.setdefault(self.currentrepo, {}) | ||||
|
r28257 | @rule( | ||
|
r46554 | target=repos, | ||
source=repos, | ||||
name=reponames, | ||||
|
r28257 | ) | ||
def clone(self, source, name): | ||||
if not os.path.exists(os.path.join("..", name)): | ||||
self.cd("..") | ||||
self.hg("clone", source, name) | ||||
self.cd(name) | ||||
return name | ||||
@rule( | ||||
|
r46554 | target=repos, | ||
name=reponames, | ||||
|
r28257 | ) | ||
def fresh(self, name): | ||||
if not os.path.exists(os.path.join("..", name)): | ||||
self.cd("..") | ||||
self.mkdirp(name) | ||||
self.cd(name) | ||||
self.hg("init") | ||||
return name | ||||
@rule(name=repos) | ||||
def switch(self, name): | ||||
self.cd(os.path.join("..", name)) | ||||
assert self.currentrepo == name | ||||
assert os.path.exists(".hg") | ||||
@rule(target=repos) | ||||
def origin(self): | ||||
return "repo1" | ||||
@rule() | ||||
def pull(self, repo=repos): | ||||
with acceptableerrors( | ||||
|
r46554 | "repository default not found", | ||
"repository is unrelated", | ||||
|
r28257 | ): | ||
self.hg("pull") | ||||
@rule(newbranch=st.booleans()) | ||||
def push(self, newbranch): | ||||
with acceptableerrors( | ||||
|
r46554 | "default repository not configured", | ||
"no changes found", | ||||
|
r28257 | ): | ||
if newbranch: | ||||
self.hg("push", "--new-branch") | ||||
else: | ||||
|
r43346 | with acceptableerrors("creates new branches"): | ||
|
r28257 | self.hg("push") | ||
|
r28255 | # Section: Simple side effect free "check" operations | ||
@rule() | ||||
def log(self): | ||||
self.hg("log") | ||||
@rule() | ||||
def verify(self): | ||||
self.hg("verify") | ||||
@rule() | ||||
def diff(self): | ||||
self.hg("diff", "--nodates") | ||||
@rule() | ||||
def status(self): | ||||
self.hg("status") | ||||
@rule() | ||||
def export(self): | ||||
self.hg("export") | ||||
|
r28256 | # Section: Branch management | ||
@rule() | ||||
def checkbranch(self): | ||||
self.hg("branch") | ||||
@rule(branch=branches) | ||||
def switchbranch(self, branch): | ||||
with acceptableerrors( | ||||
'cannot use an integer as a name', | ||||
'cannot be used in a name', | ||||
'a branch of the same name already exists', | ||||
'is reserved', | ||||
): | ||||
self.hg("branch", "--", branch) | ||||
@rule(branch=branches, clean=st.booleans()) | ||||
def update(self, branch, clean): | ||||
with acceptableerrors( | ||||
|
r46554 | 'unknown revision', | ||
'parse error', | ||||
|
r28256 | ): | ||
if clean: | ||||
self.hg("update", "-C", "--", branch) | ||||
else: | ||||
self.hg("update", "--", branch) | ||||
|
r28258 | # Section: Extension management | ||
def hasextension(self, extension): | ||||
|
r28279 | return extensionconfigkey(extension) in self.config | ||
|
r28258 | |||
def commandused(self, extension): | ||||
assert extension in self.all_extensions | ||||
self.non_skippable_extensions.add(extension) | ||||
|
r28279 | @rule(extension=extensions) | ||
|
r28258 | def addextension(self, extension): | ||
self.all_extensions.add(extension) | ||||
|
r28279 | self.config[extensionconfigkey(extension)] = "" | ||
@rule(extension=extensions) | ||||
def removeextension(self, extension): | ||||
self.config.pop(extensionconfigkey(extension), None) | ||||
|
r28258 | |||
# Section: Commands from the shelve extension | ||||
@rule() | ||||
@precondition(lambda self: self.hasextension("shelve")) | ||||
def shelve(self): | ||||
self.commandused("shelve") | ||||
with acceptableerrors("nothing changed"): | ||||
self.hg("shelve") | ||||
@rule() | ||||
@precondition(lambda self: self.hasextension("shelve")) | ||||
def unshelve(self): | ||||
self.commandused("shelve") | ||||
with acceptableerrors("no shelved changes to apply"): | ||||
self.hg("unshelve") | ||||
|
r43346 | |||
|
r28259 | class writeonlydatabase(ExampleDatabase): | ||
def __init__(self, underlying): | ||||
super(ExampleDatabase, self).__init__() | ||||
self.underlying = underlying | ||||
def fetch(self, key): | ||||
return () | ||||
def save(self, key, value): | ||||
self.underlying.save(key, value) | ||||
def delete(self, key, value): | ||||
self.underlying.delete(key, value) | ||||
def close(self): | ||||
self.underlying.close() | ||||
|
r43346 | |||
|
r28279 | def extensionconfigkey(extension): | ||
return "extensions." + extension | ||||
|
r43346 | |||
|
r28255 | settings.register_profile( | ||
|
r46554 | 'default', | ||
settings( | ||||
timeout=300, | ||||
stateful_step_count=50, | ||||
max_examples=10, | ||||
), | ||||
|
r28255 | ) | ||
settings.register_profile( | ||||
|
r43346 | 'fast', | ||
settings( | ||||
|
r28255 | timeout=10, | ||
stateful_step_count=20, | ||||
max_examples=5, | ||||
min_satisfying_examples=1, | ||||
max_shrinks=0, | ||||
|
r43346 | ), | ||
|
r28255 | ) | ||
|
r28259 | settings.register_profile( | ||
|
r43346 | 'continuous', | ||
settings( | ||||
|
r28259 | timeout=-1, | ||
stateful_step_count=1000, | ||||
max_examples=10 ** 8, | ||||
max_iterations=10 ** 8, | ||||
|
r43346 | database=writeonlydatabase(settings.default.database), | ||
), | ||||
|
r28259 | ) | ||
|
r28255 | settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default')) | ||
verifyingtest = verifyingstatemachine.TestCase | ||||
verifyingtest.settings = settings.default | ||||
if __name__ == '__main__': | ||||
try: | ||||
silenttestrunner.main(__name__) | ||||
finally: | ||||
# So as to prevent proliferation of useless test files, if we never | ||||
# actually wrote a failing test we clean up after ourselves and delete | ||||
# the file for doing so that we owned. | ||||
if os.path.exists(savefile) and os.path.getsize(savefile) == 0: | ||||
os.unlink(savefile) | ||||