Show More
@@ -0,0 +1,404 b'' | |||
|
1 | from __future__ import print_function, absolute_import | |
|
2 | ||
|
3 | """Fuzz testing for operations against a Mercurial repository | |
|
4 | ||
|
5 | This uses Hypothesis's stateful testing to generate random repository | |
|
6 | operations and test Mercurial using them, both to see if there are any | |
|
7 | unexpected errors and to compare different versions of it.""" | |
|
8 | ||
|
9 | import os | |
|
10 | import sys | |
|
11 | ||
|
12 | # These tests require Hypothesis and pytz to be installed. | |
|
13 | # Running 'pip install hypothesis pytz' will achieve that. | |
|
14 | # Note: This won't work if you're running Python < 2.7. | |
|
15 | try: | |
|
16 | from hypothesis.extra.datetime import datetimes | |
|
17 | except ImportError: | |
|
18 | sys.stderr.write("skipped: hypothesis or pytz not installed" + os.linesep) | |
|
19 | sys.exit(80) | |
|
20 | ||
|
21 | # If you are running an old version of pip you may find that the enum34 | |
|
22 | # backport is not installed automatically. If so 'pip install enum34' will | |
|
23 | # fix this problem. | |
|
24 | try: | |
|
25 | import enum | |
|
26 | assert enum # Silence pyflakes | |
|
27 | except ImportError: | |
|
28 | sys.stderr.write("skipped: enum34 not installed" + os.linesep) | |
|
29 | sys.exit(80) | |
|
30 | ||
|
31 | import binascii | |
|
32 | from contextlib import contextmanager | |
|
33 | import errno | |
|
34 | import pipes | |
|
35 | import shutil | |
|
36 | import silenttestrunner | |
|
37 | import subprocess | |
|
38 | ||
|
39 | from hypothesis.errors import HypothesisException | |
|
40 | from hypothesis.stateful import rule, RuleBasedStateMachine, Bundle | |
|
41 | from hypothesis import settings, note, strategies as st | |
|
42 | from hypothesis.configuration import set_hypothesis_home_dir | |
|
43 | ||
|
44 | testdir = os.path.abspath(os.environ["TESTDIR"]) | |
|
45 | ||
|
46 | # We store Hypothesis examples here rather in the temporary test directory | |
|
47 | # so that when rerunning a failing test this always results in refinding the | |
|
48 | # previous failure. This directory is in .hgignore and should not be checked in | |
|
49 | # but is useful to have for development. | |
|
50 | set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis")) | |
|
51 | ||
|
52 | runtests = os.path.join(os.environ["RUNTESTDIR"], "run-tests.py") | |
|
53 | testtmp = os.environ["TESTTMP"] | |
|
54 | assert os.path.isdir(testtmp) | |
|
55 | ||
|
56 | generatedtests = os.path.join(testdir, "hypothesis-generated") | |
|
57 | ||
|
58 | try: | |
|
59 | os.makedirs(generatedtests) | |
|
60 | except OSError: | |
|
61 | pass | |
|
62 | ||
|
63 | # We write out generated .t files to a file in order to ease debugging and to | |
|
64 | # give a starting point for turning failures Hypothesis finds into normal | |
|
65 | # tests. In order to ensure that multiple copies of this test can be run in | |
|
66 | # parallel we use atomic file create to ensure that we always get a unique | |
|
67 | # name. | |
|
68 | file_index = 0 | |
|
69 | while True: | |
|
70 | file_index += 1 | |
|
71 | savefile = os.path.join(generatedtests, "test-generated-%d.t" % ( | |
|
72 | file_index, | |
|
73 | )) | |
|
74 | try: | |
|
75 | os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY)) | |
|
76 | break | |
|
77 | except OSError as e: | |
|
78 | if e.errno != errno.EEXIST: | |
|
79 | raise | |
|
80 | assert os.path.exists(savefile) | |
|
81 | ||
|
82 | hgrc = os.path.join(".hg", "hgrc") | |
|
83 | ||
|
84 | filecharacters = ( | |
|
85 | "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" | |
|
86 | "[]^_`;=@{}~ !#$%&'()+,-" | |
|
87 | ) | |
|
88 | ||
|
89 | files = st.text(filecharacters, min_size=1).map(lambda x: x.strip()).filter( | |
|
90 | bool).map(lambda s: s.encode('ascii')) | |
|
91 | ||
|
92 | safetext = st.text(st.characters( | |
|
93 | min_codepoint=1, max_codepoint=127, | |
|
94 | blacklist_categories=('Cc', 'Cs')), min_size=1).map( | |
|
95 | lambda s: s.encode('utf-8') | |
|
96 | ) | |
|
97 | ||
|
98 | @contextmanager | |
|
99 | def acceptableerrors(*args): | |
|
100 | """Sometimes we know an operation we're about to perform might fail, and | |
|
101 | we're OK with some of the failures. In those cases this may be used as a | |
|
102 | context manager and will swallow expected failures, as identified by | |
|
103 | substrings of the error message Mercurial emits.""" | |
|
104 | try: | |
|
105 | yield | |
|
106 | except subprocess.CalledProcessError as e: | |
|
107 | if not any(a in e.output for a in args): | |
|
108 | note(e.output) | |
|
109 | raise | |
|
110 | ||
|
111 | class verifyingstatemachine(RuleBasedStateMachine): | |
|
112 | """This defines the set of acceptable operations on a Mercurial repository | |
|
113 | using Hypothesis's RuleBasedStateMachine. | |
|
114 | ||
|
115 | The general concept is that we manage multiple repositories inside a | |
|
116 | repos/ directory in our temporary test location. Some of these are freshly | |
|
117 | inited, some are clones of the others. Our current working directory is | |
|
118 | always inside one of these repositories while the tests are running. | |
|
119 | ||
|
120 | Hypothesis then performs a series of operations against these repositories, | |
|
121 | including hg commands, generating contents and editing the .hgrc file. | |
|
122 | If these operations fail in unexpected ways or behave differently in | |
|
123 | different configurations of Mercurial, the test will fail and a minimized | |
|
124 | .t test file will be written to the hypothesis-generated directory to | |
|
125 | exhibit that failure. | |
|
126 | ||
|
127 | Operations are defined as methods with @rule() decorators. See the | |
|
128 | Hypothesis documentation at | |
|
129 | http://hypothesis.readthedocs.org/en/release/stateful.html for more | |
|
130 | details.""" | |
|
131 | ||
|
132 | # A bundle is a reusable collection of previously generated data which may | |
|
133 | # be provided as arguments to future operations. | |
|
134 | paths = Bundle('paths') | |
|
135 | contents = Bundle('contents') | |
|
136 | committimes = Bundle('committimes') | |
|
137 | ||
|
138 | def __init__(self): | |
|
139 | super(verifyingstatemachine, self).__init__() | |
|
140 | self.repodir = os.path.join(testtmp, "repo") | |
|
141 | if os.path.exists(self.repodir): | |
|
142 | shutil.rmtree(self.repodir) | |
|
143 | os.chdir(testtmp) | |
|
144 | self.log = [] | |
|
145 | self.failed = False | |
|
146 | ||
|
147 | self.mkdirp("repo") | |
|
148 | self.cd("repo") | |
|
149 | self.hg("init") | |
|
150 | ||
|
151 | def teardown(self): | |
|
152 | """On teardown we clean up after ourselves as usual, but we also | |
|
153 | do some additional testing: We generate a .t file based on our test | |
|
154 | run using run-test.py -i to get the correct output. | |
|
155 | ||
|
156 | We then test it in a number of other configurations, verifying that | |
|
157 | each passes the same test.""" | |
|
158 | super(verifyingstatemachine, self).teardown() | |
|
159 | try: | |
|
160 | shutil.rmtree(self.repodir) | |
|
161 | except OSError: | |
|
162 | pass | |
|
163 | ttest = os.linesep.join(" " + l for l in self.log) | |
|
164 | os.chdir(testtmp) | |
|
165 | path = os.path.join(testtmp, "test-generated.t") | |
|
166 | with open(path, 'w') as o: | |
|
167 | o.write(ttest + os.linesep) | |
|
168 | with open(os.devnull, "w") as devnull: | |
|
169 | rewriter = subprocess.Popen( | |
|
170 | [runtests, "--local", "-i", path], stdin=subprocess.PIPE, | |
|
171 | stdout=devnull, stderr=devnull, | |
|
172 | ) | |
|
173 | rewriter.communicate("yes") | |
|
174 | with open(path, 'r') as i: | |
|
175 | ttest = i.read() | |
|
176 | ||
|
177 | e = None | |
|
178 | if not self.failed: | |
|
179 | try: | |
|
180 | output = subprocess.check_output([ | |
|
181 | runtests, path, "--local", "--pure" | |
|
182 | ], stderr=subprocess.STDOUT) | |
|
183 | assert "Ran 1 test" in output, output | |
|
184 | except subprocess.CalledProcessError as e: | |
|
185 | note(e.output) | |
|
186 | finally: | |
|
187 | os.unlink(path) | |
|
188 | try: | |
|
189 | os.unlink(path + ".err") | |
|
190 | except OSError: | |
|
191 | pass | |
|
192 | if self.failed or e is not None: | |
|
193 | with open(savefile, "wb") as o: | |
|
194 | o.write(ttest) | |
|
195 | if e is not None: | |
|
196 | raise e | |
|
197 | ||
|
198 | def execute_step(self, step): | |
|
199 | try: | |
|
200 | return super(verifyingstatemachine, self).execute_step(step) | |
|
201 | except (HypothesisException, KeyboardInterrupt): | |
|
202 | raise | |
|
203 | except Exception: | |
|
204 | self.failed = True | |
|
205 | raise | |
|
206 | ||
|
207 | # Section: Basic commands. | |
|
208 | def mkdirp(self, path): | |
|
209 | if os.path.exists(path): | |
|
210 | return | |
|
211 | self.log.append( | |
|
212 | "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),)) | |
|
213 | os.makedirs(path) | |
|
214 | ||
|
215 | def cd(self, path): | |
|
216 | path = os.path.relpath(path) | |
|
217 | if path == ".": | |
|
218 | return | |
|
219 | os.chdir(path) | |
|
220 | self.log.append("$ cd -- %s" % (pipes.quote(path),)) | |
|
221 | ||
|
222 | def hg(self, *args): | |
|
223 | self.command("hg", *args) | |
|
224 | ||
|
225 | def command(self, *args): | |
|
226 | self.log.append("$ " + ' '.join(map(pipes.quote, args))) | |
|
227 | subprocess.check_output(args, stderr=subprocess.STDOUT) | |
|
228 | ||
|
229 | # Section: Set up basic data | |
|
230 | # This section has no side effects but generates data that we will want | |
|
231 | # to use later. | |
|
232 | @rule( | |
|
233 | target=paths, | |
|
234 | source=st.lists(files, min_size=1).map(lambda l: os.path.join(*l))) | |
|
235 | def genpath(self, source): | |
|
236 | return source | |
|
237 | ||
|
238 | @rule( | |
|
239 | target=committimes, | |
|
240 | when=datetimes(min_year=1970, max_year=2038) | st.none()) | |
|
241 | def gentime(self, when): | |
|
242 | return when | |
|
243 | ||
|
244 | @rule( | |
|
245 | target=contents, | |
|
246 | content=st.one_of( | |
|
247 | st.binary(), | |
|
248 | st.text().map(lambda x: x.encode('utf-8')) | |
|
249 | )) | |
|
250 | def gencontent(self, content): | |
|
251 | return content | |
|
252 | ||
|
253 | @rule(target=paths, source=paths) | |
|
254 | def lowerpath(self, source): | |
|
255 | return source.lower() | |
|
256 | ||
|
257 | @rule(target=paths, source=paths) | |
|
258 | def upperpath(self, source): | |
|
259 | return source.upper() | |
|
260 | ||
|
261 | # Section: Basic path operations | |
|
262 | @rule(path=paths, content=contents) | |
|
263 | def writecontent(self, path, content): | |
|
264 | self.unadded_changes = True | |
|
265 | if os.path.isdir(path): | |
|
266 | return | |
|
267 | parent = os.path.dirname(path) | |
|
268 | if parent: | |
|
269 | try: | |
|
270 | self.mkdirp(parent) | |
|
271 | except OSError: | |
|
272 | # It may be the case that there is a regular file that has | |
|
273 | # previously been created that has the same name as an ancestor | |
|
274 | # of the current path. This will cause mkdirp to fail with this | |
|
275 | # error. We just turn this into a no-op in that case. | |
|
276 | return | |
|
277 | with open(path, 'wb') as o: | |
|
278 | o.write(content) | |
|
279 | self.log.append(( | |
|
280 | "$ python -c 'import binascii; " | |
|
281 | "print(binascii.unhexlify(\"%s\"))' > %s") % ( | |
|
282 | binascii.hexlify(content), | |
|
283 | pipes.quote(path), | |
|
284 | )) | |
|
285 | ||
|
286 | @rule(path=paths) | |
|
287 | def addpath(self, path): | |
|
288 | if os.path.exists(path): | |
|
289 | self.hg("add", "--", path) | |
|
290 | ||
|
291 | @rule(path=paths) | |
|
292 | def forgetpath(self, path): | |
|
293 | if os.path.exists(path): | |
|
294 | with acceptableerrors( | |
|
295 | "file is already untracked", | |
|
296 | ): | |
|
297 | self.hg("forget", "--", path) | |
|
298 | ||
|
299 | @rule(s=st.none() | st.integers(0, 100)) | |
|
300 | def addremove(self, s): | |
|
301 | args = ["addremove"] | |
|
302 | if s is not None: | |
|
303 | args.extend(["-s", str(s)]) | |
|
304 | self.hg(*args) | |
|
305 | ||
|
306 | @rule(path=paths) | |
|
307 | def removepath(self, path): | |
|
308 | if os.path.exists(path): | |
|
309 | with acceptableerrors( | |
|
310 | 'file is untracked', | |
|
311 | 'file has been marked for add', | |
|
312 | 'file is modified', | |
|
313 | ): | |
|
314 | self.hg("remove", "--", path) | |
|
315 | ||
|
316 | @rule( | |
|
317 | message=safetext, | |
|
318 | amend=st.booleans(), | |
|
319 | when=committimes, | |
|
320 | addremove=st.booleans(), | |
|
321 | secret=st.booleans(), | |
|
322 | close_branch=st.booleans(), | |
|
323 | ) | |
|
324 | def maybecommit( | |
|
325 | self, message, amend, when, addremove, secret, close_branch | |
|
326 | ): | |
|
327 | command = ["commit"] | |
|
328 | errors = ["nothing changed"] | |
|
329 | if amend: | |
|
330 | errors.append("cannot amend public changesets") | |
|
331 | command.append("--amend") | |
|
332 | command.append("-m" + pipes.quote(message)) | |
|
333 | if secret: | |
|
334 | command.append("--secret") | |
|
335 | if close_branch: | |
|
336 | command.append("--close-branch") | |
|
337 | errors.append("can only close branch heads") | |
|
338 | if addremove: | |
|
339 | command.append("--addremove") | |
|
340 | if when is not None: | |
|
341 | if when.year == 1970: | |
|
342 | errors.append('negative date value') | |
|
343 | if when.year == 2038: | |
|
344 | errors.append('exceeds 32 bits') | |
|
345 | command.append("--date=%s" % ( | |
|
346 | when.strftime('%Y-%m-%d %H:%M:%S %z'),)) | |
|
347 | ||
|
348 | with acceptableerrors(*errors): | |
|
349 | self.hg(*command) | |
|
350 | ||
|
351 | # Section: Simple side effect free "check" operations | |
|
352 | @rule() | |
|
353 | def log(self): | |
|
354 | self.hg("log") | |
|
355 | ||
|
356 | @rule() | |
|
357 | def verify(self): | |
|
358 | self.hg("verify") | |
|
359 | ||
|
360 | @rule() | |
|
361 | def diff(self): | |
|
362 | self.hg("diff", "--nodates") | |
|
363 | ||
|
364 | @rule() | |
|
365 | def status(self): | |
|
366 | self.hg("status") | |
|
367 | ||
|
368 | @rule() | |
|
369 | def export(self): | |
|
370 | self.hg("export") | |
|
371 | ||
|
372 | settings.register_profile( | |
|
373 | 'default', settings( | |
|
374 | timeout=300, | |
|
375 | stateful_step_count=50, | |
|
376 | max_examples=10, | |
|
377 | ) | |
|
378 | ) | |
|
379 | ||
|
380 | settings.register_profile( | |
|
381 | 'fast', settings( | |
|
382 | timeout=10, | |
|
383 | stateful_step_count=20, | |
|
384 | max_examples=5, | |
|
385 | min_satisfying_examples=1, | |
|
386 | max_shrinks=0, | |
|
387 | ) | |
|
388 | ) | |
|
389 | ||
|
390 | settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default')) | |
|
391 | ||
|
392 | verifyingtest = verifyingstatemachine.TestCase | |
|
393 | ||
|
394 | verifyingtest.settings = settings.default | |
|
395 | ||
|
396 | if __name__ == '__main__': | |
|
397 | try: | |
|
398 | silenttestrunner.main(__name__) | |
|
399 | finally: | |
|
400 | # So as to prevent proliferation of useless test files, if we never | |
|
401 | # actually wrote a failing test we clean up after ourselves and delete | |
|
402 | # the file for doing so that we owned. | |
|
403 | if os.path.exists(savefile) and os.path.getsize(savefile) == 0: | |
|
404 | os.unlink(savefile) |
General Comments 0
You need to be logged in to leave comments.
Login now