test-revlog-raw.py
504 lines
| 16.0 KiB
| text/x-python
|
PythonLexer
/ tests / test-revlog-raw.py
|
r31748 | # test revlog interaction about raw data (flagprocessor) | ||
from __future__ import absolute_import, print_function | ||||
|
r41119 | import collections | ||
|
r41064 | import hashlib | ||
|
r31748 | import sys | ||
|
r46729 | from mercurial.node import nullid | ||
|
r31748 | from mercurial import ( | ||
encoding, | ||||
revlog, | ||||
transaction, | ||||
vfs, | ||||
) | ||||
|
r41063 | from mercurial.revlogutils import ( | ||
deltas, | ||||
r42958 | flagutil, | |||
|
r41063 | ) | ||
|
r31748 | # TESTTMP is optional. This makes it convenient to run without run-tests.py | ||
|
r37913 | tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp')) | ||
|
r31748 | |||
# Enable generaldelta otherwise revlog won't use delta as expected by the test | ||||
|
r43346 | tvfs.options = { | ||
b'generaldelta': True, | ||||
b'revlogv1': True, | ||||
b'sparse-revlog': True, | ||||
} | ||||
|
r31748 | |||
# The test wants to control whether to use delta explicitly, based on | ||||
# "storedeltachains". | ||||
|
r39268 | revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self._storedeltachains | ||
|
r31748 | |||
|
r43346 | |||
|
r31748 | def abort(msg): | ||
print('abort: %s' % msg) | ||||
# Return 0 so run-tests.py could compare the output. | ||||
sys.exit() | ||||
|
r43346 | |||
|
r31748 | # Register a revlog processor for flag EXTSTORED. | ||
# | ||||
# It simply prepends a fixed header, and replaces '1' to 'i'. So it has | ||||
# insertion and replacement, and may be interesting to test revlog's line-based | ||||
# deltas. | ||||
_extheader = b'E\n' | ||||
|
r43346 | |||
|
r31748 | def readprocessor(self, rawtext): | ||
# True: the returned text could be used to verify hash | ||||
|
r43346 | text = rawtext[len(_extheader) :].replace(b'i', b'1') | ||
|
r47443 | return text, True | ||
|
r31748 | |||
|
r43346 | |||
|
r47443 | def writeprocessor(self, text): | ||
|
r31748 | # False: the returned rawtext shouldn't be used to verify hash | ||
rawtext = _extheader + text.replace(b'1', b'i') | ||||
return rawtext, False | ||||
|
r43346 | |||
|
r31748 | def rawprocessor(self, rawtext): | ||
# False: do not verify hash. Only the content returned by "readprocessor" | ||||
# can be used to verify hash. | ||||
return False | ||||
|
r43346 | |||
flagutil.addflagprocessor( | ||||
revlog.REVIDX_EXTSTORED, (readprocessor, writeprocessor, rawprocessor) | ||||
) | ||||
|
r31748 | |||
# Utilities about reading and appending revlog | ||||
|
r43346 | |||
|
r31748 | def newtransaction(): | ||
# A transaction is required to write revlogs | ||||
report = lambda msg: None | ||||
return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal') | ||||
|
r43346 | |||
|
r31748 | def newrevlog(name=b'_testrevlog.i', recreate=False): | ||
if recreate: | ||||
tvfs.tryunlink(name) | ||||
rlog = revlog.revlog(tvfs, name) | ||||
return rlog | ||||
|
r43346 | |||
|
r31748 | def appendrev(rlog, text, tr, isext=False, isdelta=True): | ||
|
r46554 | """Append a revision. If isext is True, set the EXTSTORED flag so flag | ||
|
r31748 | processor will be used (and rawtext is different from text). If isdelta is | ||
True, force the revision to be a delta, otherwise it's full text. | ||||
|
r46554 | """ | ||
|
r31748 | nextrev = len(rlog) | ||
p1 = rlog.node(nextrev - 1) | ||||
|
r46729 | p2 = nullid | ||
|
r31748 | if isext: | ||
flags = revlog.REVIDX_EXTSTORED | ||||
else: | ||||
flags = revlog.REVIDX_DEFAULT_FLAGS | ||||
# Change storedeltachains temporarily, to override revlog's delta decision | ||||
|
r39268 | rlog._storedeltachains = isdelta | ||
|
r31748 | try: | ||
rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags) | ||||
return nextrev | ||||
except Exception as ex: | ||||
abort('rev %d: failed to append: %s' % (nextrev, ex)) | ||||
finally: | ||||
# Restore storedeltachains. It is always True, see revlog.__init__ | ||||
|
r39268 | rlog._storedeltachains = True | ||
|
r31748 | |||
|
r43346 | |||
|
r31748 | def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True): | ||
|
r46554 | """Copy revlog to destname using revlog.addgroup. Return the copied revlog. | ||
|
r31748 | |||
This emulates push or pull. They use changegroup. Changegroup requires | ||||
repo to work. We don't have a repo, so a dummy changegroup is used. | ||||
If optimaldelta is True, use optimized delta parent, so the destination | ||||
revlog could probably reuse it. Otherwise it builds sub-optimal delta, and | ||||
the destination revlog needs more work to use it. | ||||
This exercises some revlog.addgroup (and revlog._addrevision(text=None)) | ||||
code path, which is not covered by "appendrev" alone. | ||||
|
r46554 | """ | ||
|
r43346 | |||
|
r31748 | class dummychangegroup(object): | ||
@staticmethod | ||||
def deltachunk(pnode): | ||||
|
r46729 | pnode = pnode or nullid | ||
|
r31748 | parentrev = rlog.rev(pnode) | ||
r = parentrev + 1 | ||||
if r >= len(rlog): | ||||
return {} | ||||
if optimaldelta: | ||||
deltaparent = parentrev | ||||
else: | ||||
# suboptimal deltaparent | ||||
deltaparent = min(0, parentrev) | ||||
|
r36765 | if not rlog.candelta(deltaparent, r): | ||
deltaparent = -1 | ||||
|
r43346 | return { | ||
b'node': rlog.node(r), | ||||
b'p1': pnode, | ||||
|
r46729 | b'p2': nullid, | ||
|
r43346 | b'cs': rlog.node(rlog.linkrev(r)), | ||
b'flags': rlog.flags(r), | ||||
b'deltabase': rlog.node(deltaparent), | ||||
b'delta': rlog.revdiff(deltaparent, r), | ||||
|
r47445 | b'sidedata': rlog.sidedata(r), | ||
|
r43346 | } | ||
|
r31748 | |||
|
r34292 | def deltaiter(self): | ||
|
r34147 | chain = None | ||
for chunkdata in iter(lambda: self.deltachunk(chain), {}): | ||||
|
r37913 | node = chunkdata[b'node'] | ||
p1 = chunkdata[b'p1'] | ||||
p2 = chunkdata[b'p2'] | ||||
cs = chunkdata[b'cs'] | ||||
deltabase = chunkdata[b'deltabase'] | ||||
delta = chunkdata[b'delta'] | ||||
flags = chunkdata[b'flags'] | ||||
|
r47445 | sidedata = chunkdata[b'sidedata'] | ||
|
r34147 | |||
chain = node | ||||
|
r47445 | yield (node, p1, p2, cs, deltabase, delta, flags, sidedata) | ||
|
r34147 | |||
|
r31748 | def linkmap(lnode): | ||
return rlog.rev(lnode) | ||||
dlog = newrevlog(destname, recreate=True) | ||||
|
r34292 | dummydeltas = dummychangegroup().deltaiter() | ||
dlog.addgroup(dummydeltas, linkmap, tr) | ||||
|
r31748 | return dlog | ||
|
r43346 | |||
|
r31748 | def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'): | ||
|
r46554 | """Like addgroupcopy, but use the low level revlog._addrevision directly. | ||
|
r31748 | |||
It exercises some code paths that are hard to reach easily otherwise. | ||||
|
r46554 | """ | ||
|
r31748 | dlog = newrevlog(destname, recreate=True) | ||
for r in rlog: | ||||
p1 = rlog.node(r - 1) | ||||
|
r46729 | p2 = nullid | ||
|
r36765 | if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED): | ||
r43045 | text = rlog.rawdata(r) | |||
|
r31748 | cachedelta = None | ||
else: | ||||
|
r36765 | # deltaparent cannot have EXTSTORED flag. | ||
|
r43346 | deltaparent = max( | ||
[-1] | ||||
+ [ | ||||
p | ||||
for p in range(r) | ||||
if rlog.flags(p) & revlog.REVIDX_EXTSTORED == 0 | ||||
] | ||||
) | ||||
|
r31748 | text = None | ||
cachedelta = (deltaparent, rlog.revdiff(deltaparent, r)) | ||||
flags = rlog.flags(r) | ||||
|
r33627 | ifh = dfh = None | ||
try: | ||||
|
r37913 | ifh = dlog.opener(dlog.indexfile, b'a+') | ||
|
r33627 | if not dlog._inline: | ||
|
r37913 | dfh = dlog.opener(dlog.datafile, b'a+') | ||
|
r43346 | dlog._addrevision( | ||
rlog.node(r), text, tr, r, p1, p2, flags, cachedelta, ifh, dfh | ||||
) | ||||
|
r33627 | finally: | ||
if dfh is not None: | ||||
dfh.close() | ||||
if ifh is not None: | ||||
ifh.close() | ||||
|
r31748 | return dlog | ||
|
r43346 | |||
|
r31748 | # Utilities to generate revisions for testing | ||
|
r43346 | |||
|
r31748 | def genbits(n): | ||
|
r46554 | """Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n). | ||
|
r31748 | i.e. the generated numbers have a width of n bits. | ||
The combination of two adjacent numbers will cover all possible cases. | ||||
That is to say, given any x, y where both x, and y are in range(2 ** n), | ||||
there is an x followed immediately by y in the generated sequence. | ||||
|
r46554 | """ | ||
|
r31748 | m = 2 ** n | ||
# Gray Code. See https://en.wikipedia.org/wiki/Gray_code | ||||
gray = lambda x: x ^ (x >> 1) | ||||
|
r44937 | reversegray = {gray(i): i for i in range(m)} | ||
|
r31748 | |||
# Generate (n * 2) bit gray code, yield lower n bits as X, and look for | ||||
# the next unused gray code where higher n bits equal to X. | ||||
# For gray codes whose higher bits are X, a[X] of them have been used. | ||||
a = [0] * m | ||||
# Iterate from 0. | ||||
x = 0 | ||||
yield x | ||||
for i in range(m * m): | ||||
|
r31763 | x = reversegray[x] | ||
|
r31748 | y = gray(a[x] + x * m) & (m - 1) | ||
|
r31763 | assert a[x] < m | ||
|
r31748 | a[x] += 1 | ||
x = y | ||||
yield x | ||||
|
r43346 | |||
|
r31748 | def gentext(rev): | ||
'''Given a revision number, generate dummy text''' | ||||
return b''.join(b'%d\n' % j for j in range(-1, rev % 5)) | ||||
|
r43346 | |||
|
r31748 | def writecases(rlog, tr): | ||
|
r46554 | """Write some revisions interested to the test. | ||
|
r31748 | |||
The test is interested in 3 properties of a revision: | ||||
- Is it a delta or a full text? (isdelta) | ||||
This is to catch some delta application issues. | ||||
- Does it have a flag of EXTSTORED? (isext) | ||||
This is to catch some flag processor issues. Especially when | ||||
interacted with revlog deltas. | ||||
- Is its text empty? (isempty) | ||||
This is less important. It is intended to try to catch some careless | ||||
checks like "if text" instead of "if text is None". Note: if flag | ||||
processor is involved, raw text may be not empty. | ||||
Write 65 revisions. So that all combinations of the above flags for | ||||
adjacent revisions are covered. That is to say, | ||||
len(set( | ||||
(r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty) | ||||
for r in range(len(rlog) - 1) | ||||
)) is 64. | ||||
Where "r.delta", "r.ext", and "r.empty" are booleans matching properties | ||||
mentioned above. | ||||
Return expected [(text, rawtext)]. | ||||
|
r46554 | """ | ||
|
r31748 | result = [] | ||
for i, x in enumerate(genbits(3)): | ||||
isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4) | ||||
if isempty: | ||||
text = b'' | ||||
else: | ||||
text = gentext(i) | ||||
rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta) | ||||
# Verify text, rawtext, and rawsize | ||||
if isext: | ||||
|
r47443 | rawtext = writeprocessor(None, text)[0] | ||
|
r31748 | else: | ||
rawtext = text | ||||
if rlog.rawsize(rev) != len(rawtext): | ||||
abort('rev %d: wrong rawsize' % rev) | ||||
if rlog.revision(rev, raw=False) != text: | ||||
abort('rev %d: wrong text' % rev) | ||||
r43045 | if rlog.rawdata(rev) != rawtext: | |||
|
r31748 | abort('rev %d: wrong rawtext' % rev) | ||
result.append((text, rawtext)) | ||||
# Verify flags like isdelta, isext work as expected | ||||
|
r36765 | # isdelta can be overridden to False if this or p1 has isext set | ||
if bool(rlog.deltaparent(rev) > -1) and not isdelta: | ||||
abort('rev %d: isdelta is unexpected' % rev) | ||||
|
r31748 | if bool(rlog.flags(rev)) != isext: | ||
abort('rev %d: isext is ineffective' % rev) | ||||
return result | ||||
|
r43346 | |||
|
r31748 | # Main test and checking | ||
|
r43346 | |||
|
r31748 | def checkrevlog(rlog, expected): | ||
'''Check if revlog has expected contents. expected is [(text, rawtext)]''' | ||||
# Test using different access orders. This could expose some issues | ||||
# depending on revlog caching (see revlog._cache). | ||||
for r0 in range(len(rlog) - 1): | ||||
r1 = r0 + 1 | ||||
for revorder in [[r0, r1], [r1, r0]]: | ||||
for raworder in [[True], [False], [True, False], [False, True]]: | ||||
nlog = newrevlog() | ||||
for rev in revorder: | ||||
for raw in raworder: | ||||
r43045 | if raw: | |||
t = nlog.rawdata(rev) | ||||
else: | ||||
t = nlog.revision(rev) | ||||
|
r31748 | if t != expected[rev][int(raw)]: | ||
|
r43346 | abort( | ||
'rev %d: corrupted %stext' | ||||
% (rev, raw and 'raw' or '') | ||||
) | ||||
|
r31748 | |||
|
r41063 | slicingdata = [ | ||
|
r43346 | ([0, 1, 2, 3, 55, 56, 58, 59, 60], [[0, 1], [2], [58], [59, 60]], 10), | ||
([0, 1, 2, 3, 55, 56, 58, 59, 60], [[0, 1], [2], [58], [59, 60]], 10), | ||||
( | ||||
[-1, 0, 1, 2, 3, 55, 56, 58, 59, 60], | ||||
[[-1, 0, 1], [2], [58], [59, 60]], | ||||
10, | ||||
), | ||||
|
r41063 | ] | ||
|
r43346 | |||
|
r41063 | def slicingtest(rlog): | ||
oldmin = rlog._srmingapsize | ||||
try: | ||||
# the test revlog is small, we remove the floor under which we | ||||
# slicing is diregarded. | ||||
rlog._srmingapsize = 0 | ||||
for item in slicingdata: | ||||
chain, expected, target = item | ||||
result = deltas.slicechunk(rlog, chain, targetsize=target) | ||||
result = list(result) | ||||
if result != expected: | ||||
print('slicing differ:') | ||||
print(' chain: %s' % chain) | ||||
print(' target: %s' % target) | ||||
print(' expected: %s' % expected) | ||||
print(' result: %s' % result) | ||||
finally: | ||||
rlog._srmingapsize = oldmin | ||||
|
r43346 | |||
|
r41064 | def md5sum(s): | ||
return hashlib.md5(s).digest() | ||||
|
r43346 | |||
|
r41064 | def _maketext(*coord): | ||
"""create piece of text according to range of integers | ||||
The test returned use a md5sum of the integer to make it less | ||||
compressible""" | ||||
pieces = [] | ||||
for start, size in coord: | ||||
num = range(start, start + size) | ||||
p = [md5sum(b'%d' % r) for r in num] | ||||
pieces.append(b'\n'.join(p)) | ||||
return b'\n'.join(pieces) + b'\n' | ||||
|
r43346 | |||
|
r41064 | data = [ | ||
_maketext((0, 120), (456, 60)), | ||||
_maketext((0, 120), (345, 60)), | ||||
_maketext((0, 120), (734, 60)), | ||||
_maketext((0, 120), (734, 60), (923, 45)), | ||||
_maketext((0, 120), (734, 60), (234, 45)), | ||||
_maketext((0, 120), (734, 60), (564, 45)), | ||||
_maketext((0, 120), (734, 60), (361, 45)), | ||||
_maketext((0, 120), (734, 60), (489, 45)), | ||||
_maketext((0, 120), (123, 60)), | ||||
_maketext((0, 120), (145, 60)), | ||||
_maketext((0, 120), (104, 60)), | ||||
_maketext((0, 120), (430, 60)), | ||||
_maketext((0, 120), (430, 60), (923, 45)), | ||||
_maketext((0, 120), (430, 60), (234, 45)), | ||||
_maketext((0, 120), (430, 60), (564, 45)), | ||||
_maketext((0, 120), (430, 60), (361, 45)), | ||||
_maketext((0, 120), (430, 60), (489, 45)), | ||||
_maketext((0, 120), (249, 60)), | ||||
_maketext((0, 120), (832, 60)), | ||||
_maketext((0, 120), (891, 60)), | ||||
_maketext((0, 120), (543, 60)), | ||||
_maketext((0, 120), (120, 60)), | ||||
_maketext((0, 120), (60, 60), (768, 30)), | ||||
_maketext((0, 120), (60, 60), (260, 30)), | ||||
_maketext((0, 120), (60, 60), (450, 30)), | ||||
_maketext((0, 120), (60, 60), (361, 30)), | ||||
_maketext((0, 120), (60, 60), (886, 30)), | ||||
_maketext((0, 120), (60, 60), (116, 30)), | ||||
_maketext((0, 120), (60, 60), (567, 30), (629, 40)), | ||||
_maketext((0, 120), (60, 60), (569, 30), (745, 40)), | ||||
_maketext((0, 120), (60, 60), (777, 30), (700, 40)), | ||||
_maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)), | ||||
] | ||||
|
r43346 | |||
|
r41064 | def makesnapshot(tr): | ||
rl = newrevlog(name=b'_snaprevlog3.i', recreate=True) | ||||
for i in data: | ||||
appendrev(rl, i, tr) | ||||
return rl | ||||
|
r43346 | |||
|
r41064 | snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30] | ||
|
r43346 | |||
|
r41064 | def issnapshottest(rlog): | ||
result = [] | ||||
if rlog.issnapshot(-1): | ||||
result.append(-1) | ||||
for rev in rlog: | ||||
if rlog.issnapshot(rev): | ||||
result.append(rev) | ||||
if snapshots != result: | ||||
print('snapshot differ:') | ||||
print(' expected: %s' % snapshots) | ||||
print(' got: %s' % result) | ||||
|
r43346 | |||
|
r41119 | snapshotmapall = {0: [6, 8, 11, 17, 19, 25], 8: [21], -1: [0, 30]} | ||
snapshotmap15 = {0: [17, 19, 25], 8: [21], -1: [30]} | ||||
|
r43346 | |||
|
r41119 | def findsnapshottest(rlog): | ||
resultall = collections.defaultdict(list) | ||||
deltas._findsnapshots(rlog, resultall, 0) | ||||
resultall = dict(resultall.items()) | ||||
if resultall != snapshotmapall: | ||||
print('snapshot map differ:') | ||||
print(' expected: %s' % snapshotmapall) | ||||
print(' got: %s' % resultall) | ||||
result15 = collections.defaultdict(list) | ||||
deltas._findsnapshots(rlog, result15, 15) | ||||
result15 = dict(result15.items()) | ||||
if result15 != snapshotmap15: | ||||
print('snapshot map differ:') | ||||
print(' expected: %s' % snapshotmap15) | ||||
print(' got: %s' % result15) | ||||
|
r43346 | |||
|
r31748 | def maintest(): | ||
with newtransaction() as tr: | ||||
rl = newrevlog(recreate=True) | ||||
expected = writecases(rl, tr) | ||||
checkrevlog(rl, expected) | ||||
print('local test passed') | ||||
# Copy via revlog.addgroup | ||||
rl1 = addgroupcopy(rl, tr) | ||||
checkrevlog(rl1, expected) | ||||
rl2 = addgroupcopy(rl, tr, optimaldelta=False) | ||||
checkrevlog(rl2, expected) | ||||
print('addgroupcopy test passed') | ||||
# Copy via revlog.clone | ||||
|
r37913 | rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True) | ||
|
r31748 | rl.clone(tr, rl3) | ||
checkrevlog(rl3, expected) | ||||
print('clone test passed') | ||||
# Copy via low-level revlog._addrevision | ||||
rl4 = lowlevelcopy(rl, tr) | ||||
checkrevlog(rl4, expected) | ||||
print('lowlevelcopy test passed') | ||||
|
r41063 | slicingtest(rl) | ||
print('slicing test passed') | ||||
|
r41064 | rl5 = makesnapshot(tr) | ||
issnapshottest(rl5) | ||||
print('issnapshot test passed') | ||||
|
r41119 | findsnapshottest(rl5) | ||
print('findsnapshot test passed') | ||||
|
r31748 | |||
|
r43346 | |||
|
r31748 | try: | ||
maintest() | ||||
except Exception as ex: | ||||
abort('crashed: %s' % ex) | ||||