debugcommands.py
477 lines
| 13.2 KiB
| text/x-python
|
PythonLexer
|
r40530 | # debugcommands.py - debug logic for remotefilelog | ||
# | ||||
# Copyright 2013 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
from __future__ import absolute_import | ||||
import os | ||||
|
r40542 | import zlib | ||
|
r40530 | |||
from mercurial.node import bin, hex, nullid, short | ||||
from mercurial.i18n import _ | ||||
|
r43355 | from mercurial.pycompat import open | ||
|
r40530 | from mercurial import ( | ||
error, | ||||
filelog, | ||||
|
r43213 | lock as lockmod, | ||
|
r40648 | node as nodemod, | ||
|
r41973 | pycompat, | ||
|
r40530 | revlog, | ||
) | ||||
|
r44519 | from mercurial.utils import hashutil | ||
|
r40530 | from . import ( | ||
constants, | ||||
datapack, | ||||
fileserverclient, | ||||
historypack, | ||||
repack, | ||||
shallowutil, | ||||
) | ||||
|
r43346 | |||
|
r40530 | def debugremotefilelog(ui, path, **opts): | ||
|
r43906 | decompress = opts.get('decompress') | ||
|
r40530 | |||
size, firstnode, mapping = parsefileblob(path, decompress) | ||||
|
r43347 | ui.status(_(b"size: %d bytes\n") % size) | ||
ui.status(_(b"path: %s \n") % path) | ||||
ui.status(_(b"key: %s \n") % (short(firstnode))) | ||||
ui.status(_(b"\n")) | ||||
|
r43346 | ui.status( | ||
|
r43347 | _(b"%12s => %12s %13s %13s %12s\n") | ||
% (b"node", b"p1", b"p2", b"linknode", b"copyfrom") | ||||
|
r43346 | ) | ||
|
r40530 | |||
queue = [firstnode] | ||||
while queue: | ||||
node = queue.pop(0) | ||||
p1, p2, linknode, copyfrom = mapping[node] | ||||
|
r43346 | ui.status( | ||
|
r43347 | _(b"%s => %s %s %s %s\n") | ||
|
r43346 | % (short(node), short(p1), short(p2), short(linknode), copyfrom) | ||
) | ||||
|
r40530 | if p1 != nullid: | ||
queue.append(p1) | ||||
if p2 != nullid: | ||||
queue.append(p2) | ||||
|
r43346 | |||
|
r40530 | def buildtemprevlog(repo, file): | ||
# get filename key | ||||
|
r44519 | filekey = nodemod.hex(hashutil.sha1(file).digest()) | ||
|
r43347 | filedir = os.path.join(repo.path, b'store/data', filekey) | ||
|
r40530 | |||
# sort all entries based on linkrev | ||||
fctxs = [] | ||||
for filenode in os.listdir(filedir): | ||||
|
r43347 | if b'_old' not in filenode: | ||
|
r40530 | fctxs.append(repo.filectx(file, fileid=bin(filenode))) | ||
fctxs = sorted(fctxs, key=lambda x: x.linkrev()) | ||||
# add to revlog | ||||
|
r43347 | temppath = repo.sjoin(b'data/temprevlog.i') | ||
|
r40530 | if os.path.exists(temppath): | ||
os.remove(temppath) | ||||
|
r43347 | r = filelog.filelog(repo.svfs, b'temprevlog') | ||
|
r40530 | |||
class faket(object): | ||||
def add(self, a, b, c): | ||||
pass | ||||
|
r43346 | |||
|
r40530 | t = faket() | ||
for fctx in fctxs: | ||||
if fctx.node() not in repo: | ||||
continue | ||||
p = fctx.filelog().parents(fctx.filenode()) | ||||
meta = {} | ||||
if fctx.renamed(): | ||||
|
r43347 | meta[b'copy'] = fctx.renamed()[0] | ||
meta[b'copyrev'] = hex(fctx.renamed()[1]) | ||||
|
r40530 | |||
r.add(fctx.data(), meta, t, fctx.linkrev(), p[0], p[1]) | ||||
return r | ||||
|
r43346 | |||
|
r40530 | def debugindex(orig, ui, repo, file_=None, **opts): | ||
"""dump the contents of an index file""" | ||||
|
r43346 | if ( | ||
|
r43906 | opts.get('changelog') | ||
or opts.get('manifest') | ||||
or opts.get('dir') | ||||
|
r43346 | or not shallowutil.isenabled(repo) | ||
or not repo.shallowmatch(file_) | ||||
): | ||||
|
r40530 | return orig(ui, repo, file_, **opts) | ||
r = buildtemprevlog(repo, file_) | ||||
# debugindex like normal | ||||
|
r43347 | format = opts.get(b'format', 0) | ||
|
r40530 | if format not in (0, 1): | ||
|
r43347 | raise error.Abort(_(b"unknown format %d") % format) | ||
|
r40530 | |||
generaldelta = r.version & revlog.FLAG_GENERALDELTA | ||||
if generaldelta: | ||||
|
r43347 | basehdr = b' delta' | ||
|
r40530 | else: | ||
|
r43347 | basehdr = b' base' | ||
|
r40530 | |||
if format == 0: | ||||
|
r43346 | ui.write( | ||
( | ||||
|
r43347 | b" rev offset length " + basehdr + b" linkrev" | ||
b" nodeid p1 p2\n" | ||||
|
r43346 | ) | ||
) | ||||
|
r40530 | elif format == 1: | ||
|
r43346 | ui.write( | ||
( | ||||
|
r43347 | b" rev flag offset length" | ||
b" size " + basehdr + b" link p1 p2" | ||||
b" nodeid\n" | ||||
|
r43346 | ) | ||
) | ||||
|
r40530 | |||
for i in r: | ||||
node = r.node(i) | ||||
if generaldelta: | ||||
base = r.deltaparent(i) | ||||
else: | ||||
base = r.chainbase(i) | ||||
if format == 0: | ||||
try: | ||||
pp = r.parents(node) | ||||
except Exception: | ||||
pp = [nullid, nullid] | ||||
|
r43346 | ui.write( | ||
|
r43347 | b"% 6d % 9d % 7d % 6d % 7d %s %s %s\n" | ||
|
r43346 | % ( | ||
i, | ||||
r.start(i), | ||||
r.length(i), | ||||
base, | ||||
r.linkrev(i), | ||||
short(node), | ||||
short(pp[0]), | ||||
short(pp[1]), | ||||
) | ||||
) | ||||
|
r40530 | elif format == 1: | ||
pr = r.parentrevs(i) | ||||
|
r43346 | ui.write( | ||
|
r43347 | b"% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" | ||
|
r43346 | % ( | ||
i, | ||||
r.flags(i), | ||||
r.start(i), | ||||
r.length(i), | ||||
r.rawsize(i), | ||||
base, | ||||
r.linkrev(i), | ||||
pr[0], | ||||
pr[1], | ||||
short(node), | ||||
) | ||||
) | ||||
|
r40530 | |||
def debugindexdot(orig, ui, repo, file_): | ||||
"""dump an index DAG as a graphviz dot file""" | ||||
|
r40549 | if not shallowutil.isenabled(repo): | ||
|
r40530 | return orig(ui, repo, file_) | ||
r = buildtemprevlog(repo, os.path.basename(file_)[:-2]) | ||||
|
r43350 | ui.writenoi18n(b"digraph G {\n") | ||
|
r40530 | for i in r: | ||
node = r.node(i) | ||||
pp = r.parents(node) | ||||
|
r43347 | ui.write(b"\t%d -> %d\n" % (r.rev(pp[0]), i)) | ||
|
r40530 | if pp[1] != nullid: | ||
|
r43347 | ui.write(b"\t%d -> %d\n" % (r.rev(pp[1]), i)) | ||
ui.write(b"}\n") | ||||
|
r40530 | |||
|
r43346 | |||
|
r40530 | def verifyremotefilelog(ui, path, **opts): | ||
|
r43906 | decompress = opts.get('decompress') | ||
|
r40530 | |||
for root, dirs, files in os.walk(path): | ||||
for file in files: | ||||
|
r43347 | if file == b"repos": | ||
|
r40530 | continue | ||
filepath = os.path.join(root, file) | ||||
size, firstnode, mapping = parsefileblob(filepath, decompress) | ||||
|
r43374 | for p1, p2, linknode, copyfrom in pycompat.itervalues(mapping): | ||
|
r40530 | if linknode == nullid: | ||
actualpath = os.path.relpath(root, path) | ||||
|
r43346 | key = fileserverclient.getcachekey( | ||
|
r43347 | b"reponame", actualpath, file | ||
|
r43346 | ) | ||
ui.status( | ||||
|
r43347 | b"%s %s\n" % (key, os.path.relpath(filepath, path)) | ||
|
r43346 | ) | ||
|
r40530 | |||
|
r40542 | def _decompressblob(raw): | ||
return zlib.decompress(raw) | ||||
|
r43346 | |||
|
r40530 | def parsefileblob(path, decompress): | ||
|
r43347 | f = open(path, b"rb") | ||
|
r40530 | try: | ||
raw = f.read() | ||||
finally: | ||||
f.close() | ||||
if decompress: | ||||
|
r40542 | raw = _decompressblob(raw) | ||
|
r40530 | |||
offset, size, flags = shallowutil.parsesizeflags(raw) | ||||
start = offset + size | ||||
firstnode = None | ||||
mapping = {} | ||||
while start < len(raw): | ||||
|
r43347 | divider = raw.index(b'\0', start + 80) | ||
|
r40530 | |||
|
r43346 | currentnode = raw[start : (start + 20)] | ||
|
r40530 | if not firstnode: | ||
firstnode = currentnode | ||||
|
r43346 | p1 = raw[(start + 20) : (start + 40)] | ||
p2 = raw[(start + 40) : (start + 60)] | ||||
linknode = raw[(start + 60) : (start + 80)] | ||||
copyfrom = raw[(start + 80) : divider] | ||||
|
r40530 | |||
mapping[currentnode] = (p1, p2, linknode, copyfrom) | ||||
start = divider + 1 | ||||
return size, firstnode, mapping | ||||
|
r43346 | |||
|
r40530 | def debugdatapack(ui, *paths, **opts): | ||
for path in paths: | ||||
|
r43347 | if b'.data' in path: | ||
path = path[: path.index(b'.data')] | ||||
ui.write(b"%s:\n" % path) | ||||
|
r40530 | dpack = datapack.datapack(path) | ||
|
r43906 | node = opts.get('node') | ||
|
r40530 | if node: | ||
|
r43347 | deltachain = dpack.getdeltachain(b'', bin(node)) | ||
|
r40530 | dumpdeltachain(ui, deltachain, **opts) | ||
return | ||||
|
r43906 | if opts.get('long'): | ||
|
r40530 | hashformatter = hex | ||
hashlen = 42 | ||||
else: | ||||
hashformatter = short | ||||
hashlen = 14 | ||||
lastfilename = None | ||||
totaldeltasize = 0 | ||||
totalblobsize = 0 | ||||
|
r43346 | |||
|
r40530 | def printtotals(): | ||
if lastfilename is not None: | ||||
|
r43347 | ui.write(b"\n") | ||
|
r40530 | if not totaldeltasize or not totalblobsize: | ||
return | ||||
difference = totalblobsize - totaldeltasize | ||||
|
r43347 | deltastr = b"%0.1f%% %s" % ( | ||
|
r40530 | (100.0 * abs(difference) / totalblobsize), | ||
|
r43347 | (b"smaller" if difference > 0 else b"bigger"), | ||
|
r43346 | ) | ||
|
r40530 | |||
|
r43350 | ui.writenoi18n( | ||
|
r43347 | b"Total:%s%s %s (%s)\n" | ||
|
r43346 | % ( | ||
|
r43347 | b"".ljust(2 * hashlen - len(b"Total:")), | ||
(b'%d' % totaldeltasize).ljust(12), | ||||
(b'%d' % totalblobsize).ljust(9), | ||||
|
r43346 | deltastr, | ||
) | ||||
) | ||||
|
r40530 | |||
bases = {} | ||||
nodes = set() | ||||
failures = 0 | ||||
for filename, node, deltabase, deltalen in dpack.iterentries(): | ||||
bases[node] = deltabase | ||||
if node in nodes: | ||||
|
r43347 | ui.write((b"Bad entry: %s appears twice\n" % short(node))) | ||
|
r40530 | failures += 1 | ||
nodes.add(node) | ||||
if filename != lastfilename: | ||||
printtotals() | ||||
|
r43347 | name = b'(empty name)' if filename == b'' else filename | ||
ui.write(b"%s:\n" % name) | ||||
|
r43346 | ui.write( | ||
|
r43347 | b"%s%s%s%s\n" | ||
|
r43346 | % ( | ||
|
r43347 | b"Node".ljust(hashlen), | ||
b"Delta Base".ljust(hashlen), | ||||
b"Delta Length".ljust(14), | ||||
b"Blob Size".ljust(9), | ||||
|
r43346 | ) | ||
) | ||||
|
r40530 | lastfilename = filename | ||
totalblobsize = 0 | ||||
totaldeltasize = 0 | ||||
# Metadata could be missing, in which case it will be an empty dict. | ||||
meta = dpack.getmeta(filename, node) | ||||
if constants.METAKEYSIZE in meta: | ||||
blobsize = meta[constants.METAKEYSIZE] | ||||
totaldeltasize += deltalen | ||||
totalblobsize += blobsize | ||||
else: | ||||
|
r43347 | blobsize = b"(missing)" | ||
|
r43346 | ui.write( | ||
|
r43347 | b"%s %s %s%s\n" | ||
|
r43346 | % ( | ||
hashformatter(node), | ||||
hashformatter(deltabase), | ||||
|
r43347 | (b'%d' % deltalen).ljust(14), | ||
|
r43346 | pycompat.bytestr(blobsize), | ||
) | ||||
) | ||||
|
r40530 | |||
if filename is not None: | ||||
printtotals() | ||||
failures += _sanitycheck(ui, set(nodes), bases) | ||||
if failures > 1: | ||||
|
r43347 | ui.warn((b"%d failures\n" % failures)) | ||
|
r40530 | return 1 | ||
|
r43346 | |||
|
r40530 | def _sanitycheck(ui, nodes, bases): | ||
""" | ||||
Does some basic sanity checking on a packfiles with ``nodes`` ``bases`` (a | ||||
mapping of node->base): | ||||
- Each deltabase must itself be a node elsewhere in the pack | ||||
- There must be no cycles | ||||
""" | ||||
failures = 0 | ||||
for node in nodes: | ||||
seen = set() | ||||
current = node | ||||
deltabase = bases[current] | ||||
while deltabase != nullid: | ||||
if deltabase not in nodes: | ||||
|
r43346 | ui.warn( | ||
( | ||||
|
r43347 | b"Bad entry: %s has an unknown deltabase (%s)\n" | ||
|
r43346 | % (short(node), short(deltabase)) | ||
) | ||||
) | ||||
|
r40530 | failures += 1 | ||
break | ||||
if deltabase in seen: | ||||
|
r43346 | ui.warn( | ||
( | ||||
|
r43347 | b"Bad entry: %s has a cycle (at %s)\n" | ||
|
r43346 | % (short(node), short(deltabase)) | ||
) | ||||
) | ||||
|
r40530 | failures += 1 | ||
break | ||||
current = deltabase | ||||
seen.add(current) | ||||
deltabase = bases[current] | ||||
# Since ``node`` begins a valid chain, reset/memoize its base to nullid | ||||
# so we don't traverse it again. | ||||
bases[node] = nullid | ||||
return failures | ||||
|
r43346 | |||
|
r40530 | def dumpdeltachain(ui, deltachain, **opts): | ||
hashformatter = hex | ||||
hashlen = 40 | ||||
lastfilename = None | ||||
for filename, node, filename, deltabasenode, delta in deltachain: | ||||
if filename != lastfilename: | ||||
|
r43347 | ui.write(b"\n%s\n" % filename) | ||
|
r40530 | lastfilename = filename | ||
|
r43346 | ui.write( | ||
|
r43347 | b"%s %s %s %s\n" | ||
|
r43346 | % ( | ||
|
r43347 | b"Node".ljust(hashlen), | ||
b"Delta Base".ljust(hashlen), | ||||
b"Delta SHA1".ljust(hashlen), | ||||
b"Delta Length".ljust(6), | ||||
|
r43346 | ) | ||
) | ||||
|
r40530 | |||
|
r43346 | ui.write( | ||
|
r43347 | b"%s %s %s %d\n" | ||
|
r43346 | % ( | ||
hashformatter(node), | ||||
hashformatter(deltabasenode), | ||||
|
r44519 | nodemod.hex(hashutil.sha1(delta).digest()), | ||
|
r43346 | len(delta), | ||
) | ||||
) | ||||
|
r40530 | |||
def debughistorypack(ui, path): | ||||
|
r43347 | if b'.hist' in path: | ||
path = path[: path.index(b'.hist')] | ||||
|
r40530 | hpack = historypack.historypack(path) | ||
lastfilename = None | ||||
for entry in hpack.iterentries(): | ||||
filename, node, p1node, p2node, linknode, copyfrom = entry | ||||
if filename != lastfilename: | ||||
|
r43347 | ui.write(b"\n%s\n" % filename) | ||
|
r43346 | ui.write( | ||
|
r43347 | b"%s%s%s%s%s\n" | ||
|
r43346 | % ( | ||
|
r43347 | b"Node".ljust(14), | ||
b"P1 Node".ljust(14), | ||||
b"P2 Node".ljust(14), | ||||
b"Link Node".ljust(14), | ||||
b"Copy From", | ||||
|
r43346 | ) | ||
) | ||||
|
r40530 | lastfilename = filename | ||
|
r43346 | ui.write( | ||
|
r43347 | b"%s %s %s %s %s\n" | ||
|
r43346 | % ( | ||
short(node), | ||||
short(p1node), | ||||
short(p2node), | ||||
short(linknode), | ||||
copyfrom, | ||||
) | ||||
) | ||||
|
r40530 | |||
def debugwaitonrepack(repo): | ||||
|
r43347 | with lockmod.lock(repack.repacklockvfs(repo), b"repacklock", timeout=-1): | ||
|
r40530 | return | ||
|
r43346 | |||
|
r40530 | def debugwaitonprefetch(repo): | ||
|
r43346 | with repo._lock( | ||
repo.svfs, | ||||
|
r43347 | b"prefetchlock", | ||
|
r43346 | True, | ||
None, | ||||
None, | ||||
|
r43347 | _(b'prefetching in %s') % repo.origroot, | ||
|
r43346 | ): | ||
|
r40530 | pass | ||