verify.py
489 lines
| 19.0 KiB
| text/x-python
|
PythonLexer
/ mercurial / verify.py
|
r2778 | # verify.py - repository integrity checking for Mercurial | ||
# | ||||
|
r4635 | # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com> | ||
|
r2778 | # | ||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r2778 | |||
|
r25991 | from __future__ import absolute_import | ||
|
r17860 | import os | ||
|
r25991 | |||
from .i18n import _ | ||||
from .node import ( | ||||
nullid, | ||||
short, | ||||
) | ||||
from . import ( | ||||
error, | ||||
|
r35603 | pycompat, | ||
|
r25991 | revlog, | ||
|
r30866 | scmutil, | ||
|
r25991 | util, | ||
) | ||||
|
r2778 | |||
def verify(repo): | ||||
|
r27849 | with repo.lock(): | ||
|
r27444 | return verifier(repo).verify() | ||
|
r4915 | |||
|
r17860 | def _normpath(f): | ||
# under hg < 2.4, convert didn't sanitize paths properly, so a | ||||
# converted repo may contain repeated slashes | ||||
while '//' in f: | ||||
f = f.replace('//', '/') | ||||
return f | ||||
|
r27443 | class verifier(object): | ||
|
r30866 | # The match argument is always None in hg core, but e.g. the narrowhg | ||
# extension will pass in a matcher here. | ||||
def __init__(self, repo, match=None): | ||||
|
r27444 | self.repo = repo.unfiltered() | ||
self.ui = repo.ui | ||||
|
r30866 | self.match = match or scmutil.matchall(repo) | ||
|
r27444 | self.badrevs = set() | ||
|
r27453 | self.errors = 0 | ||
self.warnings = 0 | ||||
|
r27444 | self.havecl = len(repo.changelog) > 0 | ||
|
r39280 | self.havemf = len(repo.manifestlog.getstorage(b'')) > 0 | ||
|
r27444 | self.revlogv1 = repo.changelog.version != revlog.REVLOGV0 | ||
|
r37318 | self.lrugetctx = util.lrucachefunc(repo.__getitem__) | ||
|
r27444 | self.refersmf = False | ||
|
r27445 | self.fncachewarned = False | ||
|
r32288 | # developer config: verify.skipflags | ||
self.skipflags = repo.ui.configint('verify', 'skipflags') | ||||
|
r37435 | self.warnorphanstorefiles = True | ||
|
r27444 | |||
|
r27446 | def warn(self, msg): | ||
self.ui.warn(msg + "\n") | ||||
|
r27453 | self.warnings += 1 | ||
|
r27446 | |||
|
r27447 | def err(self, linkrev, msg, filename=None): | ||
if linkrev is not None: | ||||
self.badrevs.add(linkrev) | ||||
|
r36203 | linkrev = "%d" % linkrev | ||
|
r27447 | else: | ||
linkrev = '?' | ||||
msg = "%s: %s" % (linkrev, msg) | ||||
if filename: | ||||
msg = "%s@%s" % (filename, msg) | ||||
self.ui.warn(" " + msg + "\n") | ||||
|
r27453 | self.errors += 1 | ||
|
r27447 | |||
|
r27448 | def exc(self, linkrev, msg, inst, filename=None): | ||
|
r36595 | fmsg = pycompat.bytestr(inst) | ||
if not fmsg: | ||||
fmsg = pycompat.byterepr(inst) | ||||
self.err(linkrev, "%s: %s" % (msg, fmsg), filename) | ||||
|
r27448 | |||
|
r27642 | def checklog(self, obj, name, linkrev): | ||
if not len(obj) and (self.havecl or self.havemf): | ||||
self.err(linkrev, _("empty or missing %s") % name) | ||||
return | ||||
d = obj.checksize() | ||||
if d[0]: | ||||
self.err(None, _("data length off by %d bytes") % d[0], name) | ||||
if d[1]: | ||||
self.err(None, _("index contains %d extra bytes") % d[1], name) | ||||
if obj.version != revlog.REVLOGV0: | ||||
if not self.revlogv1: | ||||
self.warn(_("warning: `%s' uses revlog format 1") % name) | ||||
elif self.revlogv1: | ||||
self.warn(_("warning: `%s' uses revlog format 0") % name) | ||||
|
r27643 | def checkentry(self, obj, i, node, seen, linkrevs, f): | ||
lr = obj.linkrev(obj.rev(node)) | ||||
if lr < 0 or (self.havecl and lr not in linkrevs): | ||||
if lr < 0 or lr >= len(self.repo.changelog): | ||||
msg = _("rev %d points to nonexistent changeset %d") | ||||
else: | ||||
msg = _("rev %d points to unexpected changeset %d") | ||||
self.err(None, msg % (i, lr), f) | ||||
if linkrevs: | ||||
if f and len(linkrevs) > 1: | ||||
try: | ||||
# attempt to filter down to real linkrevs | ||||
linkrevs = [l for l in linkrevs | ||||
if self.lrugetctx(l)[f].filenode() == node] | ||||
except Exception: | ||||
pass | ||||
|
r35603 | self.warn(_(" (expected %s)") % " ".join | ||
(map(pycompat.bytestr, linkrevs))) | ||||
|
r27643 | lr = None # can't be trusted | ||
try: | ||||
p1, p2 = obj.parents(node) | ||||
if p1 not in seen and p1 != nullid: | ||||
self.err(lr, _("unknown parent 1 %s of %s") % | ||||
(short(p1), short(node)), f) | ||||
if p2 not in seen and p2 != nullid: | ||||
self.err(lr, _("unknown parent 2 %s of %s") % | ||||
(short(p2), short(node)), f) | ||||
except Exception as inst: | ||||
self.exc(lr, _("checking parents of %s") % short(node), inst, f) | ||||
if node in seen: | ||||
self.err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f) | ||||
seen[node] = i | ||||
return lr | ||||
|
r27444 | def verify(self): | ||
repo = self.repo | ||||
|
r27648 | |||
|
r27443 | ui = repo.ui | ||
|
r2778 | |||
|
r27443 | if not repo.url().startswith('file:'): | ||
raise error.Abort(_("cannot verify bundle or remote repos")) | ||||
|
r6752 | |||
|
r27443 | if os.path.exists(repo.sjoin("journal")): | ||
ui.warn(_("abandoned transaction found - run hg recover\n")) | ||||
|
r27648 | if ui.verbose or not self.revlogv1: | ||
|
r27443 | ui.status(_("repository uses revlog format %d\n") % | ||
|
r27648 | (self.revlogv1 and 1 or 0)) | ||
|
r27443 | |||
|
r27695 | mflinkrevs, filelinkrevs = self._verifychangelog() | ||
|
r27647 | |||
|
r27695 | filenodes = self._verifymanifest(mflinkrevs) | ||
|
r28111 | del mflinkrevs | ||
|
r27647 | |||
|
r28111 | self._crosscheckfiles(filelinkrevs, filenodes) | ||
|
r27647 | |||
totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs) | ||||
|
r39525 | ui.status(_("checked %d changesets with %d changes to %d files\n") % | ||
(len(repo.changelog), filerevisions, totalfiles)) | ||||
|
r27647 | if self.warnings: | ||
ui.warn(_("%d warnings encountered!\n") % self.warnings) | ||||
if self.fncachewarned: | ||||
ui.warn(_('hint: run "hg debugrebuildfncache" to recover from ' | ||||
'corrupt fncache\n')) | ||||
if self.errors: | ||||
ui.warn(_("%d integrity errors encountered!\n") % self.errors) | ||||
|
r27648 | if self.badrevs: | ||
|
r27647 | ui.warn(_("(first damaged changeset appears to be %d)\n") | ||
|
r27648 | % min(self.badrevs)) | ||
|
r27647 | return 1 | ||
|
r27695 | def _verifychangelog(self): | ||
|
r27647 | ui = self.ui | ||
repo = self.repo | ||||
|
r30866 | match = self.match | ||
|
r27647 | cl = repo.changelog | ||
|
r27443 | ui.status(_("checking changesets\n")) | ||
|
r27695 | mflinkrevs = {} | ||
filelinkrevs = {} | ||||
|
r27443 | seen = {} | ||
|
r27642 | self.checklog(cl, "changelog", 0) | ||
|
r38416 | progress = ui.makeprogress(_('checking'), unit=_('changesets'), | ||
total=len(repo)) | ||||
|
r27443 | for i in repo: | ||
|
r38416 | progress.update(i) | ||
|
r27443 | n = cl.node(i) | ||
|
r27643 | self.checkentry(cl, i, n, seen, [i], "changelog") | ||
|
r2778 | |||
|
r27443 | try: | ||
changes = cl.read(n) | ||||
if changes[0] != nullid: | ||||
mflinkrevs.setdefault(changes[0], []).append(i) | ||||
|
r27444 | self.refersmf = True | ||
|
r27443 | for f in changes[3]: | ||
|
r30866 | if match(f): | ||
|
r27443 | filelinkrevs.setdefault(_normpath(f), []).append(i) | ||
except Exception as inst: | ||||
|
r27444 | self.refersmf = True | ||
|
r27448 | self.exc(i, _("unpacking changeset %s") % short(n), inst) | ||
|
r38416 | progress.complete() | ||
|
r27695 | return mflinkrevs, filelinkrevs | ||
|
r2778 | |||
|
r28205 | def _verifymanifest(self, mflinkrevs, dir="", storefiles=None, | ||
|
r38415 | subdirprogress=None): | ||
|
r27646 | repo = self.repo | ||
ui = self.ui | ||||
|
r30866 | match = self.match | ||
|
r30295 | mfl = self.repo.manifestlog | ||
|
r39280 | mf = mfl.getstorage(dir) | ||
|
r27646 | |||
|
r28203 | if not dir: | ||
self.ui.status(_("checking manifests\n")) | ||||
|
r27695 | filenodes = {} | ||
|
r28203 | subdirnodes = {} | ||
|
r27443 | seen = {} | ||
|
r28115 | label = "manifest" | ||
|
r28203 | if dir: | ||
label = dir | ||||
|
r28204 | revlogfiles = mf.files() | ||
storefiles.difference_update(revlogfiles) | ||||
|
r38415 | if subdirprogress: # should be true since we're in a subdirectory | ||
subdirprogress.increment() | ||||
|
r27444 | if self.refersmf: | ||
|
r27443 | # Do not check manifest if there are only changelog entries with | ||
# null manifests. | ||||
|
r28115 | self.checklog(mf, label, 0) | ||
|
r38416 | progress = ui.makeprogress(_('checking'), unit=_('manifests'), | ||
total=len(mf)) | ||||
|
r27443 | for i in mf: | ||
|
r28203 | if not dir: | ||
|
r38416 | progress.update(i) | ||
|
r27443 | n = mf.node(i) | ||
|
r28115 | lr = self.checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label) | ||
|
r27443 | if n in mflinkrevs: | ||
del mflinkrevs[n] | ||||
|
r28203 | elif dir: | ||
self.err(lr, _("%s not in parent-directory manifest") % | ||||
short(n), label) | ||||
|
r27443 | else: | ||
|
r28115 | self.err(lr, _("%s not in changesets") % short(n), label) | ||
|
r2778 | |||
|
r27443 | try: | ||
|
r30295 | mfdelta = mfl.get(dir, n).readdelta(shallow=True) | ||
for f, fn, fl in mfdelta.iterentries(): | ||||
|
r27443 | if not f: | ||
|
r28203 | self.err(lr, _("entry without name in manifest")) | ||
elif f == "/dev/null": # ignore this in very old repos | ||||
continue | ||||
fullpath = dir + _normpath(f) | ||||
if fl == 't': | ||||
|
r30866 | if not match.visitdir(fullpath): | ||
continue | ||||
|
r28203 | subdirnodes.setdefault(fullpath + '/', {}).setdefault( | ||
fn, []).append(lr) | ||||
else: | ||||
|
r30866 | if not match(fullpath): | ||
continue | ||||
|
r28203 | filenodes.setdefault(fullpath, {}).setdefault(fn, lr) | ||
|
r27443 | except Exception as inst: | ||
|
r28115 | self.exc(lr, _("reading delta %s") % short(n), inst, label) | ||
|
r28203 | if not dir: | ||
|
r38416 | progress.complete() | ||
|
r27443 | |||
|
r28111 | if self.havemf: | ||
for c, m in sorted([(c, m) for m in mflinkrevs | ||||
for c in mflinkrevs[m]]): | ||||
|
r28203 | if dir: | ||
self.err(c, _("parent-directory manifest refers to unknown " | ||||
"revision %s") % short(m), label) | ||||
else: | ||||
self.err(c, _("changeset refers to unknown revision %s") % | ||||
short(m), label) | ||||
if not dir and subdirnodes: | ||||
self.ui.status(_("checking directory manifests\n")) | ||||
|
r28204 | storefiles = set() | ||
|
r28205 | subdirs = set() | ||
|
r28204 | revlogv1 = self.revlogv1 | ||
for f, f2, size in repo.store.datafiles(): | ||||
if not f: | ||||
self.err(None, _("cannot decode filename '%s'") % f2) | ||||
elif (size > 0 or not revlogv1) and f.startswith('meta/'): | ||||
storefiles.add(_normpath(f)) | ||||
|
r28205 | subdirs.add(os.path.dirname(f)) | ||
|
r38415 | subdirprogress = ui.makeprogress(_('checking'), unit=_('manifests'), | ||
total=len(subdirs)) | ||||
|
r28204 | |||
|
r28203 | for subdir, linkrevs in subdirnodes.iteritems(): | ||
|
r28205 | subdirfilenodes = self._verifymanifest(linkrevs, subdir, storefiles, | ||
|
r38415 | subdirprogress) | ||
|
r28203 | for f, onefilenodes in subdirfilenodes.iteritems(): | ||
filenodes.setdefault(f, {}).update(onefilenodes) | ||||
|
r28111 | |||
|
r28204 | if not dir and subdirnodes: | ||
|
r38415 | subdirprogress.complete() | ||
|
r37435 | if self.warnorphanstorefiles: | ||
for f in sorted(storefiles): | ||||
self.warn(_("warning: orphan data file '%s'") % f) | ||||
|
r28204 | |||
|
r27695 | return filenodes | ||
|
r27645 | |||
|
r28111 | def _crosscheckfiles(self, filelinkrevs, filenodes): | ||
|
r27645 | repo = self.repo | ||
ui = self.ui | ||||
|
r27443 | ui.status(_("crosschecking files in changesets and manifests\n")) | ||
|
r2778 | |||
|
r28111 | total = len(filelinkrevs) + len(filenodes) | ||
|
r38416 | progress = ui.makeprogress(_('crosschecking'), total=total) | ||
|
r27645 | if self.havemf: | ||
|
r27443 | for f in sorted(filelinkrevs): | ||
|
r38416 | progress.increment() | ||
|
r27443 | if f not in filenodes: | ||
lr = filelinkrevs[f][0] | ||||
|
r27447 | self.err(lr, _("in changeset but not in manifest"), f) | ||
|
r6892 | |||
|
r27645 | if self.havecl: | ||
|
r27443 | for f in sorted(filenodes): | ||
|
r38416 | progress.increment() | ||
|
r27443 | if f not in filelinkrevs: | ||
try: | ||||
fl = repo.file(f) | ||||
lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]]) | ||||
except Exception: | ||||
lr = None | ||||
|
r27447 | self.err(lr, _("in manifest but not in changeset"), f) | ||
|
r27443 | |||
|
r38416 | progress.complete() | ||
|
r8291 | |||
|
r27644 | def _verifyfiles(self, filenodes, filelinkrevs): | ||
repo = self.repo | ||||
ui = self.ui | ||||
lrugetctx = self.lrugetctx | ||||
revlogv1 = self.revlogv1 | ||||
havemf = self.havemf | ||||
|
r27443 | ui.status(_("checking files\n")) | ||
|
r8291 | |||
|
r27443 | storefiles = set() | ||
for f, f2, size in repo.store.datafiles(): | ||||
if not f: | ||||
|
r27447 | self.err(None, _("cannot decode filename '%s'") % f2) | ||
|
r28007 | elif (size > 0 or not revlogv1) and f.startswith('data/'): | ||
|
r27443 | storefiles.add(_normpath(f)) | ||
|
r6892 | |||
|
r27443 | files = sorted(set(filenodes) | set(filelinkrevs)) | ||
|
r27644 | revisions = 0 | ||
|
r38416 | progress = ui.makeprogress(_('checking'), unit=_('files'), | ||
total=len(files)) | ||||
|
r27443 | for i, f in enumerate(files): | ||
|
r38416 | progress.update(i, item=f) | ||
|
r6892 | try: | ||
|
r27443 | linkrevs = filelinkrevs[f] | ||
|
r6892 | except KeyError: | ||
|
r27443 | # in manifest but not in changelog | ||
linkrevs = [] | ||||
|
r2778 | |||
|
r27443 | if linkrevs: | ||
lr = linkrevs[0] | ||||
else: | ||||
lr = None | ||||
|
r2778 | |||
|
r3744 | try: | ||
|
r27443 | fl = repo.file(f) | ||
|
r39813 | except error.StorageError as e: | ||
|
r27447 | self.err(lr, _("broken revlog! (%s)") % e, f) | ||
|
r27443 | continue | ||
for ff in fl.files(): | ||||
try: | ||||
storefiles.remove(ff) | ||||
except KeyError: | ||||
|
r37435 | if self.warnorphanstorefiles: | ||
self.warn(_(" warning: revlog '%s' not in fncache!") % | ||||
ff) | ||||
self.fncachewarned = True | ||||
|
r27443 | |||
|
r27642 | self.checklog(fl, f, lr) | ||
|
r27443 | seen = {} | ||
rp = None | ||||
for i in fl: | ||||
revisions += 1 | ||||
n = fl.node(i) | ||||
|
r27643 | lr = self.checkentry(fl, i, n, seen, linkrevs, f) | ||
|
r27443 | if f in filenodes: | ||
if havemf and n not in filenodes[f]: | ||||
|
r27447 | self.err(lr, _("%s not in manifests") % (short(n)), f) | ||
|
r6534 | else: | ||
|
r27443 | del filenodes[f][n] | ||
|
r31761 | # Verify contents. 4 cases to care about: | ||
# | ||||
# common: the most common case | ||||
# rename: with a rename | ||||
# meta: file content starts with b'\1\n', the metadata | ||||
# header defined in filelog.py, but without a rename | ||||
# ext: content stored externally | ||||
# | ||||
# More formally, their differences are shown below: | ||||
# | ||||
# | common | rename | meta | ext | ||||
# ------------------------------------------------------- | ||||
# flags() | 0 | 0 | 0 | not 0 | ||||
# renamed() | False | True | False | ? | ||||
# rawtext[0:2]=='\1\n'| False | True | True | ? | ||||
# | ||||
# "rawtext" means the raw text stored in revlog data, which | ||||
# could be retrieved by "revision(rev, raw=True)". "text" | ||||
# mentioned below is "revision(rev, raw=False)". | ||||
# | ||||
# There are 3 different lengths stored physically: | ||||
# 1. L1: rawsize, stored in revlog index | ||||
# 2. L2: len(rawtext), stored in revlog data | ||||
# 3. L3: len(text), stored in revlog data if flags==0, or | ||||
# possibly somewhere else if flags!=0 | ||||
# | ||||
# L1 should be equal to L2. L3 could be different from them. | ||||
# "text" may or may not affect commit hash depending on flag | ||||
# processors (see revlog.addflagprocessor). | ||||
# | ||||
# | common | rename | meta | ext | ||||
# ------------------------------------------------- | ||||
# rawsize() | L1 | L1 | L1 | L1 | ||||
# size() | L1 | L2-LM | L1(*) | L1 (?) | ||||
# len(rawtext) | L2 | L2 | L2 | L2 | ||||
# len(text) | L2 | L2 | L2 | L3 | ||||
# len(read()) | L2 | L2-LM | L2-LM | L3 (?) | ||||
# | ||||
# LM: length of metadata, depending on rawtext | ||||
# (*): not ideal, see comment in filelog.size | ||||
# (?): could be "- len(meta)" if the resolved content has | ||||
# rename metadata | ||||
# | ||||
# Checks needed to be done: | ||||
# 1. length check: L1 == L2, in all cases. | ||||
# 2. hash check: depending on flag processor, we may need to | ||||
# use either "text" (external), or "rawtext" (in revlog). | ||||
|
r27443 | try: | ||
|
r32288 | skipflags = self.skipflags | ||
if skipflags: | ||||
skipflags &= fl.flags(i) | ||||
if not skipflags: | ||||
fl.read(n) # side effect: read content and do checkhash | ||||
rp = fl.renamed(n) | ||||
|
r32250 | # the "L1 == L2" check | ||
l1 = fl.rawsize(i) | ||||
l2 = len(fl.revision(n, raw=True)) | ||||
if l1 != l2: | ||||
self.err(lr, _("unpacked size is %s, %s expected") % | ||||
(l2, l1), f) | ||||
|
r27443 | except error.CensoredNodeError: | ||
# experimental config: censor.policy | ||||
|
r33499 | if ui.config("censor", "policy") == "abort": | ||
|
r27447 | self.err(lr, _("censored file data"), f) | ||
|
r27443 | except Exception as inst: | ||
|
r27448 | self.exc(lr, _("unpacking %s") % short(n), inst, f) | ||
|
r3744 | |||
|
r27443 | # check renames | ||
try: | ||||
if rp: | ||||
if lr is not None and ui.verbose: | ||||
ctx = lrugetctx(lr) | ||||
|
r36357 | if not any(rp[0] in pctx for pctx in ctx.parents()): | ||
|
r27446 | self.warn(_("warning: copy source of '%s' not" | ||
" in parents of %s") % (f, ctx)) | ||||
|
r27443 | fl2 = repo.file(rp[0]) | ||
if not len(fl2): | ||||
|
r27447 | self.err(lr, _("empty or missing copy source " | ||
"revlog %s:%s") % (rp[0], short(rp[1])), f) | ||||
|
r27443 | elif rp[1] == nullid: | ||
ui.note(_("warning: %s@%s: copy source" | ||||
" revision is nullid %s:%s\n") | ||||
% (f, lr, rp[0], short(rp[1]))) | ||||
else: | ||||
fl2.rev(rp[1]) | ||||
except Exception as inst: | ||||
|
r27448 | self.exc(lr, _("checking rename of %s") % short(n), inst, f) | ||
|
r6892 | |||
|
r27443 | # cross-check | ||
if f in filenodes: | ||||
|
r30393 | fns = [(v, k) for k, v in filenodes[f].iteritems()] | ||
|
r27443 | for lr, node in sorted(fns): | ||
|
r28114 | self.err(lr, _("manifest refers to unknown revision %s") % | ||
short(node), f) | ||||
|
r38416 | progress.complete() | ||
|
r27443 | |||
|
r37435 | if self.warnorphanstorefiles: | ||
for f in sorted(storefiles): | ||||
self.warn(_("warning: orphan data file '%s'") % f) | ||||
|
r27443 | |||
|
r27644 | return len(files), revisions | ||