verify.py
381 lines
| 13.7 KiB
| text/x-python
|
PythonLexer
/ mercurial / verify.py
|
r2778 | # verify.py - repository integrity checking for Mercurial | ||
# | ||||
|
r4635 | # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com> | ||
|
r2778 | # | ||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r2778 | |||
|
r25991 | from __future__ import absolute_import | ||
|
r17860 | import os | ||
|
r25991 | |||
from .i18n import _ | ||||
from .node import ( | ||||
nullid, | ||||
short, | ||||
) | ||||
from . import ( | ||||
error, | ||||
revlog, | ||||
util, | ||||
) | ||||
|
r2778 | |||
def verify(repo): | ||||
|
r27849 | with repo.lock(): | ||
|
r27444 | return verifier(repo).verify() | ||
|
r4915 | |||
|
r17860 | def _normpath(f): | ||
# under hg < 2.4, convert didn't sanitize paths properly, so a | ||||
# converted repo may contain repeated slashes | ||||
while '//' in f: | ||||
f = f.replace('//', '/') | ||||
return f | ||||
|
r26900 | def _validpath(repo, path): | ||
"""Returns False if a path should NOT be treated as part of a repo. | ||||
For all in-core cases, this returns True, as we have no way for a | ||||
path to be mentioned in the history but not actually be | ||||
relevant. For narrow clones, this is important because many | ||||
filelogs will be missing, and changelog entries may mention | ||||
modified files that are outside the narrow scope. | ||||
""" | ||||
return True | ||||
|
r27443 | class verifier(object): | ||
|
r27444 | def __init__(self, repo): | ||
self.repo = repo.unfiltered() | ||||
self.ui = repo.ui | ||||
self.badrevs = set() | ||||
|
r27453 | self.errors = 0 | ||
self.warnings = 0 | ||||
|
r27444 | self.havecl = len(repo.changelog) > 0 | ||
self.havemf = len(repo.manifest) > 0 | ||||
self.revlogv1 = repo.changelog.version != revlog.REVLOGV0 | ||||
self.lrugetctx = util.lrucachefunc(repo.changectx) | ||||
self.refersmf = False | ||||
|
r27445 | self.fncachewarned = False | ||
|
r27444 | |||
|
r27446 | def warn(self, msg): | ||
self.ui.warn(msg + "\n") | ||||
|
r27453 | self.warnings += 1 | ||
|
r27446 | |||
|
r27447 | def err(self, linkrev, msg, filename=None): | ||
if linkrev is not None: | ||||
self.badrevs.add(linkrev) | ||||
else: | ||||
linkrev = '?' | ||||
msg = "%s: %s" % (linkrev, msg) | ||||
if filename: | ||||
msg = "%s@%s" % (filename, msg) | ||||
self.ui.warn(" " + msg + "\n") | ||||
|
r27453 | self.errors += 1 | ||
|
r27447 | |||
|
r27448 | def exc(self, linkrev, msg, inst, filename=None): | ||
if not str(inst): | ||||
inst = repr(inst) | ||||
self.err(linkrev, "%s: %s" % (msg, inst), filename) | ||||
|
r27642 | def checklog(self, obj, name, linkrev): | ||
if not len(obj) and (self.havecl or self.havemf): | ||||
self.err(linkrev, _("empty or missing %s") % name) | ||||
return | ||||
d = obj.checksize() | ||||
if d[0]: | ||||
self.err(None, _("data length off by %d bytes") % d[0], name) | ||||
if d[1]: | ||||
self.err(None, _("index contains %d extra bytes") % d[1], name) | ||||
if obj.version != revlog.REVLOGV0: | ||||
if not self.revlogv1: | ||||
self.warn(_("warning: `%s' uses revlog format 1") % name) | ||||
elif self.revlogv1: | ||||
self.warn(_("warning: `%s' uses revlog format 0") % name) | ||||
|
r27643 | def checkentry(self, obj, i, node, seen, linkrevs, f): | ||
lr = obj.linkrev(obj.rev(node)) | ||||
if lr < 0 or (self.havecl and lr not in linkrevs): | ||||
if lr < 0 or lr >= len(self.repo.changelog): | ||||
msg = _("rev %d points to nonexistent changeset %d") | ||||
else: | ||||
msg = _("rev %d points to unexpected changeset %d") | ||||
self.err(None, msg % (i, lr), f) | ||||
if linkrevs: | ||||
if f and len(linkrevs) > 1: | ||||
try: | ||||
# attempt to filter down to real linkrevs | ||||
linkrevs = [l for l in linkrevs | ||||
if self.lrugetctx(l)[f].filenode() == node] | ||||
except Exception: | ||||
pass | ||||
self.warn(_(" (expected %s)") % " ".join(map(str, linkrevs))) | ||||
lr = None # can't be trusted | ||||
try: | ||||
p1, p2 = obj.parents(node) | ||||
if p1 not in seen and p1 != nullid: | ||||
self.err(lr, _("unknown parent 1 %s of %s") % | ||||
(short(p1), short(node)), f) | ||||
if p2 not in seen and p2 != nullid: | ||||
self.err(lr, _("unknown parent 2 %s of %s") % | ||||
(short(p2), short(node)), f) | ||||
except Exception as inst: | ||||
self.exc(lr, _("checking parents of %s") % short(node), inst, f) | ||||
if node in seen: | ||||
self.err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f) | ||||
seen[node] = i | ||||
return lr | ||||
|
r27444 | def verify(self): | ||
repo = self.repo | ||||
|
r27648 | |||
|
r27443 | ui = repo.ui | ||
|
r2778 | |||
|
r27443 | if not repo.url().startswith('file:'): | ||
raise error.Abort(_("cannot verify bundle or remote repos")) | ||||
|
r6752 | |||
|
r27443 | if os.path.exists(repo.sjoin("journal")): | ||
ui.warn(_("abandoned transaction found - run hg recover\n")) | ||||
|
r27648 | if ui.verbose or not self.revlogv1: | ||
|
r27443 | ui.status(_("repository uses revlog format %d\n") % | ||
|
r27648 | (self.revlogv1 and 1 or 0)) | ||
|
r27443 | |||
|
r27695 | mflinkrevs, filelinkrevs = self._verifychangelog() | ||
|
r27647 | |||
|
r27695 | filenodes = self._verifymanifest(mflinkrevs) | ||
|
r28111 | del mflinkrevs | ||
|
r27647 | |||
|
r28111 | self._crosscheckfiles(filelinkrevs, filenodes) | ||
|
r27647 | |||
totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs) | ||||
ui.status(_("%d files, %d changesets, %d total revisions\n") % | ||||
|
r27648 | (totalfiles, len(repo.changelog), filerevisions)) | ||
|
r27647 | if self.warnings: | ||
ui.warn(_("%d warnings encountered!\n") % self.warnings) | ||||
if self.fncachewarned: | ||||
ui.warn(_('hint: run "hg debugrebuildfncache" to recover from ' | ||||
'corrupt fncache\n')) | ||||
if self.errors: | ||||
ui.warn(_("%d integrity errors encountered!\n") % self.errors) | ||||
|
r27648 | if self.badrevs: | ||
|
r27647 | ui.warn(_("(first damaged changeset appears to be %d)\n") | ||
|
r27648 | % min(self.badrevs)) | ||
|
r27647 | return 1 | ||
|
r27695 | def _verifychangelog(self): | ||
|
r27647 | ui = self.ui | ||
repo = self.repo | ||||
cl = repo.changelog | ||||
|
r27443 | ui.status(_("checking changesets\n")) | ||
|
r27695 | mflinkrevs = {} | ||
filelinkrevs = {} | ||||
|
r27443 | seen = {} | ||
|
r27642 | self.checklog(cl, "changelog", 0) | ||
|
r27443 | total = len(repo) | ||
for i in repo: | ||||
ui.progress(_('checking'), i, total=total, unit=_('changesets')) | ||||
n = cl.node(i) | ||||
|
r27643 | self.checkentry(cl, i, n, seen, [i], "changelog") | ||
|
r2778 | |||
|
r27443 | try: | ||
changes = cl.read(n) | ||||
if changes[0] != nullid: | ||||
mflinkrevs.setdefault(changes[0], []).append(i) | ||||
|
r27444 | self.refersmf = True | ||
|
r27443 | for f in changes[3]: | ||
|
r26900 | if _validpath(repo, f): | ||
|
r27443 | filelinkrevs.setdefault(_normpath(f), []).append(i) | ||
except Exception as inst: | ||||
|
r27444 | self.refersmf = True | ||
|
r27448 | self.exc(i, _("unpacking changeset %s") % short(n), inst) | ||
|
r27443 | ui.progress(_('checking'), None) | ||
|
r27695 | return mflinkrevs, filelinkrevs | ||
|
r2778 | |||
|
r27695 | def _verifymanifest(self, mflinkrevs): | ||
|
r27646 | repo = self.repo | ||
ui = self.ui | ||||
mf = self.repo.manifest | ||||
|
r27443 | ui.status(_("checking manifests\n")) | ||
|
r27695 | filenodes = {} | ||
|
r27443 | seen = {} | ||
|
r28115 | label = "manifest" | ||
|
r27444 | if self.refersmf: | ||
|
r27443 | # Do not check manifest if there are only changelog entries with | ||
# null manifests. | ||||
|
r28115 | self.checklog(mf, label, 0) | ||
|
r27443 | total = len(mf) | ||
for i in mf: | ||||
ui.progress(_('checking'), i, total=total, unit=_('manifests')) | ||||
n = mf.node(i) | ||||
|
r28115 | lr = self.checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label) | ||
|
r27443 | if n in mflinkrevs: | ||
del mflinkrevs[n] | ||||
else: | ||||
|
r28115 | self.err(lr, _("%s not in changesets") % short(n), label) | ||
|
r2778 | |||
|
r27443 | try: | ||
for f, fn in mf.readdelta(n).iteritems(): | ||||
if not f: | ||||
|
r27447 | self.err(lr, _("file without name in manifest")) | ||
|
r27443 | elif f != "/dev/null": # ignore this in very old repos | ||
if _validpath(repo, f): | ||||
filenodes.setdefault( | ||||
_normpath(f), {}).setdefault(fn, lr) | ||||
except Exception as inst: | ||||
|
r28115 | self.exc(lr, _("reading delta %s") % short(n), inst, label) | ||
|
r27443 | ui.progress(_('checking'), None) | ||
|
r28111 | if self.havemf: | ||
for c, m in sorted([(c, m) for m in mflinkrevs | ||||
for c in mflinkrevs[m]]): | ||||
|
r28113 | self.err(c, _("changeset refers to unknown revision %s") % | ||
|
r28115 | short(m), label) | ||
|
r28111 | |||
|
r27695 | return filenodes | ||
|
r27645 | |||
|
r28111 | def _crosscheckfiles(self, filelinkrevs, filenodes): | ||
|
r27645 | repo = self.repo | ||
ui = self.ui | ||||
|
r27443 | ui.status(_("crosschecking files in changesets and manifests\n")) | ||
|
r2778 | |||
|
r28111 | total = len(filelinkrevs) + len(filenodes) | ||
|
r27443 | count = 0 | ||
|
r27645 | if self.havemf: | ||
|
r27443 | for f in sorted(filelinkrevs): | ||
count += 1 | ||||
ui.progress(_('crosschecking'), count, total=total) | ||||
if f not in filenodes: | ||||
lr = filelinkrevs[f][0] | ||||
|
r27447 | self.err(lr, _("in changeset but not in manifest"), f) | ||
|
r6892 | |||
|
r27645 | if self.havecl: | ||
|
r27443 | for f in sorted(filenodes): | ||
count += 1 | ||||
ui.progress(_('crosschecking'), count, total=total) | ||||
if f not in filelinkrevs: | ||||
try: | ||||
fl = repo.file(f) | ||||
lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]]) | ||||
except Exception: | ||||
lr = None | ||||
|
r27447 | self.err(lr, _("in manifest but not in changeset"), f) | ||
|
r27443 | |||
ui.progress(_('crosschecking'), None) | ||||
|
r8291 | |||
|
r27644 | def _verifyfiles(self, filenodes, filelinkrevs): | ||
repo = self.repo | ||||
ui = self.ui | ||||
lrugetctx = self.lrugetctx | ||||
revlogv1 = self.revlogv1 | ||||
havemf = self.havemf | ||||
|
r27443 | ui.status(_("checking files\n")) | ||
|
r8291 | |||
|
r27443 | storefiles = set() | ||
for f, f2, size in repo.store.datafiles(): | ||||
if not f: | ||||
|
r27447 | self.err(None, _("cannot decode filename '%s'") % f2) | ||
|
r28007 | elif (size > 0 or not revlogv1) and f.startswith('data/'): | ||
|
r27443 | storefiles.add(_normpath(f)) | ||
|
r6892 | |||
|
r27443 | files = sorted(set(filenodes) | set(filelinkrevs)) | ||
total = len(files) | ||||
|
r27644 | revisions = 0 | ||
|
r27443 | for i, f in enumerate(files): | ||
ui.progress(_('checking'), i, item=f, total=total) | ||||
|
r6892 | try: | ||
|
r27443 | linkrevs = filelinkrevs[f] | ||
|
r6892 | except KeyError: | ||
|
r27443 | # in manifest but not in changelog | ||
linkrevs = [] | ||||
|
r2778 | |||
|
r27443 | if linkrevs: | ||
lr = linkrevs[0] | ||||
else: | ||||
lr = None | ||||
|
r2778 | |||
|
r3744 | try: | ||
|
r27443 | fl = repo.file(f) | ||
except error.RevlogError as e: | ||||
|
r27447 | self.err(lr, _("broken revlog! (%s)") % e, f) | ||
|
r27443 | continue | ||
for ff in fl.files(): | ||||
try: | ||||
storefiles.remove(ff) | ||||
except KeyError: | ||||
|
r27446 | self.warn(_(" warning: revlog '%s' not in fncache!") % ff) | ||
|
r27445 | self.fncachewarned = True | ||
|
r27443 | |||
|
r27642 | self.checklog(fl, f, lr) | ||
|
r27443 | seen = {} | ||
rp = None | ||||
for i in fl: | ||||
revisions += 1 | ||||
n = fl.node(i) | ||||
|
r27643 | lr = self.checkentry(fl, i, n, seen, linkrevs, f) | ||
|
r27443 | if f in filenodes: | ||
if havemf and n not in filenodes[f]: | ||||
|
r27447 | self.err(lr, _("%s not in manifests") % (short(n)), f) | ||
|
r6534 | else: | ||
|
r27443 | del filenodes[f][n] | ||
# verify contents | ||||
try: | ||||
l = len(fl.read(n)) | ||||
rp = fl.renamed(n) | ||||
if l != fl.size(i): | ||||
if len(fl.revision(n)) != fl.size(i): | ||||
|
r27447 | self.err(lr, _("unpacked size is %s, %s expected") % | ||
(l, fl.size(i)), f) | ||||
|
r27443 | except error.CensoredNodeError: | ||
# experimental config: censor.policy | ||||
if ui.config("censor", "policy", "abort") == "abort": | ||||
|
r27447 | self.err(lr, _("censored file data"), f) | ||
|
r27443 | except Exception as inst: | ||
|
r27448 | self.exc(lr, _("unpacking %s") % short(n), inst, f) | ||
|
r3744 | |||
|
r27443 | # check renames | ||
try: | ||||
if rp: | ||||
if lr is not None and ui.verbose: | ||||
ctx = lrugetctx(lr) | ||||
found = False | ||||
for pctx in ctx.parents(): | ||||
if rp[0] in pctx: | ||||
found = True | ||||
break | ||||
if not found: | ||||
|
r27446 | self.warn(_("warning: copy source of '%s' not" | ||
" in parents of %s") % (f, ctx)) | ||||
|
r27443 | fl2 = repo.file(rp[0]) | ||
if not len(fl2): | ||||
|
r27447 | self.err(lr, _("empty or missing copy source " | ||
"revlog %s:%s") % (rp[0], short(rp[1])), f) | ||||
|
r27443 | elif rp[1] == nullid: | ||
ui.note(_("warning: %s@%s: copy source" | ||||
" revision is nullid %s:%s\n") | ||||
% (f, lr, rp[0], short(rp[1]))) | ||||
else: | ||||
fl2.rev(rp[1]) | ||||
except Exception as inst: | ||||
|
r27448 | self.exc(lr, _("checking rename of %s") % short(n), inst, f) | ||
|
r6892 | |||
|
r27443 | # cross-check | ||
if f in filenodes: | ||||
fns = [(lr, n) for n, lr in filenodes[f].iteritems()] | ||||
for lr, node in sorted(fns): | ||||
|
r28114 | self.err(lr, _("manifest refers to unknown revision %s") % | ||
short(node), f) | ||||
|
r27443 | ui.progress(_('checking'), None) | ||
for f in storefiles: | ||||
|
r27446 | self.warn(_("warning: orphan revlog '%s'") % f) | ||
|
r27443 | |||
|
r27644 | return len(files), revisions | ||