verify: include "manifest" prefix in a few more places

We include the "manifest" prefix on most other errors, so it seems
consistent to add them to the remaining messages too. Also, having the
"manifest" prefix will be more consistent with having the directory
prefix there when we add support for treemanifests. With the
"manifest" at the beginning, let's remove the now-redundant
"manifest" in the message itself.

mercurial/verify.py

@@ -1,381 +1,381 b''
 # verify.py - repository integrity checking for Mercurial
 #
 # Copyright 2006, 2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 
 from .i18n import _
 from .node import (
     nullid,
     short,
 )
 
 from . import (
     error,
     revlog,
     util,
 )
 
 def verify(repo):
     with repo.lock():
         return verifier(repo).verify()
 
 def _normpath(f):
     # under hg < 2.4, convert didn't sanitize paths properly, so a
     # converted repo may contain repeated slashes
     while '//' in f:
         f = f.replace('//', '/')
     return f
 
 def _validpath(repo, path):
     """Returns False if a path should NOT be treated as part of a repo.
 
     For all in-core cases, this returns True, as we have no way for a
     path to be mentioned in the history but not actually be
     relevant. For narrow clones, this is important because many
     filelogs will be missing, and changelog entries may mention
     modified files that are outside the narrow scope.
     """
     return True
 
 class verifier(object):
     def __init__(self, repo):
         self.repo = repo.unfiltered()
         self.ui = repo.ui
         self.badrevs = set()
         self.errors = 0
         self.warnings = 0
         self.havecl = len(repo.changelog) > 0
         self.havemf = len(repo.manifest) > 0
         self.revlogv1 = repo.changelog.version != revlog.REVLOGV0
         self.lrugetctx = util.lrucachefunc(repo.changectx)
         self.refersmf = False
         self.fncachewarned = False
 
     def warn(self, msg):
         self.ui.warn(msg + "\n")
         self.warnings += 1
 
     def err(self, linkrev, msg, filename=None):
         if linkrev is not None:
             self.badrevs.add(linkrev)
         else:
             linkrev = '?'
         msg = "%s: %s" % (linkrev, msg)
         if filename:
             msg = "%s@%s" % (filename, msg)
         self.ui.warn(" " + msg + "\n")
         self.errors += 1
 
     def exc(self, linkrev, msg, inst, filename=None):
         if not str(inst):
             inst = repr(inst)
         self.err(linkrev, "%s: %s" % (msg, inst), filename)
 
     def checklog(self, obj, name, linkrev):
         if not len(obj) and (self.havecl or self.havemf):
             self.err(linkrev, _("empty or missing %s") % name)
             return
 
         d = obj.checksize()
         if d[0]:
             self.err(None, _("data length off by %d bytes") % d[0], name)
         if d[1]:
             self.err(None, _("index contains %d extra bytes") % d[1], name)
 
         if obj.version != revlog.REVLOGV0:
             if not self.revlogv1:
                 self.warn(_("warning: `%s' uses revlog format 1") % name)
         elif self.revlogv1:
             self.warn(_("warning: `%s' uses revlog format 0") % name)
 
     def checkentry(self, obj, i, node, seen, linkrevs, f):
         lr = obj.linkrev(obj.rev(node))
         if lr < 0 or (self.havecl and lr not in linkrevs):
             if lr < 0 or lr >= len(self.repo.changelog):
                 msg = _("rev %d points to nonexistent changeset %d")
             else:
                 msg = _("rev %d points to unexpected changeset %d")
             self.err(None, msg % (i, lr), f)
             if linkrevs:
                 if f and len(linkrevs) > 1:
                     try:
                         # attempt to filter down to real linkrevs
                         linkrevs = [l for l in linkrevs
                                     if self.lrugetctx(l)[f].filenode() == node]
                     except Exception:
                         pass
                 self.warn(_(" (expected %s)") % " ".join(map(str, linkrevs)))
             lr = None # can't be trusted
 
         try:
             p1, p2 = obj.parents(node)
             if p1 not in seen and p1 != nullid:
                 self.err(lr, _("unknown parent 1 %s of %s") %
                     (short(p1), short(node)), f)
             if p2 not in seen and p2 != nullid:
                 self.err(lr, _("unknown parent 2 %s of %s") %
                     (short(p2), short(node)), f)
         except Exception as inst:
             self.exc(lr, _("checking parents of %s") % short(node), inst, f)
 
         if node in seen:
             self.err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f)
         seen[node] = i
         return lr
 
     def verify(self):
         repo = self.repo
 
         ui = repo.ui
 
         if not repo.url().startswith('file:'):
             raise error.Abort(_("cannot verify bundle or remote repos"))
 
         if os.path.exists(repo.sjoin("journal")):
             ui.warn(_("abandoned transaction found - run hg recover\n"))
 
         if ui.verbose or not self.revlogv1:
             ui.status(_("repository uses revlog format %d\n") %
                            (self.revlogv1 and 1 or 0))
 
         mflinkrevs, filelinkrevs = self._verifychangelog()
 
         filenodes = self._verifymanifest(mflinkrevs)
         del mflinkrevs
 
         self._crosscheckfiles(filelinkrevs, filenodes)
 
         totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs)
 
         ui.status(_("%d files, %d changesets, %d total revisions\n") %
                        (totalfiles, len(repo.changelog), filerevisions))
         if self.warnings:
             ui.warn(_("%d warnings encountered!\n") % self.warnings)
         if self.fncachewarned:
             ui.warn(_('hint: run "hg debugrebuildfncache" to recover from '
                       'corrupt fncache\n'))
         if self.errors:
             ui.warn(_("%d integrity errors encountered!\n") % self.errors)
             if self.badrevs:
                 ui.warn(_("(first damaged changeset appears to be %d)\n")
                         % min(self.badrevs))
             return 1
 
     def _verifychangelog(self):
         ui = self.ui
         repo = self.repo
         cl = repo.changelog
 
         ui.status(_("checking changesets\n"))
         mflinkrevs = {}
         filelinkrevs = {}
         seen = {}
         self.checklog(cl, "changelog", 0)
         total = len(repo)
         for i in repo:
             ui.progress(_('checking'), i, total=total, unit=_('changesets'))
             n = cl.node(i)
             self.checkentry(cl, i, n, seen, [i], "changelog")
 
             try:
                 changes = cl.read(n)
                 if changes[0] != nullid:
                     mflinkrevs.setdefault(changes[0], []).append(i)
                     self.refersmf = True
                 for f in changes[3]:
                     if _validpath(repo, f):
                         filelinkrevs.setdefault(_normpath(f), []).append(i)
             except Exception as inst:
                 self.refersmf = True
                 self.exc(i, _("unpacking changeset %s") % short(n), inst)
         ui.progress(_('checking'), None)
         return mflinkrevs, filelinkrevs
 
     def _verifymanifest(self, mflinkrevs):
         repo = self.repo
         ui = self.ui
         mf = self.repo.manifest
 
         ui.status(_("checking manifests\n"))
         filenodes = {}
         seen = {}
         if self.refersmf:
             # Do not check manifest if there are only changelog entries with
             # null manifests.
             self.checklog(mf, "manifest", 0)
         total = len(mf)
         for i in mf:
             ui.progress(_('checking'), i, total=total, unit=_('manifests'))
             n = mf.node(i)
             lr = self.checkentry(mf, i, n, seen, mflinkrevs.get(n, []),
                                  "manifest")
             if n in mflinkrevs:
                 del mflinkrevs[n]
             else:
                 self.err(lr, _("%s not in changesets") % short(n), "manifest")
 
             try:
                 for f, fn in mf.readdelta(n).iteritems():
                     if not f:
                         self.err(lr, _("file without name in manifest"))
                     elif f != "/dev/null": # ignore this in very old repos
                         if _validpath(repo, f):
                             filenodes.setdefault(
                                 _normpath(f), {}).setdefault(fn, lr)
             except Exception as inst:
-                self.exc(lr, _("reading manifest delta %s") % short(n), inst)
+                self.exc(lr, _("reading delta %s") % short(n), inst)
         ui.progress(_('checking'), None)
 
         if self.havemf:
             for c, m in sorted([(c, m) for m in mflinkrevs
                         for c in mflinkrevs[m]]):
-                self.err(c, _("changeset refers to unknown manifest %s") %
-                         short(m))
+                self.err(c, _("changeset refers to unknown revision %s") %
+                         short(m), "manifest")
 
         return filenodes
 
     def _crosscheckfiles(self, filelinkrevs, filenodes):
         repo = self.repo
         ui = self.ui
         ui.status(_("crosschecking files in changesets and manifests\n"))
 
         total = len(filelinkrevs) + len(filenodes)
         count = 0
         if self.havemf:
             for f in sorted(filelinkrevs):
                 count += 1
                 ui.progress(_('crosschecking'), count, total=total)
                 if f not in filenodes:
                     lr = filelinkrevs[f][0]
                     self.err(lr, _("in changeset but not in manifest"), f)
 
         if self.havecl:
             for f in sorted(filenodes):
                 count += 1
                 ui.progress(_('crosschecking'), count, total=total)
                 if f not in filelinkrevs:
                     try:
                         fl = repo.file(f)
                         lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]])
                     except Exception:
                         lr = None
                     self.err(lr, _("in manifest but not in changeset"), f)
 
         ui.progress(_('crosschecking'), None)
 
     def _verifyfiles(self, filenodes, filelinkrevs):
         repo = self.repo
         ui = self.ui
         lrugetctx = self.lrugetctx
         revlogv1 = self.revlogv1
         havemf = self.havemf
         ui.status(_("checking files\n"))
 
         storefiles = set()
         for f, f2, size in repo.store.datafiles():
             if not f:
                 self.err(None, _("cannot decode filename '%s'") % f2)
             elif (size > 0 or not revlogv1) and f.startswith('data/'):
                 storefiles.add(_normpath(f))
 
         files = sorted(set(filenodes) | set(filelinkrevs))
         total = len(files)
         revisions = 0
         for i, f in enumerate(files):
             ui.progress(_('checking'), i, item=f, total=total)
             try:
                 linkrevs = filelinkrevs[f]
             except KeyError:
                 # in manifest but not in changelog
                 linkrevs = []
 
             if linkrevs:
                 lr = linkrevs[0]
             else:
                 lr = None
 
             try:
                 fl = repo.file(f)
             except error.RevlogError as e:
                 self.err(lr, _("broken revlog! (%s)") % e, f)
                 continue
 
             for ff in fl.files():
                 try:
                     storefiles.remove(ff)
                 except KeyError:
                     self.warn(_(" warning: revlog '%s' not in fncache!") % ff)
                     self.fncachewarned = True
 
             self.checklog(fl, f, lr)
             seen = {}
             rp = None
             for i in fl:
                 revisions += 1
                 n = fl.node(i)
                 lr = self.checkentry(fl, i, n, seen, linkrevs, f)
                 if f in filenodes:
                     if havemf and n not in filenodes[f]:
                         self.err(lr, _("%s not in manifests") % (short(n)), f)
                     else:
                         del filenodes[f][n]
 
                 # verify contents
                 try:
                     l = len(fl.read(n))
                     rp = fl.renamed(n)
                     if l != fl.size(i):
                         if len(fl.revision(n)) != fl.size(i):
                             self.err(lr, _("unpacked size is %s, %s expected") %
                                      (l, fl.size(i)), f)
                 except error.CensoredNodeError:
                     # experimental config: censor.policy
                     if ui.config("censor", "policy", "abort") == "abort":
                         self.err(lr, _("censored file data"), f)
                 except Exception as inst:
                     self.exc(lr, _("unpacking %s") % short(n), inst, f)
 
                 # check renames
                 try:
                     if rp:
                         if lr is not None and ui.verbose:
                             ctx = lrugetctx(lr)
                             found = False
                             for pctx in ctx.parents():
                                 if rp[0] in pctx:
                                     found = True
                                     break
                             if not found:
                                 self.warn(_("warning: copy source of '%s' not"
                                             " in parents of %s") % (f, ctx))
                         fl2 = repo.file(rp[0])
                         if not len(fl2):
                             self.err(lr, _("empty or missing copy source "
                                      "revlog %s:%s") % (rp[0], short(rp[1])), f)
                         elif rp[1] == nullid:
                             ui.note(_("warning: %s@%s: copy source"
                                       " revision is nullid %s:%s\n")
                                 % (f, lr, rp[0], short(rp[1])))
                         else:
                             fl2.rev(rp[1])
                 except Exception as inst:
                     self.exc(lr, _("checking rename of %s") % short(n), inst, f)
 
             # cross-check
             if f in filenodes:
                 fns = [(lr, n) for n, lr in filenodes[f].iteritems()]
                 for lr, node in sorted(fns):
                     self.err(lr, _("%s in manifests not found") % short(node),
                              f)
         ui.progress(_('checking'), None)
 
         for f in storefiles:
             self.warn(_("warning: orphan revlog '%s'") % f)
 
         return len(files), revisions

tests/test-verify.t

@@ -1,319 +1,319 b''
 prepare repo
 
   $ hg init a
   $ cd a
   $ echo "some text" > FOO.txt
   $ echo "another text" > bar.txt
   $ echo "more text" > QUICK.txt
   $ hg add
   adding FOO.txt
   adding QUICK.txt
   adding bar.txt
   $ hg ci -mtest1
 
 verify
 
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   3 files, 1 changesets, 3 total revisions
 
 verify with journal
 
   $ touch .hg/store/journal
   $ hg verify
   abandoned transaction found - run hg recover
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   3 files, 1 changesets, 3 total revisions
   $ rm .hg/store/journal
 
 introduce some bugs in repo
 
   $ cd .hg/store/data
   $ mv _f_o_o.txt.i X_f_o_o.txt.i
   $ mv bar.txt.i xbar.txt.i
   $ rm _q_u_i_c_k.txt.i
 
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
    warning: revlog 'data/FOO.txt.i' not in fncache!
    0: empty or missing FOO.txt
    FOO.txt@0: f62022d3d590 in manifests not found
    warning: revlog 'data/QUICK.txt.i' not in fncache!
    0: empty or missing QUICK.txt
    QUICK.txt@0: 88b857db8eba in manifests not found
    warning: revlog 'data/bar.txt.i' not in fncache!
    0: empty or missing bar.txt
    bar.txt@0: 256559129457 in manifests not found
   3 files, 1 changesets, 0 total revisions
   3 warnings encountered!
   hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   6 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
 
   $ cd ../../..
   $ cd ..
 
 Set up a repo for testing missing revlog entries
 
   $ hg init missing-entries
   $ cd missing-entries
   $ echo 0 > file
   $ hg ci -Aqm0
   $ cp -r .hg/store .hg/store-partial
   $ echo 1 > file
   $ hg ci -Aqm1
   $ cp -r .hg/store .hg/store-full
 
 Entire changelog missing
 
   $ rm .hg/store/00changelog.*
   $ hg verify -q
    0: empty or missing changelog
    manifest@0: d0b6632564d4 not in changesets
    manifest@1: 941fc4534185 not in changesets
   3 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Entire manifest log missing
 
   $ rm .hg/store/00manifest.*
   $ hg verify -q
    0: empty or missing manifest
   1 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Entire filelog missing
 
   $ rm .hg/store/data/file.*
   $ hg verify -q
    warning: revlog 'data/file.i' not in fncache!
    0: empty or missing file
    file@0: 362fef284ce2 in manifests not found
    file@1: c10f2164107d in manifests not found
   1 warnings encountered!
   hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   3 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Entire changelog and manifest log missing
 
   $ rm .hg/store/00changelog.*
   $ rm .hg/store/00manifest.*
   $ hg verify -q
   warning: orphan revlog 'data/file.i'
   1 warnings encountered!
   $ cp -r .hg/store-full/* .hg/store
 
 Entire changelog and filelog missing
 
   $ rm .hg/store/00changelog.*
   $ rm .hg/store/data/file.*
   $ hg verify -q
    0: empty or missing changelog
    manifest@0: d0b6632564d4 not in changesets
    manifest@1: 941fc4534185 not in changesets
    warning: revlog 'data/file.i' not in fncache!
    ?: empty or missing file
    file@0: 362fef284ce2 in manifests not found
    file@1: c10f2164107d in manifests not found
   1 warnings encountered!
   hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   6 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Entire manifest log and filelog missing
 
   $ rm .hg/store/00manifest.*
   $ rm .hg/store/data/file.*
   $ hg verify -q
    0: empty or missing manifest
    warning: revlog 'data/file.i' not in fncache!
    0: empty or missing file
   1 warnings encountered!
   hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   2 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Changelog missing entry
 
   $ cp -f .hg/store-partial/00changelog.* .hg/store
   $ hg verify -q
    manifest@?: rev 1 points to nonexistent changeset 1
    manifest@?: 941fc4534185 not in changesets
    file@?: rev 1 points to nonexistent changeset 1
    (expected 0)
   1 warnings encountered!
   3 integrity errors encountered!
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Manifest log missing entry
 
   $ cp -f .hg/store-partial/00manifest.* .hg/store
   $ hg verify -q
-   1: changeset refers to unknown manifest 941fc4534185
+   manifest@1: changeset refers to unknown revision 941fc4534185
    file@1: c10f2164107d not in manifests
   2 integrity errors encountered!
   (first damaged changeset appears to be 1)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Filelog missing entry
 
   $ cp -f .hg/store-partial/data/file.* .hg/store/data
   $ hg verify -q
    file@1: c10f2164107d in manifests not found
   1 integrity errors encountered!
   (first damaged changeset appears to be 1)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Changelog and manifest log missing entry
 
   $ cp -f .hg/store-partial/00changelog.* .hg/store
   $ cp -f .hg/store-partial/00manifest.* .hg/store
   $ hg verify -q
    file@?: rev 1 points to nonexistent changeset 1
    (expected 0)
    file@?: c10f2164107d not in manifests
   1 warnings encountered!
   2 integrity errors encountered!
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Changelog and filelog missing entry
 
   $ cp -f .hg/store-partial/00changelog.* .hg/store
   $ cp -f .hg/store-partial/data/file.* .hg/store/data
   $ hg verify -q
    manifest@?: rev 1 points to nonexistent changeset 1
    manifest@?: 941fc4534185 not in changesets
    file@?: c10f2164107d in manifests not found
   3 integrity errors encountered!
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Manifest and filelog missing entry
 
   $ cp -f .hg/store-partial/00manifest.* .hg/store
   $ cp -f .hg/store-partial/data/file.* .hg/store/data
   $ hg verify -q
-   1: changeset refers to unknown manifest 941fc4534185
+   manifest@1: changeset refers to unknown revision 941fc4534185
   1 integrity errors encountered!
   (first damaged changeset appears to be 1)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Corrupt changelog base node to cause failure to read revision
 
   $ printf abcd | dd conv=notrunc of=.hg/store/00changelog.i bs=1 seek=16 \
   >   2> /dev/null
   $ hg verify -q
    0: unpacking changeset 08b1860757c2: * (glob)
    manifest@?: rev 0 points to unexpected changeset 0
    manifest@?: d0b6632564d4 not in changesets
    file@?: rev 0 points to unexpected changeset 0
    (expected 1)
   1 warnings encountered!
   4 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Corrupt manifest log base node to cause failure to read revision
 
   $ printf abcd | dd conv=notrunc of=.hg/store/00manifest.i bs=1 seek=16 \
   >   2> /dev/null
   $ hg verify -q
-   0: reading manifest delta d0b6632564d4: * (glob)
+   manifest@0: reading delta d0b6632564d4: * (glob)
    file@0: 362fef284ce2 not in manifests
   2 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
 Corrupt filelog base node to cause failure to read revision
 
   $ printf abcd | dd conv=notrunc of=.hg/store/data/file.i bs=1 seek=16 \
   >   2> /dev/null
   $ hg verify -q
    file@0: unpacking 362fef284ce2: * (glob)
   1 integrity errors encountered!
   (first damaged changeset appears to be 0)
   [1]
   $ cp -r .hg/store-full/* .hg/store
 
   $ cd ..
 
 test changelog without a manifest
 
   $ hg init b
   $ cd b
   $ hg branch foo
   marked working directory as branch foo
   (branches are permanent and global, did you want a bookmark?)
   $ hg ci -m branchfoo
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   0 files, 1 changesets, 0 total revisions
 
 test revlog corruption
 
   $ touch a
   $ hg add a
   $ hg ci -m a
 
   $ echo 'corrupted' > b
   $ dd if=.hg/store/data/a.i of=start bs=1 count=20 2>/dev/null
   $ cat start b > .hg/store/data/a.i
 
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
    a@1: broken revlog! (index data/a.i is corrupted)
   warning: orphan revlog 'data/a.i'
   1 files, 2 changesets, 0 total revisions
   1 warnings encountered!
   1 integrity errors encountered!
   (first damaged changeset appears to be 1)
   [1]
 
   $ cd ..
 
 test revlog format 0
 
   $ revlog-formatv0.py
   $ cd formatv0
   $ hg verify
   repository uses revlog format 0
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   1 files, 1 changesets, 1 total revisions
   $ cd ..