##// END OF EJS Templates
verify: document the `_verifymanifest` method
marmoute -
r42042:5ad5a70d default
parent child Browse files
Show More
@@ -1,506 +1,533 b''
1 1 # verify.py - repository integrity checking for Mercurial
2 2 #
3 3 # Copyright 2006, 2007 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import os
11 11
12 12 from .i18n import _
13 13 from .node import (
14 14 nullid,
15 15 short,
16 16 )
17 17
18 18 from . import (
19 19 error,
20 20 pycompat,
21 21 revlog,
22 22 util,
23 23 )
24 24
25 25 def verify(repo):
26 26 with repo.lock():
27 27 return verifier(repo).verify()
28 28
29 29 def _normpath(f):
30 30 # under hg < 2.4, convert didn't sanitize paths properly, so a
31 31 # converted repo may contain repeated slashes
32 32 while '//' in f:
33 33 f = f.replace('//', '/')
34 34 return f
35 35
36 36 class verifier(object):
37 37 def __init__(self, repo):
38 38 self.repo = repo.unfiltered()
39 39 self.ui = repo.ui
40 40 self.match = repo.narrowmatch()
41 41 self.badrevs = set()
42 42 self.errors = 0
43 43 self.warnings = 0
44 44 self.havecl = len(repo.changelog) > 0
45 45 self.havemf = len(repo.manifestlog.getstorage(b'')) > 0
46 46 self.revlogv1 = repo.changelog.version != revlog.REVLOGV0
47 47 self.lrugetctx = util.lrucachefunc(repo.__getitem__)
48 48 self.refersmf = False
49 49 self.fncachewarned = False
50 50 # developer config: verify.skipflags
51 51 self.skipflags = repo.ui.configint('verify', 'skipflags')
52 52 self.warnorphanstorefiles = True
53 53
54 54 def _warn(self, msg):
55 55 """record a "warning" level issue"""
56 56 self.ui.warn(msg + "\n")
57 57 self.warnings += 1
58 58
59 59 def _err(self, linkrev, msg, filename=None):
60 60 """record a "error" level issue"""
61 61 if linkrev is not None:
62 62 self.badrevs.add(linkrev)
63 63 linkrev = "%d" % linkrev
64 64 else:
65 65 linkrev = '?'
66 66 msg = "%s: %s" % (linkrev, msg)
67 67 if filename:
68 68 msg = "%s@%s" % (filename, msg)
69 69 self.ui.warn(" " + msg + "\n")
70 70 self.errors += 1
71 71
72 72 def _exc(self, linkrev, msg, inst, filename=None):
73 73 """record exception raised during the verify process"""
74 74 fmsg = pycompat.bytestr(inst)
75 75 if not fmsg:
76 76 fmsg = pycompat.byterepr(inst)
77 77 self._err(linkrev, "%s: %s" % (msg, fmsg), filename)
78 78
79 79 def _checkrevlog(self, obj, name, linkrev):
80 80 """verify high level property of a revlog
81 81
82 82 - revlog is present,
83 83 - revlog is non-empty,
84 84 - sizes (index and data) are correct,
85 85 - revlog's format version is correct.
86 86 """
87 87 if not len(obj) and (self.havecl or self.havemf):
88 88 self._err(linkrev, _("empty or missing %s") % name)
89 89 return
90 90
91 91 d = obj.checksize()
92 92 if d[0]:
93 93 self.err(None, _("data length off by %d bytes") % d[0], name)
94 94 if d[1]:
95 95 self.err(None, _("index contains %d extra bytes") % d[1], name)
96 96
97 97 if obj.version != revlog.REVLOGV0:
98 98 if not self.revlogv1:
99 99 self._warn(_("warning: `%s' uses revlog format 1") % name)
100 100 elif self.revlogv1:
101 101 self._warn(_("warning: `%s' uses revlog format 0") % name)
102 102
103 103 def _checkentry(self, obj, i, node, seen, linkrevs, f):
104 104 """verify a single revlog entry
105 105
106 106 arguments are:
107 107 - obj: the source revlog
108 108 - i: the revision number
109 109 - node: the revision node id
110 110 - seen: nodes previously seen for this revlog
111 111 - linkrevs: [changelog-revisions] introducing "node"
112 112 - f: string label ("changelog", "manifest", or filename)
113 113
114 114 Performs the following checks:
115 115 - linkrev points to an existing changelog revision,
116 116 - linkrev points to a changelog revision that introduces this revision,
117 117 - linkrev points to the lowest of these changesets,
118 118 - both parents exist in the revlog,
119 119 - the revision is not duplicated.
120 120
121 121 Return the linkrev of the revision (or None for changelog's revisions).
122 122 """
123 123 lr = obj.linkrev(obj.rev(node))
124 124 if lr < 0 or (self.havecl and lr not in linkrevs):
125 125 if lr < 0 or lr >= len(self.repo.changelog):
126 126 msg = _("rev %d points to nonexistent changeset %d")
127 127 else:
128 128 msg = _("rev %d points to unexpected changeset %d")
129 129 self._err(None, msg % (i, lr), f)
130 130 if linkrevs:
131 131 if f and len(linkrevs) > 1:
132 132 try:
133 133 # attempt to filter down to real linkrevs
134 134 linkrevs = [l for l in linkrevs
135 135 if self.lrugetctx(l)[f].filenode() == node]
136 136 except Exception:
137 137 pass
138 138 self._warn(_(" (expected %s)") % " ".join
139 139 (map(pycompat.bytestr, linkrevs)))
140 140 lr = None # can't be trusted
141 141
142 142 try:
143 143 p1, p2 = obj.parents(node)
144 144 if p1 not in seen and p1 != nullid:
145 145 self._err(lr, _("unknown parent 1 %s of %s") %
146 146 (short(p1), short(node)), f)
147 147 if p2 not in seen and p2 != nullid:
148 148 self._err(lr, _("unknown parent 2 %s of %s") %
149 149 (short(p2), short(node)), f)
150 150 except Exception as inst:
151 151 self._exc(lr, _("checking parents of %s") % short(node), inst, f)
152 152
153 153 if node in seen:
154 154 self._err(lr, _("duplicate revision %d (%d)") % (i, seen[node]), f)
155 155 seen[node] = i
156 156 return lr
157 157
158 158 def verify(self):
159 159 """verify the content of the Mercurial repository
160 160
161 161 This method run all verifications, displaying issues as they are found.
162 162
163 163 return 1 if any error have been encountered, 0 otherwise."""
164 164 # initial validation and generic report
165 165 repo = self.repo
166 166 ui = repo.ui
167 167 if not repo.url().startswith('file:'):
168 168 raise error.Abort(_("cannot verify bundle or remote repos"))
169 169
170 170 if os.path.exists(repo.sjoin("journal")):
171 171 ui.warn(_("abandoned transaction found - run hg recover\n"))
172 172
173 173 if ui.verbose or not self.revlogv1:
174 174 ui.status(_("repository uses revlog format %d\n") %
175 175 (self.revlogv1 and 1 or 0))
176 176
177 177 # data verification
178 178 mflinkrevs, filelinkrevs = self._verifychangelog()
179 179 filenodes = self._verifymanifest(mflinkrevs)
180 180 del mflinkrevs
181 181 self._crosscheckfiles(filelinkrevs, filenodes)
182 182 totalfiles, filerevisions = self._verifyfiles(filenodes, filelinkrevs)
183 183
184 184 # final report
185 185 ui.status(_("checked %d changesets with %d changes to %d files\n") %
186 186 (len(repo.changelog), filerevisions, totalfiles))
187 187 if self.warnings:
188 188 ui.warn(_("%d warnings encountered!\n") % self.warnings)
189 189 if self.fncachewarned:
190 190 ui.warn(_('hint: run "hg debugrebuildfncache" to recover from '
191 191 'corrupt fncache\n'))
192 192 if self.errors:
193 193 ui.warn(_("%d integrity errors encountered!\n") % self.errors)
194 194 if self.badrevs:
195 195 ui.warn(_("(first damaged changeset appears to be %d)\n")
196 196 % min(self.badrevs))
197 197 return 1
198 198 return 0
199 199
200 200 def _verifychangelog(self):
201 201 """verify the changelog of a repository
202 202
203 203 The following checks are performed:
204 204 - all of `_checkrevlog` checks,
205 205 - all of `_checkentry` checks (for each revisions),
206 206 - each revision can be read.
207 207
208 208 The function returns some of the data observed in the changesets as a
209 209 (mflinkrevs, filelinkrevs) tuples:
210 210 - mflinkrevs: is a { manifest-node -> [changelog-rev] } mapping
211 211 - filelinkrevs: is a { file-path -> [changelog-rev] } mapping
212 212
213 213 If a matcher was specified, filelinkrevs will only contains matched
214 214 files.
215 215 """
216 216 ui = self.ui
217 217 repo = self.repo
218 218 match = self.match
219 219 cl = repo.changelog
220 220
221 221 ui.status(_("checking changesets\n"))
222 222 mflinkrevs = {}
223 223 filelinkrevs = {}
224 224 seen = {}
225 225 self._checkrevlog(cl, "changelog", 0)
226 226 progress = ui.makeprogress(_('checking'), unit=_('changesets'),
227 227 total=len(repo))
228 228 for i in repo:
229 229 progress.update(i)
230 230 n = cl.node(i)
231 231 self._checkentry(cl, i, n, seen, [i], "changelog")
232 232
233 233 try:
234 234 changes = cl.read(n)
235 235 if changes[0] != nullid:
236 236 mflinkrevs.setdefault(changes[0], []).append(i)
237 237 self.refersmf = True
238 238 for f in changes[3]:
239 239 if match(f):
240 240 filelinkrevs.setdefault(_normpath(f), []).append(i)
241 241 except Exception as inst:
242 242 self.refersmf = True
243 243 self._exc(i, _("unpacking changeset %s") % short(n), inst)
244 244 progress.complete()
245 245 return mflinkrevs, filelinkrevs
246 246
247 247 def _verifymanifest(self, mflinkrevs, dir="", storefiles=None,
248 248 subdirprogress=None):
249 """verify the manifestlog content
250
251 Inputs:
252 - mflinkrevs: a {manifest-node -> [changelog-revisions]} mapping
253 - dir: a subdirectory to check (for tree manifest repo)
254 - storefiles: set of currently "orphan" files.
255 - subdirprogress: a progress object
256
257 This function checks:
258 * all of `_checkrevlog` checks (for all manifest related revlogs)
259 * all of `_checkentry` checks (for all manifest related revisions)
260 * nodes for subdirectory exists in the sub-directory manifest
261 * each manifest entries have a file path
262 * each manifest node refered in mflinkrevs exist in the manifest log
263
264 If tree manifest is in use and a matchers is specified, only the
265 sub-directories matching it will be verified.
266
267 return a two level mapping:
268 {"path" -> { filenode -> changelog-revision}}
269
270 This mapping primarily contains entries for every files in the
271 repository. In addition, when tree-manifest is used, it also contains
272 sub-directory entries.
273
274 If a matcher is provided, only matching paths will be included.
275 """
249 276 repo = self.repo
250 277 ui = self.ui
251 278 match = self.match
252 279 mfl = self.repo.manifestlog
253 280 mf = mfl.getstorage(dir)
254 281
255 282 if not dir:
256 283 self.ui.status(_("checking manifests\n"))
257 284
258 285 filenodes = {}
259 286 subdirnodes = {}
260 287 seen = {}
261 288 label = "manifest"
262 289 if dir:
263 290 label = dir
264 291 revlogfiles = mf.files()
265 292 storefiles.difference_update(revlogfiles)
266 293 if subdirprogress: # should be true since we're in a subdirectory
267 294 subdirprogress.increment()
268 295 if self.refersmf:
269 296 # Do not check manifest if there are only changelog entries with
270 297 # null manifests.
271 298 self._checkrevlog(mf, label, 0)
272 299 progress = ui.makeprogress(_('checking'), unit=_('manifests'),
273 300 total=len(mf))
274 301 for i in mf:
275 302 if not dir:
276 303 progress.update(i)
277 304 n = mf.node(i)
278 305 lr = self._checkentry(mf, i, n, seen, mflinkrevs.get(n, []), label)
279 306 if n in mflinkrevs:
280 307 del mflinkrevs[n]
281 308 elif dir:
282 309 self._err(lr, _("%s not in parent-directory manifest") %
283 310 short(n), label)
284 311 else:
285 312 self._err(lr, _("%s not in changesets") % short(n), label)
286 313
287 314 try:
288 315 mfdelta = mfl.get(dir, n).readdelta(shallow=True)
289 316 for f, fn, fl in mfdelta.iterentries():
290 317 if not f:
291 318 self._err(lr, _("entry without name in manifest"))
292 319 elif f == "/dev/null": # ignore this in very old repos
293 320 continue
294 321 fullpath = dir + _normpath(f)
295 322 if fl == 't':
296 323 if not match.visitdir(fullpath):
297 324 continue
298 325 subdirnodes.setdefault(fullpath + '/', {}).setdefault(
299 326 fn, []).append(lr)
300 327 else:
301 328 if not match(fullpath):
302 329 continue
303 330 filenodes.setdefault(fullpath, {}).setdefault(fn, lr)
304 331 except Exception as inst:
305 332 self._exc(lr, _("reading delta %s") % short(n), inst, label)
306 333 if not dir:
307 334 progress.complete()
308 335
309 336 if self.havemf:
310 337 for c, m in sorted([(c, m) for m in mflinkrevs
311 338 for c in mflinkrevs[m]]):
312 339 if dir:
313 340 self._err(c, _("parent-directory manifest refers to unknown"
314 341 " revision %s") % short(m), label)
315 342 else:
316 343 self._err(c, _("changeset refers to unknown revision %s") %
317 344 short(m), label)
318 345
319 346 if not dir and subdirnodes:
320 347 self.ui.status(_("checking directory manifests\n"))
321 348 storefiles = set()
322 349 subdirs = set()
323 350 revlogv1 = self.revlogv1
324 351 for f, f2, size in repo.store.datafiles():
325 352 if not f:
326 353 self._err(None, _("cannot decode filename '%s'") % f2)
327 354 elif (size > 0 or not revlogv1) and f.startswith('meta/'):
328 355 storefiles.add(_normpath(f))
329 356 subdirs.add(os.path.dirname(f))
330 357 subdirprogress = ui.makeprogress(_('checking'), unit=_('manifests'),
331 358 total=len(subdirs))
332 359
333 360 for subdir, linkrevs in subdirnodes.iteritems():
334 361 subdirfilenodes = self._verifymanifest(linkrevs, subdir, storefiles,
335 362 subdirprogress)
336 363 for f, onefilenodes in subdirfilenodes.iteritems():
337 364 filenodes.setdefault(f, {}).update(onefilenodes)
338 365
339 366 if not dir and subdirnodes:
340 367 subdirprogress.complete()
341 368 if self.warnorphanstorefiles:
342 369 for f in sorted(storefiles):
343 370 self._warn(_("warning: orphan data file '%s'") % f)
344 371
345 372 return filenodes
346 373
347 374 def _crosscheckfiles(self, filelinkrevs, filenodes):
348 375 repo = self.repo
349 376 ui = self.ui
350 377 ui.status(_("crosschecking files in changesets and manifests\n"))
351 378
352 379 total = len(filelinkrevs) + len(filenodes)
353 380 progress = ui.makeprogress(_('crosschecking'), unit=_('files'),
354 381 total=total)
355 382 if self.havemf:
356 383 for f in sorted(filelinkrevs):
357 384 progress.increment()
358 385 if f not in filenodes:
359 386 lr = filelinkrevs[f][0]
360 387 self._err(lr, _("in changeset but not in manifest"), f)
361 388
362 389 if self.havecl:
363 390 for f in sorted(filenodes):
364 391 progress.increment()
365 392 if f not in filelinkrevs:
366 393 try:
367 394 fl = repo.file(f)
368 395 lr = min([fl.linkrev(fl.rev(n)) for n in filenodes[f]])
369 396 except Exception:
370 397 lr = None
371 398 self._err(lr, _("in manifest but not in changeset"), f)
372 399
373 400 progress.complete()
374 401
375 402 def _verifyfiles(self, filenodes, filelinkrevs):
376 403 repo = self.repo
377 404 ui = self.ui
378 405 lrugetctx = self.lrugetctx
379 406 revlogv1 = self.revlogv1
380 407 havemf = self.havemf
381 408 ui.status(_("checking files\n"))
382 409
383 410 storefiles = set()
384 411 for f, f2, size in repo.store.datafiles():
385 412 if not f:
386 413 self._err(None, _("cannot decode filename '%s'") % f2)
387 414 elif (size > 0 or not revlogv1) and f.startswith('data/'):
388 415 storefiles.add(_normpath(f))
389 416
390 417 state = {
391 418 # TODO this assumes revlog storage for changelog.
392 419 'expectedversion': self.repo.changelog.version & 0xFFFF,
393 420 'skipflags': self.skipflags,
394 421 # experimental config: censor.policy
395 422 'erroroncensored': ui.config('censor', 'policy') == 'abort',
396 423 }
397 424
398 425 files = sorted(set(filenodes) | set(filelinkrevs))
399 426 revisions = 0
400 427 progress = ui.makeprogress(_('checking'), unit=_('files'),
401 428 total=len(files))
402 429 for i, f in enumerate(files):
403 430 progress.update(i, item=f)
404 431 try:
405 432 linkrevs = filelinkrevs[f]
406 433 except KeyError:
407 434 # in manifest but not in changelog
408 435 linkrevs = []
409 436
410 437 if linkrevs:
411 438 lr = linkrevs[0]
412 439 else:
413 440 lr = None
414 441
415 442 try:
416 443 fl = repo.file(f)
417 444 except error.StorageError as e:
418 445 self._err(lr, _("broken revlog! (%s)") % e, f)
419 446 continue
420 447
421 448 for ff in fl.files():
422 449 try:
423 450 storefiles.remove(ff)
424 451 except KeyError:
425 452 if self.warnorphanstorefiles:
426 453 self._warn(_(" warning: revlog '%s' not in fncache!") %
427 454 ff)
428 455 self.fncachewarned = True
429 456
430 457 if not len(fl) and (self.havecl or self.havemf):
431 458 self._err(lr, _("empty or missing %s") % f)
432 459 else:
433 460 # Guard against implementations not setting this.
434 461 state['skipread'] = set()
435 462 for problem in fl.verifyintegrity(state):
436 463 if problem.node is not None:
437 464 linkrev = fl.linkrev(fl.rev(problem.node))
438 465 else:
439 466 linkrev = None
440 467
441 468 if problem.warning:
442 469 self._warn(problem.warning)
443 470 elif problem.error:
444 471 self._err(linkrev if linkrev is not None else lr,
445 472 problem.error, f)
446 473 else:
447 474 raise error.ProgrammingError(
448 475 'problem instance does not set warning or error '
449 476 'attribute: %s' % problem.msg)
450 477
451 478 seen = {}
452 479 for i in fl:
453 480 revisions += 1
454 481 n = fl.node(i)
455 482 lr = self._checkentry(fl, i, n, seen, linkrevs, f)
456 483 if f in filenodes:
457 484 if havemf and n not in filenodes[f]:
458 485 self._err(lr, _("%s not in manifests") % (short(n)), f)
459 486 else:
460 487 del filenodes[f][n]
461 488
462 489 if n in state['skipread']:
463 490 continue
464 491
465 492 # check renames
466 493 try:
467 494 # This requires resolving fulltext (at least on revlogs). We
468 495 # may want ``verifyintegrity()`` to pass a set of nodes with
469 496 # rename metadata as an optimization.
470 497 rp = fl.renamed(n)
471 498 if rp:
472 499 if lr is not None and ui.verbose:
473 500 ctx = lrugetctx(lr)
474 501 if not any(rp[0] in pctx for pctx in ctx.parents()):
475 502 self._warn(_("warning: copy source of '%s' not"
476 503 " in parents of %s") % (f, ctx))
477 504 fl2 = repo.file(rp[0])
478 505 if not len(fl2):
479 506 self._err(lr,
480 507 _("empty or missing copy source revlog "
481 508 "%s:%s") % (rp[0],
482 509 short(rp[1])),
483 510 f)
484 511 elif rp[1] == nullid:
485 512 ui.note(_("warning: %s@%s: copy source"
486 513 " revision is nullid %s:%s\n")
487 514 % (f, lr, rp[0], short(rp[1])))
488 515 else:
489 516 fl2.rev(rp[1])
490 517 except Exception as inst:
491 518 self._exc(lr, _("checking rename of %s") % short(n),
492 519 inst, f)
493 520
494 521 # cross-check
495 522 if f in filenodes:
496 523 fns = [(v, k) for k, v in filenodes[f].iteritems()]
497 524 for lr, node in sorted(fns):
498 525 self._err(lr, _("manifest refers to unknown revision %s") %
499 526 short(node), f)
500 527 progress.complete()
501 528
502 529 if self.warnorphanstorefiles:
503 530 for f in sorted(storefiles):
504 531 self._warn(_("warning: orphan data file '%s'") % f)
505 532
506 533 return len(files), revisions
General Comments 0
You need to be logged in to leave comments. Login now