Show More
@@ -289,7 +289,7 b' def _narrow(' | |||||
289 | repair.strip(ui, unfi, tostrip, topic=b'narrow', backup=backup) |
|
289 | repair.strip(ui, unfi, tostrip, topic=b'narrow', backup=backup) | |
290 |
|
290 | |||
291 | todelete = [] |
|
291 | todelete = [] | |
292 |
for t, f |
|
292 | for t, f, size in repo.store.datafiles(): | |
293 | if f.startswith(b'data/'): |
|
293 | if f.startswith(b'data/'): | |
294 | file = f[5:-2] |
|
294 | file = f[5:-2] | |
295 | if not newmatch(file): |
|
295 | if not newmatch(file): |
@@ -378,7 +378,7 b' class manifestrevlogstore(object):' | |||||
378 | ledger.markdataentry(self, treename, node) |
|
378 | ledger.markdataentry(self, treename, node) | |
379 | ledger.markhistoryentry(self, treename, node) |
|
379 | ledger.markhistoryentry(self, treename, node) | |
380 |
|
380 | |||
381 |
for t, path |
|
381 | for t, path, size in self._store.datafiles(): | |
382 | if path[:5] != b'meta/' or path[-2:] != b'.i': |
|
382 | if path[:5] != b'meta/' or path[-2:] != b'.i': | |
383 | continue |
|
383 | continue | |
384 |
|
384 |
@@ -166,24 +166,24 b' def onetimesetup(ui):' | |||||
166 | n = util.pconvert(fp[striplen:]) |
|
166 | n = util.pconvert(fp[striplen:]) | |
167 | d = store.decodedir(n) |
|
167 | d = store.decodedir(n) | |
168 | t = store.FILETYPE_OTHER |
|
168 | t = store.FILETYPE_OTHER | |
169 |
yield (t, d, |
|
169 | yield (t, d, st.st_size) | |
170 | if kind == stat.S_IFDIR: |
|
170 | if kind == stat.S_IFDIR: | |
171 | visit.append(fp) |
|
171 | visit.append(fp) | |
172 |
|
172 | |||
173 | if scmutil.istreemanifest(repo): |
|
173 | if scmutil.istreemanifest(repo): | |
174 |
for (t, u |
|
174 | for (t, u, s) in repo.store.datafiles(): | |
175 | if u.startswith(b'meta/') and ( |
|
175 | if u.startswith(b'meta/') and ( | |
176 | u.endswith(b'.i') or u.endswith(b'.d') |
|
176 | u.endswith(b'.i') or u.endswith(b'.d') | |
177 | ): |
|
177 | ): | |
178 |
yield (t, u, |
|
178 | yield (t, u, s) | |
179 |
|
179 | |||
180 | # Return .d and .i files that do not match the shallow pattern |
|
180 | # Return .d and .i files that do not match the shallow pattern | |
181 | match = state.match |
|
181 | match = state.match | |
182 | if match and not match.always(): |
|
182 | if match and not match.always(): | |
183 |
for (t, u |
|
183 | for (t, u, s) in repo.store.datafiles(): | |
184 | f = u[5:-2] # trim data/... and .i/.d |
|
184 | f = u[5:-2] # trim data/... and .i/.d | |
185 | if not state.match(f): |
|
185 | if not state.match(f): | |
186 |
yield (t, u, |
|
186 | yield (t, u, s) | |
187 |
|
187 | |||
188 | for x in repo.store.topfiles(): |
|
188 | for x in repo.store.topfiles(): | |
189 | if state.noflatmf and x[1][:11] == b'00manifest.': |
|
189 | if state.noflatmf and x[1][:11] == b'00manifest.': |
@@ -433,7 +433,7 b' def manifestrevlogs(repo):' | |||||
433 | if scmutil.istreemanifest(repo): |
|
433 | if scmutil.istreemanifest(repo): | |
434 | # This logic is safe if treemanifest isn't enabled, but also |
|
434 | # This logic is safe if treemanifest isn't enabled, but also | |
435 | # pointless, so we skip it if treemanifest isn't enabled. |
|
435 | # pointless, so we skip it if treemanifest isn't enabled. | |
436 |
for t, unencoded |
|
436 | for t, unencoded, size in repo.store.datafiles(): | |
437 | if unencoded.startswith(b'meta/') and unencoded.endswith( |
|
437 | if unencoded.startswith(b'meta/') and unencoded.endswith( | |
438 | b'00manifest.i' |
|
438 | b'00manifest.i' | |
439 | ): |
|
439 | ): |
@@ -824,7 +824,7 b' def repair_issue6528(' | |||||
824 | with context(): |
|
824 | with context(): | |
825 | files = list( |
|
825 | files = list( | |
826 | (file_type, path) |
|
826 | (file_type, path) | |
827 |
for (file_type, path |
|
827 | for (file_type, path, _s) in repo.store.datafiles() | |
828 | if path.endswith(b'.i') and file_type & store.FILEFLAGS_FILELOG |
|
828 | if path.endswith(b'.i') and file_type & store.FILEFLAGS_FILELOG | |
829 | ) |
|
829 | ) | |
830 |
|
830 |
@@ -472,7 +472,7 b' class basicstore(object):' | |||||
472 | return self.path + b'/' + encodedir(f) |
|
472 | return self.path + b'/' + encodedir(f) | |
473 |
|
473 | |||
474 | def _walk(self, relpath, recurse): |
|
474 | def _walk(self, relpath, recurse): | |
475 |
'''yields ( |
|
475 | '''yields (revlog_type, unencoded, size)''' | |
476 | path = self.path |
|
476 | path = self.path | |
477 | if relpath: |
|
477 | if relpath: | |
478 | path += b'/' + relpath |
|
478 | path += b'/' + relpath | |
@@ -488,7 +488,7 b' class basicstore(object):' | |||||
488 | rl_type = is_revlog(f, kind, st) |
|
488 | rl_type = is_revlog(f, kind, st) | |
489 | if rl_type is not None: |
|
489 | if rl_type is not None: | |
490 | n = util.pconvert(fp[striplen:]) |
|
490 | n = util.pconvert(fp[striplen:]) | |
491 |
l.append((rl_type, decodedir(n), |
|
491 | l.append((rl_type, decodedir(n), st.st_size)) | |
492 | elif kind == stat.S_IFDIR and recurse: |
|
492 | elif kind == stat.S_IFDIR and recurse: | |
493 | visit.append(fp) |
|
493 | visit.append(fp) | |
494 | l.sort() |
|
494 | l.sort() | |
@@ -505,26 +505,32 b' class basicstore(object):' | |||||
505 | rootstore = manifest.manifestrevlog(repo.nodeconstants, self.vfs) |
|
505 | rootstore = manifest.manifestrevlog(repo.nodeconstants, self.vfs) | |
506 | return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch) |
|
506 | return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch) | |
507 |
|
507 | |||
508 | def datafiles(self, matcher=None): |
|
508 | def datafiles(self, matcher=None, undecodable=None): | |
|
509 | """Like walk, but excluding the changelog and root manifest. | |||
|
510 | ||||
|
511 | When [undecodable] is None, revlogs names that can't be | |||
|
512 | decoded cause an exception. When it is provided, it should | |||
|
513 | be a list and the filenames that can't be decoded are added | |||
|
514 | to it instead. This is very rarely needed.""" | |||
509 | files = self._walk(b'data', True) + self._walk(b'meta', True) |
|
515 | files = self._walk(b'data', True) + self._walk(b'meta', True) | |
510 |
for (t, u, |
|
516 | for (t, u, s) in files: | |
511 |
yield (FILEFLAGS_FILELOG | t, u, |
|
517 | yield (FILEFLAGS_FILELOG | t, u, s) | |
512 |
|
518 | |||
513 | def topfiles(self): |
|
519 | def topfiles(self): | |
514 | # yield manifest before changelog |
|
520 | # yield manifest before changelog | |
515 | files = reversed(self._walk(b'', False)) |
|
521 | files = reversed(self._walk(b'', False)) | |
516 |
for (t, u, |
|
522 | for (t, u, s) in files: | |
517 | if u.startswith(b'00changelog'): |
|
523 | if u.startswith(b'00changelog'): | |
518 |
yield (FILEFLAGS_CHANGELOG | t, u, |
|
524 | yield (FILEFLAGS_CHANGELOG | t, u, s) | |
519 | elif u.startswith(b'00manifest'): |
|
525 | elif u.startswith(b'00manifest'): | |
520 |
yield (FILEFLAGS_MANIFESTLOG | t, u, |
|
526 | yield (FILEFLAGS_MANIFESTLOG | t, u, s) | |
521 | else: |
|
527 | else: | |
522 |
yield (FILETYPE_OTHER | t, u, |
|
528 | yield (FILETYPE_OTHER | t, u, s) | |
523 |
|
529 | |||
524 | def walk(self, matcher=None): |
|
530 | def walk(self, matcher=None): | |
525 | """return file related to data storage (ie: revlogs) |
|
531 | """return file related to data storage (ie: revlogs) | |
526 |
|
532 | |||
527 |
yields (file_type, unencoded, |
|
533 | yields (file_type, unencoded, size) | |
528 |
|
534 | |||
529 | if a matcher is passed, storage files of only those tracked paths |
|
535 | if a matcher is passed, storage files of only those tracked paths | |
530 | are passed with matches the matcher |
|
536 | are passed with matches the matcher | |
@@ -574,15 +580,20 b' class encodedstore(basicstore):' | |||||
574 | # However that might change so we should probably add a test and encoding |
|
580 | # However that might change so we should probably add a test and encoding | |
575 | # decoding for it too. see issue6548 |
|
581 | # decoding for it too. see issue6548 | |
576 |
|
582 | |||
577 | def datafiles(self, matcher=None): |
|
583 | def datafiles(self, matcher=None, undecodable=None): | |
578 |
for t, |
|
584 | for t, f1, size in super(encodedstore, self).datafiles(): | |
579 | try: |
|
585 | try: | |
580 |
|
|
586 | f2 = decodefilename(f1) | |
581 | except KeyError: |
|
587 | except KeyError: | |
582 |
|
|
588 | if undecodable is None: | |
583 | if a is not None and not _matchtrackedpath(a, matcher): |
|
589 | msg = _(b'undecodable revlog name %s') % f1 | |
|
590 | raise error.StorageError(msg) | |||
|
591 | else: | |||
|
592 | undecodable.append(f1) | |||
|
593 | continue | |||
|
594 | if not _matchtrackedpath(f2, matcher): | |||
584 | continue |
|
595 | continue | |
585 |
yield t, |
|
596 | yield t, f2, size | |
586 |
|
597 | |||
587 | def join(self, f): |
|
598 | def join(self, f): | |
588 | return self.path + b'/' + encodefilename(f) |
|
599 | return self.path + b'/' + encodefilename(f) | |
@@ -770,7 +781,7 b' class fncachestore(basicstore):' | |||||
770 | def getsize(self, path): |
|
781 | def getsize(self, path): | |
771 | return self.rawvfs.stat(path).st_size |
|
782 | return self.rawvfs.stat(path).st_size | |
772 |
|
783 | |||
773 | def datafiles(self, matcher=None): |
|
784 | def datafiles(self, matcher=None, undecodable=None): | |
774 | for f in sorted(self.fncache): |
|
785 | for f in sorted(self.fncache): | |
775 | if not _matchtrackedpath(f, matcher): |
|
786 | if not _matchtrackedpath(f, matcher): | |
776 | continue |
|
787 | continue | |
@@ -779,7 +790,7 b' class fncachestore(basicstore):' | |||||
779 | t = revlog_type(f) |
|
790 | t = revlog_type(f) | |
780 | assert t is not None, f |
|
791 | assert t is not None, f | |
781 | t |= FILEFLAGS_FILELOG |
|
792 | t |= FILEFLAGS_FILELOG | |
782 |
yield t, f, |
|
793 | yield t, f, self.getsize(ef) | |
783 | except OSError as err: |
|
794 | except OSError as err: | |
784 | if err.errno != errno.ENOENT: |
|
795 | if err.errno != errno.ENOENT: | |
785 | raise |
|
796 | raise |
@@ -248,7 +248,7 b' def generatev1(repo):' | |||||
248 | # Get consistent snapshot of repo, lock during scan. |
|
248 | # Get consistent snapshot of repo, lock during scan. | |
249 | with repo.lock(): |
|
249 | with repo.lock(): | |
250 | repo.ui.debug(b'scanning\n') |
|
250 | repo.ui.debug(b'scanning\n') | |
251 |
for file_type, name |
|
251 | for file_type, name, size in _walkstreamfiles(repo): | |
252 | if size: |
|
252 | if size: | |
253 | entries.append((name, size)) |
|
253 | entries.append((name, size)) | |
254 | total_bytes += size |
|
254 | total_bytes += size | |
@@ -650,7 +650,7 b' def _v2_walk(repo, includes, excludes, i' | |||||
650 | if includes or excludes: |
|
650 | if includes or excludes: | |
651 | matcher = narrowspec.match(repo.root, includes, excludes) |
|
651 | matcher = narrowspec.match(repo.root, includes, excludes) | |
652 |
|
652 | |||
653 |
for rl_type, name |
|
653 | for rl_type, name, size in _walkstreamfiles(repo, matcher): | |
654 | if size: |
|
654 | if size: | |
655 | ft = _fileappend |
|
655 | ft = _fileappend | |
656 | if rl_type & store.FILEFLAGS_VOLATILE: |
|
656 | if rl_type & store.FILEFLAGS_VOLATILE: |
@@ -201,7 +201,7 b' def _clonerevlogs(' | |||||
201 |
|
201 | |||
202 | # Perform a pass to collect metadata. This validates we can open all |
|
202 | # Perform a pass to collect metadata. This validates we can open all | |
203 | # source files and allows a unified progress bar to be displayed. |
|
203 | # source files and allows a unified progress bar to be displayed. | |
204 |
for rl_type, unencoded, |
|
204 | for rl_type, unencoded, size in alldatafiles: | |
205 | if not rl_type & store.FILEFLAGS_REVLOG_MAIN: |
|
205 | if not rl_type & store.FILEFLAGS_REVLOG_MAIN: | |
206 | continue |
|
206 | continue | |
207 |
|
207 |
@@ -395,12 +395,13 b' class verifier(object):' | |||||
395 | storefiles = set() |
|
395 | storefiles = set() | |
396 | subdirs = set() |
|
396 | subdirs = set() | |
397 | revlogv1 = self.revlogv1 |
|
397 | revlogv1 = self.revlogv1 | |
398 | for t, f, f2, size in repo.store.datafiles(): |
|
398 | undecodable = [] | |
399 | if not f: |
|
399 | for t, f, size in repo.store.datafiles(undecodable=undecodable): | |
400 | self._err(None, _(b"cannot decode filename '%s'") % f2) |
|
400 | if (size > 0 or not revlogv1) and f.startswith(b'meta/'): | |
401 | elif (size > 0 or not revlogv1) and f.startswith(b'meta/'): |
|
|||
402 | storefiles.add(_normpath(f)) |
|
401 | storefiles.add(_normpath(f)) | |
403 | subdirs.add(os.path.dirname(f)) |
|
402 | subdirs.add(os.path.dirname(f)) | |
|
403 | for f in undecodable: | |||
|
404 | self._err(None, _(b"cannot decode filename '%s'") % f) | |||
404 | subdirprogress = ui.makeprogress( |
|
405 | subdirprogress = ui.makeprogress( | |
405 | _(b'checking'), unit=_(b'manifests'), total=len(subdirs) |
|
406 | _(b'checking'), unit=_(b'manifests'), total=len(subdirs) | |
406 | ) |
|
407 | ) | |
@@ -459,11 +460,12 b' class verifier(object):' | |||||
459 | ui.status(_(b"checking files\n")) |
|
460 | ui.status(_(b"checking files\n")) | |
460 |
|
461 | |||
461 | storefiles = set() |
|
462 | storefiles = set() | |
462 | for rl_type, f, f2, size in repo.store.datafiles(): |
|
463 | undecodable = [] | |
463 | if not f: |
|
464 | for t, f, size in repo.store.datafiles(undecodable=undecodable): | |
464 | self._err(None, _(b"cannot decode filename '%s'") % f2) |
|
465 | if (size > 0 or not revlogv1) and f.startswith(b'data/'): | |
465 | elif (size > 0 or not revlogv1) and f.startswith(b'data/'): |
|
|||
466 | storefiles.add(_normpath(f)) |
|
466 | storefiles.add(_normpath(f)) | |
|
467 | for f in undecodable: | |||
|
468 | self._err(None, _(b"cannot decode filename '%s'") % f) | |||
467 |
|
469 | |||
468 | state = { |
|
470 | state = { | |
469 | # TODO this assumes revlog storage for changelog. |
|
471 | # TODO this assumes revlog storage for changelog. |
@@ -1579,7 +1579,7 b' def rawstorefiledata(repo, proto, files,' | |||||
1579 |
|
1579 | |||
1580 | # TODO this is a bunch of storage layer interface abstractions because |
|
1580 | # TODO this is a bunch of storage layer interface abstractions because | |
1581 | # it assumes revlogs. |
|
1581 | # it assumes revlogs. | |
1582 |
for rl_type, name, |
|
1582 | for rl_type, name, size in topfiles: | |
1583 | # XXX use the `rl_type` for that |
|
1583 | # XXX use the `rl_type` for that | |
1584 | if b'changelog' in files and name.startswith(b'00changelog'): |
|
1584 | if b'changelog' in files and name.startswith(b'00changelog'): | |
1585 | pass |
|
1585 | pass |
@@ -665,20 +665,24 b' def issimplestorefile(f, kind, st):' | |||||
665 |
|
665 | |||
666 |
|
666 | |||
667 | class simplestore(store.encodedstore): |
|
667 | class simplestore(store.encodedstore): | |
668 | def datafiles(self): |
|
668 | def datafiles(self, undecodable=None): | |
669 | for x in super(simplestore, self).datafiles(): |
|
669 | for x in super(simplestore, self).datafiles(): | |
670 | yield x |
|
670 | yield x | |
671 |
|
671 | |||
672 | # Supplement with non-revlog files. |
|
672 | # Supplement with non-revlog files. | |
673 | extrafiles = self._walk('data', True, filefilter=issimplestorefile) |
|
673 | extrafiles = self._walk('data', True, filefilter=issimplestorefile) | |
674 |
|
674 | |||
675 |
for |
|
675 | for f1, size in extrafiles: | |
676 | try: |
|
676 | try: | |
677 |
|
|
677 | f2 = store.decodefilename(f1) | |
678 | except KeyError: |
|
678 | except KeyError: | |
679 |
|
|
679 | if undecodable is None: | |
|
680 | raise error.StorageError(b'undecodable revlog name %s' % f1) | |||
|
681 | else: | |||
|
682 | undecodable.append(f1) | |||
|
683 | continue | |||
680 |
|
684 | |||
681 |
yield |
|
685 | yield f2, size | |
682 |
|
686 | |||
683 |
|
687 | |||
684 | def reposetup(ui, repo): |
|
688 | def reposetup(ui, repo): |
General Comments 0
You need to be logged in to leave comments.
Login now