# HG changeset patch # User Martin von Zweigbergk # Date 2016-02-04 16:34:07 # Node ID fb92927f977526795e9141e02141b589ebac902f # Parent bb45190a54b754dafa66b24c2414d33401c0596e treemanifests: fix streaming clone Similar to the previous patch, the .hg/store/meta/ directory does not get copied when when using "hg clone --uncompressed". Fix by including "meta/" in store.datafiles(). This seems safe to do, as there are only a few users of this method. "hg manifest" already filters the paths by "data/" prefix. The calls from largefiles also seem safe. The use in verify needs updating to prevent it from mistaking dirlogs for orphaned filelogs. That change is included in this patch. Since the dirlogs will now be in the fncache when using fncachestore, let's also update debugrebuildfncache(). That will also allow any existing treemanifest repos to get their dirlogs into the fncache. Also update test-treemanifest.t to use an a directory name that requires dot-encoding and uppercase-encoding so we test that the path encoding works. diff --git a/mercurial/repair.py b/mercurial/repair.py --- a/mercurial/repair.py +++ b/mercurial/repair.py @@ -273,6 +273,16 @@ def rebuildfncache(ui, repo): ui.progress(_('changeset'), None) + if 'treemanifest' in repo: # safe but unnecessary otherwise + for dir in util.dirs(seenfiles): + i = 'meta/%s/00manifest.i' % dir + d = 'meta/%s/00manifest.d' % dir + + if repo.store._exists(i): + newentries.add(i) + if repo.store._exists(d): + newentries.add(d) + addcount = len(newentries - oldentries) removecount = len(oldentries - newentries) for p in sorted(oldentries - newentries): diff --git a/mercurial/store.py b/mercurial/store.py --- a/mercurial/store.py +++ b/mercurial/store.py @@ -330,7 +330,7 @@ class basicstore(object): return l def datafiles(self): - return self._walk('data', True) + return self._walk('data', True) + self._walk('meta', True) def topfiles(self): # yield manifest before changelog @@ -378,7 +378,7 @@ class encodedstore(basicstore): self.opener = self.vfs def datafiles(self): - for a, b, size in self._walk('data', True): + for a, b, size in super(encodedstore, self).datafiles(): try: a = decodefilename(a) except KeyError: @@ -460,7 +460,8 @@ class _fncachevfs(scmutil.abstractvfs, s self.encode = encode def __call__(self, path, mode='r', *args, **kw): - if mode not in ('r', 'rb') and path.startswith('data/'): + if mode not in ('r', 'rb') and (path.startswith('data/') or + path.startswith('meta/')): self.fncache.add(path) return self.vfs(self.encode(path), mode, *args, **kw) diff --git a/mercurial/verify.py b/mercurial/verify.py --- a/mercurial/verify.py +++ b/mercurial/verify.py @@ -284,7 +284,7 @@ class verifier(object): for f, f2, size in repo.store.datafiles(): if not f: self.err(None, _("cannot decode filename '%s'") % f2) - elif size > 0 or not revlogv1: + elif (size > 0 or not revlogv1) and f.startswith('data/'): storefiles.add(_normpath(f)) files = sorted(set(filenodes) | set(filelinkrevs)) diff --git a/tests/run-tests.py b/tests/run-tests.py --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -719,6 +719,8 @@ class Test(unittest.TestCase): (br':%d\b' % self._startport, b':$HGPORT'), (br':%d\b' % (self._startport + 1), b':$HGPORT1'), (br':%d\b' % (self._startport + 2), b':$HGPORT2'), + (br':%d\b' % (self._startport + 2), b':$HGPORT3'), + (br':%d\b' % (self._startport + 2), b':$HGPORT4'), (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$', br'\1 (glob)'), ] @@ -741,6 +743,8 @@ class Test(unittest.TestCase): env["HGPORT"] = str(self._startport) env["HGPORT1"] = str(self._startport + 1) env["HGPORT2"] = str(self._startport + 2) + env["HGPORT3"] = str(self._startport + 3) + env["HGPORT4"] = str(self._startport + 4) env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc') env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids') env["HGEDITOR"] = ('"' + sys.executable + '"' diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t --- a/tests/test-treemanifest.t +++ b/tests/test-treemanifest.t @@ -367,7 +367,7 @@ Create deeper repo with tree manifests. $ hg --config experimental.treemanifest=True init deeprepo $ cd deeprepo - $ mkdir a + $ mkdir .A $ mkdir b $ mkdir b/bar $ mkdir b/bar/orange @@ -376,8 +376,8 @@ Create deeper repo with tree manifests. $ mkdir b/foo/apple $ mkdir b/foo/apple/bees - $ touch a/one.txt - $ touch a/two.txt + $ touch .A/one.txt + $ touch .A/two.txt $ touch b/bar/fruits.txt $ touch b/bar/orange/fly/gnat.py $ touch b/bar/orange/fly/housefly.txt @@ -393,8 +393,8 @@ the files command with various parameter Test files from the root. $ hg files -r . - a/one.txt (glob) - a/two.txt (glob) + .A/one.txt (glob) + .A/two.txt (glob) b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) @@ -412,7 +412,7 @@ Excludes with a glob should not exclude Test files for a subdirectory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ hg files -r . b b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) @@ -422,7 +422,7 @@ Test files for a subdirectory. Test files with just includes and excludes. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/bar/orange/fly $ rm -r .hg/store/meta/b/foo/apple/bees $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees @@ -431,7 +431,7 @@ Test files with just includes and exclud Test files for a subdirectory, excluding a directory within it. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -X path:b/foo b b/bar/fruits.txt (glob) @@ -442,7 +442,7 @@ Test files for a subdirectory, excluding Test files for a sub directory, including only a directory within it, and including an unrelated directory. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ hg files -r . -I path:b/bar/orange -I path:a b b/bar/orange/fly/gnat.py (glob) @@ -452,7 +452,7 @@ including an unrelated directory. Test files for a pattern, including a directory, and excluding a directory within that. - $ rm -r .hg/store/meta/a + $ rm -r .hg/store/meta/~2e_a $ rm -r .hg/store/meta/b/foo $ rm -r .hg/store/meta/b/bar/orange $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange @@ -488,8 +488,6 @@ requires got updated to include treemani Tree manifest revlogs exist. $ find deepclone/.hg/store/meta | sort deepclone/.hg/store/meta - deepclone/.hg/store/meta/a - deepclone/.hg/store/meta/a/00manifest.i deepclone/.hg/store/meta/b deepclone/.hg/store/meta/b/00manifest.i deepclone/.hg/store/meta/b/bar @@ -504,6 +502,8 @@ Tree manifest revlogs exist. deepclone/.hg/store/meta/b/foo/apple/00manifest.i deepclone/.hg/store/meta/b/foo/apple/bees deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i + deepclone/.hg/store/meta/~2e_a + deepclone/.hg/store/meta/~2e_a/00manifest.i Verify passes. $ cd deepclone $ hg verify @@ -525,8 +525,12 @@ Create clones using old repo formats to added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep store deeprepo-basicstore/.hg/requires + $ cd deeprepo-basicstore + $ grep store .hg/requires [1] + $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. $ hg clone --config format.usefncache=False \ > --config experimental.changegroup3=True \ > http://localhost:$HGPORT2 deeprepo-encodedstore @@ -537,8 +541,12 @@ Create clones using old repo formats to added 3 changesets with 10 changes to 8 files updating to branch default 8 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ grep fncache deeprepo-encodedstore/.hg/requires + $ cd deeprepo-encodedstore + $ grep fncache .hg/requires [1] + $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log + $ cat hg.pid >> $DAEMON_PIDS + $ cd .. Local clone with basicstore $ hg clone -U deeprepo-basicstore local-clone-basicstore @@ -566,3 +574,55 @@ Local clone with fncachestore crosschecking files in changesets and manifests checking files 8 files, 3 changesets, 10 total revisions + +Stream clone with basicstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT3 stream-clone-basicstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-basicstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with encodedstore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT4 stream-clone-encodedstore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-encodedstore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Stream clone with fncachestore + $ hg clone --config experimental.changegroup3=True --uncompressed -U \ + > http://localhost:$HGPORT2 stream-clone-fncachestore + streaming all changes + 18 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) + searching for changes + no changes found + $ hg -R stream-clone-fncachestore verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 8 files, 3 changesets, 10 total revisions + +Packed bundle + $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg + writing 3349 bytes for 18 files + bundle requirements: generaldelta, revlogv1, treemanifest + $ hg debugbundle --spec repo-packed.hg + none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest