treemanifests: fix streaming clone

Similar to the previous patch, the .hg/store/meta/ directory does not
get copied when when using "hg clone --uncompressed". Fix by including
"meta/" in store.datafiles(). This seems safe to do, as there are only
a few users of this method. "hg manifest" already filters the paths by
"data/" prefix. The calls from largefiles also seem safe. The use in
verify needs updating to prevent it from mistaking dirlogs for
orphaned filelogs. That change is included in this patch.

Since the dirlogs will now be in the fncache when using fncachestore,
let's also update debugrebuildfncache(). That will also allow any
existing treemanifest repos to get their dirlogs into the fncache.

Also update test-treemanifest.t to use an a directory name that
requires dot-encoding and uppercase-encoding so we test that the path
encoding works.

mercurial/repair.py

@@ -273,6 +273,16 b' def rebuildfncache(ui, repo):'
 
         ui.progress(_('changeset'), None)
 
+        if 'treemanifest' in repo: # safe but unnecessary otherwise
+            for dir in util.dirs(seenfiles):
+                i = 'meta/%s/00manifest.i' % dir
+                d = 'meta/%s/00manifest.d' % dir
+
+                if repo.store._exists(i):
+                    newentries.add(i)
+                if repo.store._exists(d):
+                    newentries.add(d)
+
         addcount = len(newentries - oldentries)
         removecount = len(oldentries - newentries)
         for p in sorted(oldentries - newentries):

mercurial/store.py

@@ -330,7 +330,7 b' class basicstore(object):'
         return l
 
     def datafiles(self):
-        return self._walk('data', True)
+        return self._walk('data', True) + self._walk('meta', True)
 
     def topfiles(self):
         # yield manifest before changelog
@@ -378,7 +378,7 b' class encodedstore(basicstore):'
         self.opener = self.vfs
 
     def datafiles(self):
-        for a, b, size in self._walk('data', True):
+        for a, b, size in super(encodedstore, self).datafiles():
             try:
                 a = decodefilename(a)
             except KeyError:
@@ -460,7 +460,8 b' class _fncachevfs(scmutil.abstractvfs, s'
         self.encode = encode
 
     def __call__(self, path, mode='r', *args, **kw):
-        if mode not in ('r', 'rb') and path.startswith('data/'):
+        if mode not in ('r', 'rb') and (path.startswith('data/') or
+                                        path.startswith('meta/')):
             self.fncache.add(path)
         return self.vfs(self.encode(path), mode, *args, **kw)
 

mercurial/verify.py

@@ -284,7 +284,7 b' class verifier(object):'
         for f, f2, size in repo.store.datafiles():
             if not f:
                 self.err(None, _("cannot decode filename '%s'") % f2)
-            elif size > 0 or not revlogv1:
+            elif (size > 0 or not revlogv1) and f.startswith('data/'):
                 storefiles.add(_normpath(f))
 
         files = sorted(set(filenodes) | set(filelinkrevs))

tests/run-tests.py

@@ -719,6 +719,8 b' class Test(unittest.TestCase):'
             (br':%d\b' % self._startport, b':$HGPORT'),
             (br':%d\b' % (self._startport + 1), b':$HGPORT1'),
             (br':%d\b' % (self._startport + 2), b':$HGPORT2'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT3'),
+            (br':%d\b' % (self._startport + 2), b':$HGPORT4'),
             (br'(?m)^(saved backup bundle to .*\.hg)( \(glob\))?$',
              br'\1 (glob)'),
             ]
@@ -741,6 +743,8 b' class Test(unittest.TestCase):'
         env["HGPORT"] = str(self._startport)
         env["HGPORT1"] = str(self._startport + 1)
         env["HGPORT2"] = str(self._startport + 2)
+        env["HGPORT3"] = str(self._startport + 3)
+        env["HGPORT4"] = str(self._startport + 4)
         env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc')
         env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids')
         env["HGEDITOR"] = ('"' + sys.executable + '"'

tests/test-treemanifest.t

@@ -367,7 +367,7 b' Create deeper repo with tree manifests.'
   $ hg --config experimental.treemanifest=True init deeprepo
   $ cd deeprepo
 
-  $ mkdir a
+  $ mkdir .A
   $ mkdir b
   $ mkdir b/bar
   $ mkdir b/bar/orange
@@ -376,8 +376,8 b' Create deeper repo with tree manifests.'
   $ mkdir b/foo/apple
   $ mkdir b/foo/apple/bees
 
-  $ touch a/one.txt
-  $ touch a/two.txt
+  $ touch .A/one.txt
+  $ touch .A/two.txt
   $ touch b/bar/fruits.txt
   $ touch b/bar/orange/fly/gnat.py
   $ touch b/bar/orange/fly/housefly.txt
@@ -393,8 +393,8 b' the files command with various parameter'
 Test files from the root.
 
   $ hg files -r .
-  a/one.txt (glob)
-  a/two.txt (glob)
+  .A/one.txt (glob)
+  .A/two.txt (glob)
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
   b/bar/orange/fly/housefly.txt (glob)
@@ -412,7 +412,7 b' Excludes with a glob should not exclude '
 
 Test files for a subdirectory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ hg files -r . b
   b/bar/fruits.txt (glob)
   b/bar/orange/fly/gnat.py (glob)
@@ -422,7 +422,7 b' Test files for a subdirectory.'
 
 Test files with just includes and excludes.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/bar/orange/fly
   $ rm -r .hg/store/meta/b/foo/apple/bees
   $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees
@@ -431,7 +431,7 b' Test files with just includes and exclud'
 
 Test files for a subdirectory, excluding a directory within it.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -X path:b/foo b
   b/bar/fruits.txt (glob)
@@ -442,7 +442,7 b' Test files for a subdirectory, excluding'
 Test files for a sub directory, including only a directory within it, and
 including an unrelated directory.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ hg files -r . -I path:b/bar/orange -I path:a b
   b/bar/orange/fly/gnat.py (glob)
@@ -452,7 +452,7 b' including an unrelated directory.'
 Test files for a pattern, including a directory, and excluding a directory
 within that.
 
-  $ rm -r .hg/store/meta/a
+  $ rm -r .hg/store/meta/~2e_a
   $ rm -r .hg/store/meta/b/foo
   $ rm -r .hg/store/meta/b/bar/orange
   $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange
@@ -488,8 +488,6 b' requires got updated to include treemani'
 Tree manifest revlogs exist.
   $ find deepclone/.hg/store/meta | sort
   deepclone/.hg/store/meta
-  deepclone/.hg/store/meta/a
-  deepclone/.hg/store/meta/a/00manifest.i
   deepclone/.hg/store/meta/b
   deepclone/.hg/store/meta/b/00manifest.i
   deepclone/.hg/store/meta/b/bar
@@ -504,6 +502,8 b' Tree manifest revlogs exist.'
   deepclone/.hg/store/meta/b/foo/apple/00manifest.i
   deepclone/.hg/store/meta/b/foo/apple/bees
   deepclone/.hg/store/meta/b/foo/apple/bees/00manifest.i
+  deepclone/.hg/store/meta/~2e_a
+  deepclone/.hg/store/meta/~2e_a/00manifest.i
 Verify passes.
   $ cd deepclone
   $ hg verify
@@ -525,8 +525,12 b' Create clones using old repo formats to '
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep store deeprepo-basicstore/.hg/requires
+  $ cd deeprepo-basicstore
+  $ grep store .hg/requires
   [1]
+  $ hg serve -p $HGPORT3 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
   $ hg clone --config format.usefncache=False \
   >   --config experimental.changegroup3=True \
   >   http://localhost:$HGPORT2 deeprepo-encodedstore
@@ -537,8 +541,12 b' Create clones using old repo formats to '
   added 3 changesets with 10 changes to 8 files
   updating to branch default
   8 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ grep fncache deeprepo-encodedstore/.hg/requires
+  $ cd deeprepo-encodedstore
+  $ grep fncache .hg/requires
   [1]
+  $ hg serve -p $HGPORT4 -d --pid-file=hg.pid --errorlog=errors.log
+  $ cat hg.pid >> $DAEMON_PIDS
+  $ cd ..
 
 Local clone with basicstore
   $ hg clone -U deeprepo-basicstore local-clone-basicstore
@@ -566,3 +574,55 b' Local clone with fncachestore'
   crosschecking files in changesets and manifests
   checking files
   8 files, 3 changesets, 10 total revisions
+
+Stream clone with basicstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT3 stream-clone-basicstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-basicstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with encodedstore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT4 stream-clone-encodedstore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-encodedstore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Stream clone with fncachestore
+  $ hg clone --config experimental.changegroup3=True --uncompressed -U \
+  >   http://localhost:$HGPORT2 stream-clone-fncachestore
+  streaming all changes
+  18 files to transfer, * of data (glob)
+  transferred * in * seconds (*) (glob)
+  searching for changes
+  no changes found
+  $ hg -R stream-clone-fncachestore verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  8 files, 3 changesets, 10 total revisions
+
+Packed bundle
+  $ hg -R deeprepo debugcreatestreamclonebundle repo-packed.hg
+  writing 3349 bytes for 18 files
+  bundle requirements: generaldelta, revlogv1, treemanifest
+  $ hg debugbundle --spec repo-packed.hg
+  none-packed1;requirements%3Dgeneraldelta%2Crevlogv1%2Ctreemanifest