stream-clone: allow to change persistent-nodemap format during stream clone

Persistent nodemap affect the store format. However it is fairly isolated and
fast to generate locally. So not making it a fixed part of the stream clone is
useful.

This allow clients without persistent-nodemap support (default for client
without Rust enabled, or simply older client). So it make it possible to enable
persistent nodemap on client, where it can provide a massive boost. without too
much consequence.

To do so, we stop using it in the advertisement requirements for streaming and
let the client add/remove the necessary file depending of its configuration.

We still send the files as it seems like a small save to not regenerate them.
In addition, the way we match them will overlap with the changelog-v2/revlog-v2
so we can't simply skip the associated patterns.

Differential Revision: https://phab.mercurial-scm.org/D12096

mercurial/requirements.py

@@ -113,7 +113,6 b' STREAM_FIXED_REQUIREMENTS = {'
     COPIESSDC_REQUIREMENT,
     GENERALDELTA_REQUIREMENT,
     INTERNAL_PHASE_REQUIREMENT,
-    NODEMAP_REQUIREMENT,
     REVLOG_COMPRESSION_ZSTD,
     REVLOGV1_REQUIREMENT,
     REVLOGV2_REQUIREMENT,

mercurial/revlogutils/nodemap.py

@@ -16,6 +16,7 b' from ..node import hex'
 
 from .. import (
     error,
+    requirements,
     util,
 )
 from . import docket as docket_mod
@@ -34,6 +35,19 b' def test_race_hook_1():'
     pass
 
 
+def post_stream_cleanup(repo):
+    """The stream clone might needs to remove some file if persisten nodemap
+    was dropped while stream cloning
+    """
+    if requirements.REVLOGV1_REQUIREMENT not in repo.requirements:
+        return
+    if requirements.NODEMAP_REQUIREMENT in repo.requirements:
+        return
+    unfi = repo.unfiltered()
+    delete_nodemap(None, unfi, unfi.changelog)
+    delete_nodemap(None, repo, unfi.manifestlog._rootstore._revlog)
+
+
 def persisted_data(revlog):
     """read the nodemap for a revlog from disk"""
     if revlog._nodemap_file is None:

mercurial/streamclone.py

@@ -27,6 +27,9 b' from . import ('
     store,
     util,
 )
+from .revlogutils import (
+    nodemap,
+)
 from .utils import (
     stringutil,
 )
@@ -216,6 +219,7 b' def maybeperformlegacystreamclone(pullop'
             repo.ui, repo.requirements, repo.features
         )
         scmutil.writereporequirements(repo)
+        nodemap.post_stream_cleanup(repo)
 
         if rbranchmap:
             repo._branchcaches.replace(repo, rbranchmap)
@@ -510,6 +514,7 b' def applybundlev1(repo, fp):'
         )
 
     consumev1(repo, fp, filecount, bytecount)
+    nodemap.post_stream_cleanup(repo)
 
 
 class streamcloneapplier(object):
@@ -826,6 +831,7 b' def applybundlev2(repo, fp, filecount, f'
         repo.ui, repo.requirements, repo.features
     )
     scmutil.writereporequirements(repo)
+    nodemap.post_stream_cleanup(repo)
 
 
 def _copy_files(src_vfs_map, dst_vfs_map, entries, progress):

tests/test-bundle.t

@@ -297,16 +297,16 b' packed1 is produced properly'
 
   $ hg -R test debugcreatestreamclonebundle packed.hg
   writing 2665 bytes for 6 files
-  bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog
+  bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog
 
   $ f -B 64 --size --sha1 --hexdump packed.hg
-  packed.hg: size=2884, sha1=b0c868701f8a9fe44daf094b2f5bf661cf90c789
+  packed.hg: size=2865, sha1=353d10311f4befa195d9a1ca4b8e26518115c702
   0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 06 00 00 |HGS1UN..........|
-  0010: 00 00 00 00 0a 69 00 4e 67 65 6e 65 72 61 6c 64 |.....i.Ngenerald|
-  0020: 65 6c 74 61 2c 70 65 72 73 69 73 74 65 6e 74 2d |elta,persistent-|
-  0030: 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f 67 2d 63 |nodemap,revlog-c|
+  0010: 00 00 00 00 0a 69 00 3b 67 65 6e 65 72 61 6c 64 |.....i.;generald|
+  0020: 65 6c 74 61 2c 72 65 76 6c 6f 67 2d 63 6f 6d 70 |elta,revlog-comp|
+  0030: 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c 72 65 76 |ression-zstd,rev|
   $ hg debugbundle --spec packed.hg
-  none-packed1;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog
+  none-packed1;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog
 #endif
 
 #if reporevlogstore no-rust zstd
@@ -357,17 +357,17 b' generaldelta requirement is not listed i'
 
   $ hg -R testnongd debugcreatestreamclonebundle packednongd.hg
   writing 301 bytes for 3 files
-  bundle requirements: persistent-nodemap, revlog-compression-zstd, revlogv1
+  bundle requirements: revlog-compression-zstd, revlogv1
 
   $ f -B 64 --size --sha1 --hexdump packednongd.hg
-  packednongd.hg: size=426, sha1=79563ccd6ef779bcfe62a4da64f89a1b308e92e0
+  packednongd.hg: size=407, sha1=0b8714422b785ba8eb98c916b41ffd5fb994c9b5
   0000: 48 47 53 31 55 4e 00 00 00 00 00 00 00 03 00 00 |HGS1UN..........|
-  0010: 00 00 00 00 01 2d 00 34 70 65 72 73 69 73 74 65 |.....-.4persiste|
-  0020: 6e 74 2d 6e 6f 64 65 6d 61 70 2c 72 65 76 6c 6f |nt-nodemap,revlo|
-  0030: 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 |g-compression-zs|
+  0010: 00 00 00 00 01 2d 00 21 72 65 76 6c 6f 67 2d 63 |.....-.!revlog-c|
+  0020: 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a 73 74 64 2c |ompression-zstd,|
+  0030: 72 65 76 6c 6f 67 76 31 00 64 61 74 61 2f 66 6f |revlogv1.data/fo|
 
   $ hg debugbundle --spec packednongd.hg
-  none-packed1;requirements%3Dpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1
+  none-packed1;requirements%3Drevlog-compression-zstd%2Crevlogv1
 
 #endif
 
@@ -427,7 +427,7 b' Warning emitted when packed bundles cont'
   $ hg -R testsecret debugcreatestreamclonebundle packedsecret.hg
   (warning: stream clone bundle will contain secret revisions)
   writing 301 bytes for 3 files
-  bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog
+  bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog
 
 #endif
 

tests/test-clone-stream-format.t

@@ -4,6 +4,11 b' This file contains tests case that deal '
 
 #testcases stream-legacy stream-bundle2
 
+  $ cat << EOF >> $HGRCPATH
+  > [storage]
+  > revlog.persistent-nodemap.slow-path=allow
+  > EOF
+
 #if stream-legacy
   $ cat << EOF >> $HGRCPATH
   > [server]
@@ -13,7 +18,7 b' This file contains tests case that deal '
 
 Initialize repository
 
-  $ hg init server --config format.use-share-safe=yes
+  $ hg init server --config format.use-share-safe=yes --config format.use-persistent-nodemap=yes
   $ cd server
   $ sh $TESTDIR/testlib/stream_clone_setup.sh
   adding 00changelog-ab349180a0405010.nd
@@ -345,3 +350,86 b' no-share-safe \xe2\x86\x92 share-safe cloning'
 
 
   $ killdaemons.py
+
+
+Test streaming from/to repository without a persistent-nodemap
+==============================================================
+
+persistent nodemap affects revlog, but they are easy to generate locally, so we allow it to be changed over a stream clone
+
+  $ rm hg-*.pid errors-*.txt
+  $ hg clone --pull --config format.use-persistent-nodemap=no server server-no-persistent-nodemap
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 5004 changesets with 1088 changes to 1088 files (+1 heads)
+  new changesets 96ee1d7354c4:06ddac466af5
+  updating to branch default
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg verify -R server-no-persistent-nodemap
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  checked 5004 changesets with 1088 changes to 1088 files
+  $ hg -R server serve -p $HGPORT -d --pid-file=hg-1.pid --error errors-1.txt
+  $ cat hg-1.pid > $DAEMON_PIDS
+  $ hg -R server-no-persistent-nodemap serve -p $HGPORT2 -d --pid-file=hg-2.pid --error errors-2.txt
+  $ cat hg-2.pid >> $DAEMON_PIDS
+  $ hg debugrequires -R server | grep persistent-nodemap
+  persistent-nodemap
+  $ hg debugrequires -R server-no-persistent-nodemap | grep persistent-nodemap
+  [1]
+  $ ls -1 server/.hg/store/00changelog*
+  server/.hg/store/00changelog-*.nd (glob)
+  server/.hg/store/00changelog.d
+  server/.hg/store/00changelog.i
+  server/.hg/store/00changelog.n
+  $ ls -1 server-no-persistent-nodemap/.hg/store/00changelog*
+  server-no-persistent-nodemap/.hg/store/00changelog.d
+  server-no-persistent-nodemap/.hg/store/00changelog.i
+
+persistent-nodemap → no-persistent-nodemap cloning
+
+  $ hg clone --quiet --stream -U http://localhost:$HGPORT clone-remove-persistent-nodemap --config format.use-persistent-nodemap=no
+  $ cat errors-1.txt
+  $ hg -R clone-remove-persistent-nodemap verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  checked 5004 changesets with 1088 changes to 1088 files
+  $ hg debugrequires -R clone-remove-persistent-nodemap | grep persistent-nodemap
+  [1]
+
+The persistent-nodemap files should no longer exists
+
+  $ ls -1 clone-remove-persistent-nodemap/.hg/store/00changelog*
+  clone-remove-persistent-nodemap/.hg/store/00changelog.d
+  clone-remove-persistent-nodemap/.hg/store/00changelog.i
+
+
+no-persistent-nodemap → persistent-nodemap cloning
+
+  $ hg clone --quiet --stream -U http://localhost:$HGPORT2 clone-add-persistent-nodemap --config format.use-persistent-nodemap=yes
+  $ cat errors-2.txt
+  $ hg -R clone-add-persistent-nodemap verify
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  checked 5004 changesets with 1088 changes to 1088 files
+  $ hg debugrequires -R clone-add-persistent-nodemap | grep persistent-nodemap
+  persistent-nodemap
+
+The persistent-nodemap files should exists
+
+  $ ls -1 clone-add-persistent-nodemap/.hg/store/00changelog*
+  clone-add-persistent-nodemap/.hg/store/00changelog-*.nd (glob)
+  clone-add-persistent-nodemap/.hg/store/00changelog.d
+  clone-add-persistent-nodemap/.hg/store/00changelog.i
+  clone-add-persistent-nodemap/.hg/store/00changelog.n
+
+
+  $ killdaemons.py

tests/test-clone-stream.t

@@ -338,23 +338,23 b' getbundle requests with stream=1 are unc'
 #endif
 #if zstd rust no-dirstate-v2
   $ f --size --hex --bytes 256 body
-  body: size=116331
+  body: size=116310
   0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......|
-  0010: 91 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 |..STREAM2.......|
-  0020: 06 09 04 0c 55 62 79 74 65 63 6f 75 6e 74 31 30 |....Ubytecount10|
+  0010: 7c 07 53 54 52 45 41 4d 32 00 00 00 00 03 00 09 ||.STREAM2.......|
+  0020: 06 09 04 0c 40 62 79 74 65 63 6f 75 6e 74 31 30 |....@bytecount10|
   0030: 31 32 37 36 66 69 6c 65 63 6f 75 6e 74 31 30 39 |1276filecount109|
   0040: 33 72 65 71 75 69 72 65 6d 65 6e 74 73 67 65 6e |3requirementsgen|
-  0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 70 65 72 73 |eraldelta%2Cpers|
-  0060: 69 73 74 65 6e 74 2d 6e 6f 64 65 6d 61 70 25 32 |istent-nodemap%2|
-  0070: 43 72 65 76 6c 6f 67 2d 63 6f 6d 70 72 65 73 73 |Crevlog-compress|
-  0080: 69 6f 6e 2d 7a 73 74 64 25 32 43 72 65 76 6c 6f |ion-zstd%2Crevlo|
-  0090: 67 76 31 25 32 43 73 70 61 72 73 65 72 65 76 6c |gv1%2Csparserevl|
-  00a0: 6f 67 00 00 80 00 73 08 42 64 61 74 61 2f 30 2e |og....s.Bdata/0.|
-  00b0: 69 00 03 00 01 00 00 00 00 00 00 00 02 00 00 00 |i...............|
-  00c0: 01 00 00 00 00 00 00 00 01 ff ff ff ff ff ff ff |................|
-  00d0: ff 80 29 63 a0 49 d3 23 87 bf ce fe 56 67 92 67 |..)c.I.#....Vg.g|
-  00e0: 2c 69 d1 ec 39 00 00 00 00 00 00 00 00 00 00 00 |,i..9...........|
-  00f0: 00 75 30 73 26 45 64 61 74 61 2f 30 30 63 68 61 |.u0s&Edata/00cha|
+  0050: 65 72 61 6c 64 65 6c 74 61 25 32 43 72 65 76 6c |eraldelta%2Crevl|
+  0060: 6f 67 2d 63 6f 6d 70 72 65 73 73 69 6f 6e 2d 7a |og-compression-z|
+  0070: 73 74 64 25 32 43 72 65 76 6c 6f 67 76 31 25 32 |std%2Crevlogv1%2|
+  0080: 43 73 70 61 72 73 65 72 65 76 6c 6f 67 00 00 80 |Csparserevlog...|
+  0090: 00 73 08 42 64 61 74 61 2f 30 2e 69 00 03 00 01 |.s.Bdata/0.i....|
+  00a0: 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 |................|
+  00b0: 00 00 00 01 ff ff ff ff ff ff ff ff 80 29 63 a0 |.............)c.|
+  00c0: 49 d3 23 87 bf ce fe 56 67 92 67 2c 69 d1 ec 39 |I.#....Vg.g,i..9|
+  00d0: 00 00 00 00 00 00 00 00 00 00 00 00 75 30 73 26 |............u0s&|
+  00e0: 45 64 61 74 61 2f 30 30 63 68 61 6e 67 65 6c 6f |Edata/00changelo|
+  00f0: 67 2d 61 62 33 34 39 31 38 30 61 30 34 30 35 30 |g-ab349180a04050|
 #endif
 #if zstd dirstate-v2
   $ f --size --hex --bytes 256 body

tests/test-clonebundles.t

@@ -281,7 +281,7 b' Stream clone bundles are supported'
   writing 613 bytes for 4 files
   bundle requirements: generaldelta, revlogv1, sparserevlog (no-rust no-zstd !)
   bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (no-rust zstd !)
-  bundle requirements: generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog (rust !)
+  bundle requirements: generaldelta, revlog-compression-zstd, revlogv1, sparserevlog (rust !)
 
 No bundle spec should work
 

tests/test-debugcommands.t

@@ -657,8 +657,8 b' Test debugpeer'
   devel-peer-request:   pairs: 81 bytes
   sending hello command
   sending between command
-  remote: 487
-  remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,persistent-nodemap,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
+  remote: 468
+  remote: capabilities: batch branchmap $USUAL_BUNDLE2_CAPS$ changegroupsubset getbundle known lookup protocaps pushkey streamreqs=generaldelta,revlog-compression-zstd,revlogv1,sparserevlog unbundle=HG10GZ,HG10BZ,HG10UN unbundlehash
   remote: 1
   devel-peer-request: protocaps
   devel-peer-request:   caps: * bytes (glob)

tests/test-stream-bundle-v2.t

@@ -47,11 +47,11 b' The extension requires a repo (currently'
   Stream params: {}
   stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlogv1%2Csparserevlog} (mandatory: True) (no-zstd !)
   stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (zstd no-rust !)
-  stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !)
+  stream2 -- {bytecount: 1693, filecount: 11, requirements: generaldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog} (mandatory: True) (rust !)
   $ hg debugbundle --spec bundle.hg
   none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlogv1%2Csparserevlog (no-zstd !)
   none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (zstd no-rust !)
-  none-v2;stream=v2;requirements%3Dgeneraldelta%2Cpersistent-nodemap%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !)
+  none-v2;stream=v2;requirements%3Dgeneraldelta%2Crevlog-compression-zstd%2Crevlogv1%2Csparserevlog (rust !)
 
 Test that we can apply the bundle as a stream clone bundle