# HG changeset patch # User Pierre-Yves David # Date 2021-01-12 21:51:45 # Node ID f62598402ef0c73d66bcae5a45ae68b56e737ff3 # Parent de2ed219fd005cf2be64e868651898ab21ab75db clone: make sure we warm the cache after a clone This work around any deviciency/limitation of the clone process. In our case this ensure the persistent nodemap exist with valid content. Ideally, the cloning process would also do "the right thing". However since older server will never be able to do "the right thing". The local workaround will be necessary anyway. I am not worried by the performance impact of this as `hg clone` is non-instant on large repositories where is could matters. Warming the cache if they are already correct is very fast. And if they are not already warm, this seems like a good time to do so. This impact various test as more cache are now warmed sooner, all the change should be harmless. Differential Revision: https://phab.mercurial-scm.org/D9735 diff --git a/mercurial/hg.py b/mercurial/hg.py --- a/mercurial/hg.py +++ b/mercurial/hg.py @@ -1038,6 +1038,14 @@ def clone( _update(destrepo, uprev) if update in destrepo._bookmarks: bookmarks.activate(destrepo, update) + if destlock is not None: + release(destlock) + # here is a tiny windows were someone could end up writing the + # repository before the cache are sure to be warm. This if "fine" + # as the only "bad" outcome would be some slowness. That potential + # slowness already affect reader. + with destrepo.lock(): + destrepo.updatecaches(full=True) finally: release(srclock, destlock) if cleandir is not None: diff --git a/tests/test-acl.t b/tests/test-acl.t --- a/tests/test-acl.t +++ b/tests/test-acl.t @@ -807,6 +807,7 @@ fred is not blocked from moving bookmark acl: bookmark access granted: "ef1ea85a6374b77d6da9dcda9541f498f2d17df7" on bookmark "moving-bookmark" bundle2-input-bundle: 7 parts total updating the branch cache + invalid branch cache (served.hidden): tip differs added 1 changesets with 1 changes to 1 files bundle2-output-bundle: "HG20", 1 parts total bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload @@ -1788,6 +1789,7 @@ No branch acls specified bundle2-input-part: total payload size 48 bundle2-input-bundle: 5 parts total updating the branch cache + invalid branch cache (served.hidden): tip differs added 4 changesets with 4 changes to 4 files (+1 heads) bundle2-output-bundle: "HG20", 1 parts total bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload @@ -2094,6 +2096,7 @@ Branch acl allow other bundle2-input-part: total payload size 48 bundle2-input-bundle: 5 parts total updating the branch cache + invalid branch cache (served.hidden): tip differs added 4 changesets with 4 changes to 4 files (+1 heads) bundle2-output-bundle: "HG20", 1 parts total bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload @@ -2185,6 +2188,7 @@ push foobar into the remote bundle2-input-part: total payload size 48 bundle2-input-bundle: 5 parts total updating the branch cache + invalid branch cache (served.hidden): tip differs added 4 changesets with 4 changes to 4 files (+1 heads) bundle2-output-bundle: "HG20", 1 parts total bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload @@ -2348,6 +2352,7 @@ User 'astro' must not be denied bundle2-input-part: total payload size 48 bundle2-input-bundle: 5 parts total updating the branch cache + invalid branch cache (served.hidden): tip differs added 4 changesets with 4 changes to 4 files (+1 heads) bundle2-output-bundle: "HG20", 1 parts total bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload diff --git a/tests/test-blackbox.t b/tests/test-blackbox.t --- a/tests/test-blackbox.t +++ b/tests/test-blackbox.t @@ -128,9 +128,9 @@ clone, commit, pull new changesets d02f48003e62 (run 'hg update' to get a working copy) $ hg blackbox -l 6 - 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> pull - 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> updated branch cache (served) in * seconds (glob) 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> wrote branch cache (served) with 1 labels and 2 nodes + 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> updated branch cache (served.hidden) in * seconds (glob) + 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> wrote branch cache (served.hidden) with 1 labels and 2 nodes 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> 1 incoming changes - new heads: d02f48003e62 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> pull exited 0 after * seconds (glob) 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> blackbox -l 6 @@ -193,8 +193,8 @@ backup bundles get logged $ hg blackbox -l 6 1970/01/01 00:00:00 bob @73f6ee326b27d820b0472f1a825e3a50f3dc489b (5000)> strip tip 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> saved backup bundle to $TESTTMP/blackboxtest2/.hg/strip-backup/73f6ee326b27-7612e004-backup.hg - 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> updated branch cache (base) in * seconds (glob) - 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> wrote branch cache (base) with 1 labels and 2 nodes + 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> updated branch cache (immutable) in * seconds (glob) + 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> wrote branch cache (immutable) with 1 labels and 2 nodes 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> strip tip exited 0 after * seconds (glob) 1970/01/01 00:00:00 bob @6563da9dcf87b1949716e38ff3e3dfaa3198eb06 (5000)> blackbox -l 6 diff --git a/tests/test-clone-uncompressed.t b/tests/test-clone-uncompressed.t --- a/tests/test-clone-uncompressed.t +++ b/tests/test-clone-uncompressed.t @@ -192,9 +192,17 @@ Basic clone transferred 96.5 KB in * seconds (* */sec) (glob) $ ls -1 clone1/.hg/cache + branch2-base + branch2-immutable branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden + hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served #endif getbundle requests with stream=1 are uncompressed @@ -265,6 +273,7 @@ Clone with background file closing enabl bundle2-input-part: total payload size 24 bundle2-input-bundle: 2 parts total checking for updated bookmarks + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) #endif #if stream-bundle2 @@ -287,6 +296,7 @@ Clone with background file closing enabl bundle2-input-part: "listkeys" (params: 1 mandatory) supported bundle2-input-bundle: 2 parts total checking for updated bookmarks + updating the branch cache (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) #endif diff --git a/tests/test-clone.t b/tests/test-clone.t --- a/tests/test-clone.t +++ b/tests/test-clone.t @@ -56,9 +56,17 @@ Default operation: Ensure branchcache got copied over: $ ls .hg/cache + branch2-base + branch2-immutable branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden + hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served $ cat a a @@ -99,6 +107,7 @@ No update, with debug option: linking: 17 files (reposimplestore !) linking: 18 files (reposimplestore !) linked 18 files (reposimplestore !) + updating the branch cache #else $ hg --debug clone -U . ../c --config progress.debug=true linking: 1 files @@ -127,9 +136,17 @@ No update, with debug option: Ensure branchcache got copied over: $ ls .hg/cache + branch2-base + branch2-immutable branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden + hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served $ cat a 2>/dev/null || echo "a not present" a not present diff --git a/tests/test-clonebundles.t b/tests/test-clonebundles.t --- a/tests/test-clonebundles.t +++ b/tests/test-clonebundles.t @@ -597,6 +597,7 @@ on a 32MB system. added 2 changesets with 2 changes to 2 files new changesets 53245c60e682:aaff8d2ffbbf calling hook changegroup.lfiles: hgext.largefiles.reposetup.checkrequireslfiles + updating the branch cache (sent 4 HTTP requests and * bytes; received * bytes in responses) (glob) $ hg clone -U --debug --config ui.available-memory=32MB http://localhost:$HGPORT gzip-too-large2 @@ -634,5 +635,6 @@ on a 32MB system. checking for updated bookmarks 2 local changesets published calling hook changegroup.lfiles: hgext.largefiles.reposetup.checkrequireslfiles + updating the branch cache (sent 4 HTTP requests and * bytes; received * bytes in responses) (glob) $ killdaemons.py diff --git a/tests/test-empty.t b/tests/test-empty.t --- a/tests/test-empty.t +++ b/tests/test-empty.t @@ -44,6 +44,7 @@ Poke at a clone: checked 0 changesets with 0 changes to 0 files $ ls .hg 00changelog.i + cache hgrc requires store diff --git a/tests/test-eol-clone.t b/tests/test-eol-clone.t --- a/tests/test-eol-clone.t +++ b/tests/test-eol-clone.t @@ -89,6 +89,7 @@ the source repo: a.txt: remote created -> g getting a.txt 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + updating the branch cache $ cd repo-3 $ cat a.txt diff --git a/tests/test-hardlinks.t b/tests/test-hardlinks.t --- a/tests/test-hardlinks.t +++ b/tests/test-hardlinks.t @@ -69,6 +69,7 @@ Create hardlinked clone r2: linking: 6 files linking: 7 files linked 7 files + updating the branch cache Create non-hardlinked clone r3: @@ -238,9 +239,16 @@ r4 has hardlinks in the working dir (not 2 r4/.hg/00changelog.i 2 r4/.hg/branch 2 r4/.hg/cache/branch2-base + 2 r4/.hg/cache/branch2-immutable 2 r4/.hg/cache/branch2-served + 2 r4/.hg/cache/branch2-served.hidden + 2 r4/.hg/cache/branch2-visible + 2 r4/.hg/cache/branch2-visible-hidden + 2 r4/.hg/cache/hgtagsfnodes1 2 r4/.hg/cache/rbc-names-v1 2 r4/.hg/cache/rbc-revs-v1 + 2 r4/.hg/cache/tags2 + 2 r4/.hg/cache/tags2-served 2 r4/.hg/dirstate 2 r4/.hg/fsmonitor.state (fsmonitor !) 2 r4/.hg/hgrc @@ -289,9 +297,16 @@ Update back to revision 12 in r4 should 2 r4/.hg/00changelog.i 1 r4/.hg/branch 2 r4/.hg/cache/branch2-base + 2 r4/.hg/cache/branch2-immutable 2 r4/.hg/cache/branch2-served + 2 r4/.hg/cache/branch2-served.hidden + 2 r4/.hg/cache/branch2-visible + 2 r4/.hg/cache/branch2-visible-hidden + 2 r4/.hg/cache/hgtagsfnodes1 2 r4/.hg/cache/rbc-names-v1 2 r4/.hg/cache/rbc-revs-v1 + 2 r4/.hg/cache/tags2 + 2 r4/.hg/cache/tags2-served 1 r4/.hg/dirstate 1 r4/.hg/fsmonitor.state (fsmonitor !) 2 r4/.hg/hgrc diff --git a/tests/test-lfs-serve-access.t b/tests/test-lfs-serve-access.t --- a/tests/test-lfs-serve-access.t +++ b/tests/test-lfs-serve-access.t @@ -158,6 +158,7 @@ Blob URIs are correct when --prefix is u getting lfs.bin lfs: found f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e in the local lfs store 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + updating the branch cache (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t --- a/tests/test-persistent-nodemap.t +++ b/tests/test-persistent-nodemap.t @@ -608,8 +608,17 @@ The persistent nodemap should exist afte $ hg clone -U test-repo local-clone $ ls -1 local-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)' - [1] + 00changelog-*.nd (glob) + 00changelog.n + 00manifest-*.nd (glob) + 00manifest.n $ hg -R local-clone debugnodemap --metadata + uid: * (glob) + tip-rev: 5005 + tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe + data-length: 121088 + data-unused: 0 + data-unused: 0.000% stream clone ------------ @@ -618,5 +627,14 @@ The persistent nodemap should exist afte $ hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone --quiet $ ls -1 stream-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)' - [1] + 00changelog-*.nd (glob) + 00changelog.n + 00manifest-*.nd (glob) + 00manifest.n $ hg -R stream-clone debugnodemap --metadata + uid: * (glob) + tip-rev: 5005 + tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe + data-length: 121088 + data-unused: 0 + data-unused: 0.000% diff --git a/tests/test-share.t b/tests/test-share.t --- a/tests/test-share.t +++ b/tests/test-share.t @@ -62,9 +62,17 @@ Cloning a shared repo should pick up the updating to branch default 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ ls -1 ../repo2-clone/.hg/cache + branch2-base + branch2-immutable branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden + hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served tags2-visible Some sed versions appends newline, some don't, and some just fails diff --git a/tests/test-ssh.t b/tests/test-ssh.t --- a/tests/test-ssh.t +++ b/tests/test-ssh.t @@ -84,8 +84,8 @@ clone bookmarks via stream $ hg -R local-stream book mybook $ hg clone -e "\"$PYTHON\" \"$TESTDIR/dummyssh\"" --stream ssh://user@dummy/local-stream stream2 streaming all changes - 9 files to transfer, 870 bytes of data - transferred 870 bytes in * seconds (*) (glob) + 16 files to transfer, * of data (glob) + transferred * in * seconds (*) (glob) updating to branch default 2 files updated, 0 files merged, 0 files removed, 0 files unresolved $ cd stream2 diff --git a/tests/test-stream-bundle-v2.t b/tests/test-stream-bundle-v2.t --- a/tests/test-stream-bundle-v2.t +++ b/tests/test-stream-bundle-v2.t @@ -117,6 +117,7 @@ Test that we can apply the bundle as a s E: remote created -> g getting E 5 files updated, 0 files merged, 0 files removed, 0 files unresolved + updating the branch cache (sent 4 HTTP requests and * bytes; received * bytes in responses) (glob) $ hg clone --stream http://localhost:$HGPORT streamv2-clone-explicit --debug @@ -173,4 +174,5 @@ Test that we can apply the bundle as a s E: remote created -> g getting E 5 files updated, 0 files merged, 0 files removed, 0 files unresolved + updating the branch cache (sent 4 HTTP requests and * bytes; received * bytes in responses) (glob) diff --git a/tests/test-tags.t b/tests/test-tags.t --- a/tests/test-tags.t +++ b/tests/test-tags.t @@ -721,19 +721,26 @@ Missing tags2* files means the cache was $ ls tagsclient/.hg/cache branch2-base + branch2-immutable + branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served Cache should contain the head only, even though other nodes have tags data $ f --size --hexdump tagsclient/.hg/cache/hgtagsfnodes1 tagsclient/.hg/cache/hgtagsfnodes1: size=96 - 0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0030: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0040: ff ff ff ff ff ff ff ff 40 f0 35 8c 19 e0 a7 d3 |........@.5.....| + 0000: 96 ee 1d 73 00 00 00 00 00 00 00 00 00 00 00 00 |...s............| + 0010: 00 00 00 00 00 00 00 00 c4 da b0 c2 94 65 e1 c6 |.............e..| + 0020: 0d f7 f0 dd 32 04 ea 57 78 c8 97 97 79 fc d5 95 |....2..Wx...y...| + 0030: f6 3c c8 fe 94 65 e1 c6 0d f7 f0 dd 32 04 ea 57 |.<...e......2..W| + 0040: 78 c8 97 97 79 fc d5 95 40 f0 35 8c 19 e0 a7 d3 |x...y...@.5.....| 0050: 8a 5c 6a 82 4d cf fb a5 87 d0 2f a3 1e 4f 2f 8a |.\j.M...../..O/.| Running hg tags should produce tags2* file and not change cache @@ -745,18 +752,25 @@ Running hg tags should produce tags2* fi $ ls tagsclient/.hg/cache branch2-base + branch2-immutable + branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 + tags2 + tags2-served tags2-visible $ f --size --hexdump tagsclient/.hg/cache/hgtagsfnodes1 tagsclient/.hg/cache/hgtagsfnodes1: size=96 - 0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0030: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................| - 0040: ff ff ff ff ff ff ff ff 40 f0 35 8c 19 e0 a7 d3 |........@.5.....| + 0000: 96 ee 1d 73 00 00 00 00 00 00 00 00 00 00 00 00 |...s............| + 0010: 00 00 00 00 00 00 00 00 c4 da b0 c2 94 65 e1 c6 |.............e..| + 0020: 0d f7 f0 dd 32 04 ea 57 78 c8 97 97 79 fc d5 95 |....2..Wx...y...| + 0030: f6 3c c8 fe 94 65 e1 c6 0d f7 f0 dd 32 04 ea 57 |.<...e......2..W| + 0040: 78 c8 97 97 79 fc d5 95 40 f0 35 8c 19 e0 a7 d3 |x...y...@.5.....| 0050: 8a 5c 6a 82 4d cf fb a5 87 d0 2f a3 1e 4f 2f 8a |.\j.M...../..O/.| Check that the bundle includes cache data @@ -779,6 +793,8 @@ Check that local clone includes cache da updating to branch default 2 files updated, 0 files merged, 0 files removed, 0 files unresolved $ (cd tags-local-clone/.hg/cache/; ls -1 tag*) + tags2 + tags2-served tags2-visible Avoid writing logs on trying to delete an already deleted tag diff --git a/tests/test-treemanifest.t b/tests/test-treemanifest.t --- a/tests/test-treemanifest.t +++ b/tests/test-treemanifest.t @@ -792,7 +792,7 @@ Stream clone with basicstore $ hg clone --config experimental.changegroup3=True --stream -U \ > http://localhost:$HGPORT1 stream-clone-basicstore streaming all changes - 21 files to transfer, * of data (glob) + 29 files to transfer, * of data (glob) transferred * in * seconds (*) (glob) $ hg -R stream-clone-basicstore verify checking changesets @@ -806,7 +806,7 @@ Stream clone with encodedstore $ hg clone --config experimental.changegroup3=True --stream -U \ > http://localhost:$HGPORT2 stream-clone-encodedstore streaming all changes - 21 files to transfer, * of data (glob) + 29 files to transfer, * of data (glob) transferred * in * seconds (*) (glob) $ hg -R stream-clone-encodedstore verify checking changesets diff --git a/tests/test-wireproto-exchangev2.t b/tests/test-wireproto-exchangev2.t --- a/tests/test-wireproto-exchangev2.t +++ b/tests/test-wireproto-exchangev2.t @@ -111,6 +111,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:caa2a465451d (3 drafts) + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -239,6 +240,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:4432d83626e8 + updating the branch cache (sent 6 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -557,6 +559,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:caa2a465451d (1 drafts) + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -758,6 +761,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:97765fc3cd62 + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -872,6 +876,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:97765fc3cd62 + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -988,6 +993,7 @@ Output is flaky, save it in a file and c } updating the branch cache new changesets 3390ef850073:97765fc3cd62 + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -1087,6 +1093,7 @@ Output is flaky, save it in a file and c } ] } + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame" @@ -1183,6 +1190,7 @@ Output is flaky, save it in a file and c } ] } + updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) $ cat clone-output | grep "received frame"