bundle2: store changeset count when creating file bundles

The bundle2 changegroup part has an advisory param saying how many
changesets are in the part. Before this patch, we were setting
this part when generating bundle2 parts via the wire protocol but
not when generating local bundle2 files.

A side effect of not setting the changeset count part is that progress
bars don't work when applying changesets. As the tests show, this
impacted clone bundles, shelve, backup bundles, `hg unbundle`, and
anything touching bundle2 files.

This patch adds a backdoor to allow us to pass state from
changegroup generation into the unbundler. We store the number
of changesets in the changegroup in this state and use it to
populate the aforementioned advisory part parameter when generating
the bundle2 bundle.

I concede that I'm not thrilled by how state is being passed in
changegroup.py (it feels a bit hacky). I would love to overhaul the
rather confusing set of functions in changegroup.py with something that
passes rich objects around instead of e.g. low-level generators.
However, given the code freeze for 3.9 is imminent, I'd rather not
undertake this endeavor right now. This feels like the easiest way
to get the parameter added to the changegroup part.

mercurial/bundle2.py

@@ -1294,6 +1294,9 b' def writebundle(ui, cg, filename, bundle'
         bundle.setcompression(compression)
         part = bundle.newpart('changegroup', data=cg.getchunks())
         part.addparam('version', cg.version)
+        if 'clcount' in cg.extras:
+            part.addparam('nbchanges', str(cg.extras['clcount']),
+                          mandatory=False)
         chunkiter = bundle.getchunks()
     else:
         # compression argument is only for the bundle2 case

mercurial/changegroup.py

@@ -135,7 +135,7 b' class cg1unpacker(object):'
     version = '01'
     _grouplistcount = 1 # One list of files after the manifests
 
-    def __init__(self, fh, alg):
+    def __init__(self, fh, alg, extras=None):
         if alg == 'UN':
             alg = None # get more modern without breaking too much
         if not alg in util.decompressors:
@@ -145,6 +145,7 b' class cg1unpacker(object):'
             alg = '_truncatedBZ'
         self._stream = util.decompressors[alg](fh)
         self._type = alg
+        self.extras = extras or {}
         self.callback = None
 
     # These methods (compressed, read, seek, tell) all appear to only
@@ -900,8 +901,8 b' def getbundler(version, repo, bundlecaps'
     assert version in supportedoutgoingversions(repo)
     return _packermap[version][0](repo, bundlecaps)
 
-def getunbundler(version, fh, alg):
-    return _packermap[version][1](fh, alg)
+def getunbundler(version, fh, alg, extras=None):
+    return _packermap[version][1](fh, alg, extras=extras)
 
 def _changegroupinfo(repo, nodes, source):
     if repo.ui.verbose or source == 'bundle':
@@ -929,7 +930,8 b' def getsubsetraw(repo, outgoing, bundler'
 
 def getsubset(repo, outgoing, bundler, source, fastpath=False):
     gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
-    return getunbundler(bundler.version, util.chunkbuffer(gengroup), None)
+    return getunbundler(bundler.version, util.chunkbuffer(gengroup), None,
+                        {'clcount': len(outgoing.missing)})
 
 def changegroupsubset(repo, roots, heads, source, version='01'):
     """Compute a changegroup consisting of all the nodes that are

tests/test-bundle-type.t

@@ -52,7 +52,7 b' test bundle types'
   1 changesets found
   HG20\x00\x00 (esc)
   Stream params: {}
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
   none-v2
   
@@ -61,7 +61,7 b' test bundle types'
   1 changesets found
   HG20\x00\x00 (esc)
   Stream params: sortdict([('Compression', 'BZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
   bzip2-v2
   
@@ -70,7 +70,7 b' test bundle types'
   1 changesets found
   HG20\x00\x00 (esc)
   Stream params: sortdict([('Compression', 'GZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
   gzip-v2
   
@@ -79,7 +79,7 b' test bundle types'
   1 changesets found
   HG20\x00\x00 (esc)
   Stream params: {}
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
   none-v2
   
@@ -88,7 +88,7 b' test bundle types'
   1 changesets found
   HG20\x00\x00 (esc)
   Stream params: sortdict([('Compression', 'BZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       c35a0f9217e65d1fdb90c936ffa7dbe679f83ddf
   bzip2-v2
   

tests/test-bundle.t

@@ -708,7 +708,7 b' bundle single branch'
   1a38c1b849e8b70c756d2d80b0b9a3ac0b7ea11a
   057f4db07f61970e1c11e83be79e9d08adc4dc31
   bundle2-output-bundle: "HG20", (1 params) 1 parts total
-  bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
+  bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload
   bundling: 1/2 changesets (50.00%)
   bundling: 2/2 changesets (100.00%)
   bundling: 1/2 manifests (50.00%)

tests/test-clonebundles.t

@@ -156,33 +156,34 b' changes, clone bundles produced by new M'
 by old clients.
 
   $ f --size --hexdump full.hg
-  full.hg: size=406
+  full.hg: size=418
   0000: 48 47 32 30 00 00 00 0e 43 6f 6d 70 72 65 73 73 |HG20....Compress|
-  0010: 69 6f 6e 3d 47 5a 78 9c 63 60 60 90 e5 76 f6 70 |ion=GZx.c``..v.p|
-  0020: f4 73 77 75 0f f2 0f 0d 60 00 02 46 06 76 a6 b2 |.swu....`..F.v..|
-  0030: d4 a2 e2 cc fc 3c 03 23 06 06 e6 65 40 b1 4d c1 |.....<.#...e@.M.|
-  0040: 2a 31 09 cf 9a 3a 52 04 b7 fc db f0 95 e5 a4 f4 |*1...:R.........|
-  0050: 97 17 b2 c9 0c 14 00 02 e6 d9 99 25 1a a7 a4 99 |...........%....|
-  0060: a4 a4 1a 5b 58 a4 19 27 9b a4 59 a4 1a 59 a4 99 |...[X..'..Y..Y..|
-  0070: a4 59 26 5a 18 9a 18 59 5a 26 1a 27 27 25 99 a6 |.Y&Z...YZ&.''%..|
-  0080: 99 1a 70 95 a4 16 97 70 19 28 18 70 a5 e5 e7 73 |..p....p.(.p...s|
-  0090: 71 25 a6 a4 28 00 19 40 13 0e ac fa df ab ff 7b |q%..(..@.......{|
-  00a0: 3f fb 92 dc 8b 1f 62 bb 9e b7 d7 d9 87 3d 5a 44 |?.....b......=ZD|
-  00b0: ac 2f b0 a9 c3 66 1e 54 b9 26 08 a7 1a 1b 1a a7 |./...f.T.&......|
-  00c0: 25 1b 9a 1b 99 19 9a 5a 18 9b a6 18 19 00 dd 67 |%......Z.......g|
-  00d0: 61 61 98 06 f4 80 49 4a 8a 65 52 92 41 9a 81 81 |aa....IJ.eR.A...|
-  00e0: a5 11 17 50 31 30 58 19 cc 80 98 25 29 b1 08 c4 |...P10X....%)...|
-  00f0: 37 07 79 19 88 d9 41 ee 07 8a 41 cd 5d 98 65 fb |7.y...A...A.].e.|
-  0100: e5 9e 45 bf 8d 7f 9f c6 97 9f 2b 44 34 67 d9 ec |..E.......+D4g..|
-  0110: 8e 0f a0 61 a8 eb 82 82 2e c9 c2 20 25 d5 34 c5 |...a....... %.4.|
-  0120: d0 d8 c2 dc d4 c2 d4 c4 30 d9 34 cd c0 d4 c8 cc |........0.4.....|
-  0130: 34 31 c5 d0 c4 24 31 c9 32 2d d1 c2 2c c5 30 25 |41...$1.2-..,.0%|
-  0140: 09 e4 ee 85 8f 85 ff 88 ab 89 36 c7 2a c4 47 34 |..........6.*.G4|
-  0150: fe f8 ec 7b 73 37 3f c3 24 62 1d 8d 4d 1d 9e 40 |...{s7?.$b..M..@|
-  0160: 06 3b 10 14 36 a4 38 10 04 d8 21 01 9a b1 83 f7 |.;..6.8...!.....|
-  0170: e9 45 8b d2 56 c7 a3 1f 82 52 d7 8a 78 ed fc d5 |.E..V....R..x...|
-  0180: 76 f1 36 25 81 89 c7 ad ec 90 34 48 75 2b 89 49 |v.6%......4Hu+.I|
-  0190: bf 00 d6 97 f0 8d                               |......|
+  0010: 69 6f 6e 3d 47 5a 78 9c 63 60 60 d0 e4 76 f6 70 |ion=GZx.c``..v.p|
+  0020: f4 73 77 75 0f f2 0f 0d 60 00 02 46 46 76 26 4e |.swu....`..FFv&N|
+  0030: c6 b2 d4 a2 e2 cc fc 3c 03 a3 bc a4 e4 8c c4 bc |.......<........|
+  0040: f4 d4 62 23 06 06 e6 65 40 f9 4d c1 2a 31 09 cf |..b#...e@.M.*1..|
+  0050: 9a 3a 52 04 b7 fc db f0 95 e5 a4 f4 97 17 b2 c9 |.:R.............|
+  0060: 0c 14 00 02 e6 d9 99 25 1a a7 a4 99 a4 a4 1a 5b |.......%.......[|
+  0070: 58 a4 19 27 9b a4 59 a4 1a 59 a4 99 a4 59 26 5a |X..'..Y..Y...Y&Z|
+  0080: 18 9a 18 59 5a 26 1a 27 27 25 99 a6 99 1a 70 95 |...YZ&.''%....p.|
+  0090: a4 16 97 70 19 28 18 70 a5 e5 e7 73 71 25 a6 a4 |...p.(.p...sq%..|
+  00a0: 28 00 19 40 13 0e ac fa df ab ff 7b 3f fb 92 dc |(..@.......{?...|
+  00b0: 8b 1f 62 bb 9e b7 d7 d9 87 3d 5a 44 ac 2f b0 a9 |..b......=ZD./..|
+  00c0: c3 66 1e 54 b9 26 08 a7 1a 1b 1a a7 25 1b 9a 1b |.f.T.&......%...|
+  00d0: 99 19 9a 5a 18 9b a6 18 19 00 dd 67 61 61 98 06 |...Z.......gaa..|
+  00e0: f4 80 49 4a 8a 65 52 92 41 9a 81 81 a5 11 17 50 |..IJ.eR.A......P|
+  00f0: 31 30 58 19 cc 80 98 25 29 b1 08 c4 37 07 79 19 |10X....%)...7.y.|
+  0100: 88 d9 41 ee 07 8a 41 cd 5d 98 65 fb e5 9e 45 bf |..A...A.].e...E.|
+  0110: 8d 7f 9f c6 97 9f 2b 44 34 67 d9 ec 8e 0f a0 61 |......+D4g.....a|
+  0120: a8 eb 82 82 2e c9 c2 20 25 d5 34 c5 d0 d8 c2 dc |....... %.4.....|
+  0130: d4 c2 d4 c4 30 d9 34 cd c0 d4 c8 cc 34 31 c5 d0 |....0.4.....41..|
+  0140: c4 24 31 c9 32 2d d1 c2 2c c5 30 25 09 e4 ee 85 |.$1.2-..,.0%....|
+  0150: 8f 85 ff 88 ab 89 36 c7 2a c4 47 34 fe f8 ec 7b |......6.*.G4...{|
+  0160: 73 37 3f c3 24 62 1d 8d 4d 1d 9e 40 06 3b 10 14 |s7?.$b..M..@.;..|
+  0170: 36 a4 38 10 04 d8 21 01 9a b1 83 f7 e9 45 8b d2 |6.8...!......E..|
+  0180: 56 c7 a3 1f 82 52 d7 8a 78 ed fc d5 76 f1 36 25 |V....R..x...v.6%|
+  0190: 81 89 c7 ad ec 90 34 48 75 2b 89 49 bf 00 cf 72 |......4Hu+.I...r|
+  01a0: f4 7f                                           |..|
 
   $ echo "http://localhost:$HGPORT1/full.hg" > server/.hg/clonebundles.manifest
   $ hg clone -U http://localhost:$HGPORT full-bundle

tests/test-debugbundle.t

@@ -31,7 +31,7 b' Terse output:'
 
   $ hg debugbundle bundle2.hg
   Stream params: {}
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '2')])"
       0e067c57feba1a5694ca4844f05588bb1bf82342
       991a3460af53952d10ec8a295d3d2cc2e5fa9690
 
@@ -56,7 +56,7 b' Verbose output:'
 
   $ hg debugbundle --all bundle2.hg
   Stream params: {}
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '2')])"
       format: id, p1, p2, cset, delta base, len(delta)
   
       changelog

tests/test-generaldelta.t

@@ -155,7 +155,7 b' Test that strip bundle use bundle2'
   saved backup bundle to $TESTTMP/aggressive/.hg/strip-backup/1c5d4dc9a8b8-6c68e60c-backup.hg (glob)
   $ hg debugbundle .hg/strip-backup/*
   Stream params: sortdict([('Compression', 'BZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       1c5d4dc9a8b8d6e1750966d343e94db665e7a1e9
 
   $ cd ..

tests/test-getbundle.t

@@ -170,7 +170,7 b' Get branch and merge:'
   $ hg debuggetbundle repo bundle -t bundle2
   $ hg debugbundle bundle
   Stream params: {}
-  changegroup -- "sortdict([('version', '01')])"
+  changegroup -- "sortdict([('version', '01'), ('nbchanges', '18')])"
       7704483d56b2a7b5db54dcee7c62378ac629b348
       29a4d1f17bd3f0779ca0525bebb1cfb51067c738
       713346a995c363120712aed1aee7e04afd867638

tests/test-patchbomb.t

@@ -352,14 +352,14 b' test bundle and description:'
   Content-Disposition: attachment; filename="bundle.hg"
   Content-Transfer-Encoding: base64
   
-  SEcyMAAAAA5Db21wcmVzc2lvbj1CWkJaaDkxQVkmU1lCZFwPAAAKf//7nFYSWD1/4H7R09C/I70I
-  Ak0E4peoSIYIgQCgGUQOcLABGY2hqoAAAaBMTTAAAahgTCZoAAAAAMQaqn5GmapojQ00DEGI/VGJ
-  kDAJoGTDUAAyM0QaAEqalPTUaMhoyDIDR6IxAGEGmgAehMRhDRsoyB6TYTC8JyLN+jTGqitRAgRJ
-  b3SRlhd8/+VxlAUqAilLoKPEEyxFQkaEGo+DzItFeNiFAo8NMMweVtvXJFIMhjoKC18DeYwjLKBz
-  wrMcs86qJrctDNJorwBMuLcqvTVWHh1IlsIaaaYSUIP2IZsogT1+pSSZS+bSTJrgfKsO9go/f0HF
-  uW4Yr2vXpxDreOgSIAdK/xC8Yay48SLpxIuqc/BZ6rVZCgG21rr0zhCaEgXOTqNaYEvANvg0B0Qo
-  dgtqAs1FDcZgzYitwJh6ZAG0C4mA7FPrp9b7h0h/A44Xgd+0it1gvF0mFE/CCPwymXS+OisOOCAF
-  mDUDAC1pBvsXckU4UJBCZFwP
+  SEcyMAAAAA5Db21wcmVzc2lvbj1CWkJaaDkxQVkmU1kIqE7KAAAKf//7vFYQWD1/4H7R09C/470I
+  Ak0E4peoSIYIgQCgGUQOcLABGY2hqoTTCYaBqaYAAACaaMAATIwAA1MIYDaQaqn6p+jRop+oJkA2
+  oNqD1PU0PUBoxqaMmjMUepoBoDT1GmQNBKmlTT1GTCNMEAYTQ0NNDI0BoMQHpAZAA8o2pkyNJHfX
+  RRbXoyxKRUlAg41B3lpmMOnr77dEpFKAvEUGEkWuC4wioiMjC2Y2a84EXhsNCFIrbXUGId07PJnS
+  ELAOIpL/gE8R8CUeXuw2NKMtkFoLPkcTSomXtgHSg1IKaCNlWwVU3CpmMYqh5gkFYJKOD4UhVVQ6
+  SiF1DpE8ghWvF1ih+fYgagfYHI96w/QsrRATpYiP7VRbINFrQy2c21mZ7M4pXXrPBypoXAIhtum7
+  aKDJCpUqMDF5dfiDChMfgH9nQ4B60Uvgb4AK9dsbSYc+O3tEyNq9g9gZeA5Je2T82GzjC4DbY4F2
+  0kdrTBwslErFshCgDzeEBwICg13oQaQawQA1WWd3F3JFOFCQCKhOyg==
   --===============*==-- (glob)
 
 with a specific bundle type

tests/test-rebase-conflicts.t

@@ -279,7 +279,7 b' Check that the right ancestors is used w'
   e31216eec445e44352c5f01588856059466a24c9
   2f2496ddf49d69b5ef23ad8cf9fb2e0e4faf0ac2
   bundle2-output-bundle: "HG20", (1 params) 1 parts total
-  bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
+  bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload
   saved backup bundle to $TESTTMP/issue4041/.hg/strip-backup/e31216eec445-15f7a814-backup.hg (glob)
   3 changesets found
   list of changesets:
@@ -287,10 +287,10 b' Check that the right ancestors is used w'
   19c888675e133ab5dff84516926a65672eaf04d9
   2a7f09cac94c7f4b73ebd5cd1a62d3b2e8e336bf
   bundle2-output-bundle: "HG20", 1 parts total
-  bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
+  bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload
   adding branch
   bundle2-input-bundle: with-transaction
-  bundle2-input-part: "changegroup" (params: 1 mandatory) supported
+  bundle2-input-part: "changegroup" (params: 1 mandatory 1 advisory) supported
   adding changesets
   add changeset 4c9fbe56a16f
   add changeset 19c888675e13

tests/test-shelve.t

@@ -1045,7 +1045,7 b' with general delta'
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg debugbundle .hg/shelved/*.hg
   Stream params: sortdict([('Compression', 'BZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       45993d65fe9dc3c6d8764b9c3b07fa831ee7d92d
   $ cd ..
 

tests/test-strip.t

@@ -211,7 +211,7 b''
   
   $ hg debugbundle .hg/strip-backup/*
   Stream params: sortdict([('Compression', 'BZ')])
-  changegroup -- "sortdict([('version', '02')])"
+  changegroup -- "sortdict([('version', '02'), ('nbchanges', '1')])"
       264128213d290d868c54642d13aeaa3675551a78
   $ hg pull .hg/strip-backup/*
   pulling from .hg/strip-backup/264128213d29-0b39d6bf-backup.hg
@@ -799,7 +799,7 b' check strip behavior'
   6625a516847449b6f0fa3737b9ba56e9f0f3032c
   d8db9d1372214336d2b5570f20ee468d2c72fa8b
   bundle2-output-bundle: "HG20", (1 params) 1 parts total
-  bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
+  bundle2-output-part: "changegroup" (params: 1 mandatory 1 advisory) streamed payload
   saved backup bundle to $TESTTMP/issue4736/.hg/strip-backup/6625a5168474-345bb43d-backup.hg (glob)
   invalid branchheads cache (served): tip differs
   truncating cache/rbc-revs-v1 to 24