bundle2: change header size and make them signed (new format)

We are changing all integers that denote the size of a chunk to read to int32.
There are two main motivations for that.

First, we change everything to the same width (32 bits) to make it possible for
a reasonably agnostic actor to forward a bundle2 without any extra processing.
With this change, this could be achieved by just reading int32s and forwarding
chunks of the size read. A bit a smartness would be logic to detect the end of
stream but nothing too complicated.

Second, we need some capacity to transmit special information during the bundle
processing. For example we would like to be able to raise an exception while a
part is being read if this exception happend while this part was generated.
Having signed integer let us use negative numbers to trigger special events
during the parsing of the bundle.

The format is renamed for B2X to B2Y because this breaks binary
compatibility. The B2X format support is dropped. It was experimental to
allow this kind of things. All elements not directly related to the binary
format remain flagged "b2x" because they are still compatible.

mercurial/bundle2.py

@@ -31,7 +31,7 b' stream level parameters'
 
 Binary format is as follow
 
-:params size: (16 bits integer)
+:params size: int32
 
   The total number of Bytes used by the parameters
 
@@ -64,7 +64,7 b' Payload part'
 
 Binary format is as follow
 
-:header size: (16 bits inter)
+:header size: int32
 
   The total number of Bytes used by the part headers. When the header is empty
   (size = 0) this is interpreted as the end of stream marker.
@@ -119,12 +119,15 b' Binary format is as follow'
 
     payload is a series of `<chunksize><chunkdata>`.
 
-    `chunksize` is a 32 bits integer, `chunkdata` are plain bytes (as much as
+    `chunksize` is an int32, `chunkdata` are plain bytes (as much as
     `chunksize` says)` The payload part is concluded by a zero size chunk.
 
     The current implementation always produces either zero or one chunk.
     This is an implementation limitation that will ultimately be lifted.
 
+    `chunksize` can be negative to trigger special case processing. No such
+    processing is in place yet.
+
 Bundle processing
 ============================
 
@@ -155,13 +158,13 b' from i18n import _'
 _pack = struct.pack
 _unpack = struct.unpack
 
-_magicstring = 'HG2X'
+_magicstring = 'HG2Y'
 
-_fstreamparamsize = '>H'
-_fpartheadersize = '>H'
+_fstreamparamsize = '>i'
+_fpartheadersize = '>i'
 _fparttypesize = '>B'
 _fpartid = '>I'
-_fpayloadsize = '>I'
+_fpayloadsize = '>i'
 _fpartparamcount = '>BB'
 
 preferedchunksize = 4096
@@ -496,7 +499,7 b' class unbundle20(unpackermixin):'
             magic, version = header[0:2], header[2:4]
             if magic != 'HG':
                 raise util.Abort(_('not a Mercurial bundle'))
-            if version != '2X':
+            if version != '2Y':
                 raise util.Abort(_('unknown bundle version %s') % version)
         self.ui.debug('start processing of %s stream\n' % header)
 
@@ -781,7 +784,7 b' class unbundlepart(unpackermixin):'
             self.consumed = True
         return data
 
-capabilities = {'HG2X': (),
+capabilities = {'HG2Y': (),
                 'b2x:listkeys': (),
                 'b2x:pushkey': (),
                 'b2x:changegroup': (),

mercurial/exchange.py

@@ -32,7 +32,7 b' def readbundle(ui, fh, fname, vfs=None):'
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
-    elif version == '2X':
+    elif version == '2Y':
         return bundle2.unbundle20(ui, fh, header=magic + version)
     else:
         raise util.Abort(_('%s: unknown bundle version %s') % (fname, version))
@@ -1099,7 +1099,7 b' def _pullobsolete(pullop):'
 
 def caps20to10(repo):
     """return a set with appropriate options to use bundle20 during getbundle"""
-    caps = set(['HG2X'])
+    caps = set(['HG2Y'])
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo))
     caps.add('bundle2=' + urllib.quote(capsblob))
     return caps
@@ -1132,7 +1132,7 b' def getbundle(repo, source, heads=None, '
               **kwargs):
     """return a full bundle (with potentially multiple kind of parts)
 
-    Could be a bundle HG10 or a bundle HG2X depending on bundlecaps
+    Could be a bundle HG10 or a bundle HG2Y depending on bundlecaps
     passed. For now, the bundle can contain only changegroup, but this will
     changes when more part type will be available for bundle2.
 
@@ -1144,7 +1144,7 b' def getbundle(repo, source, heads=None, '
     when the API of bundle is refined.
     """
     # bundle10 case
-    if bundlecaps is None or 'HG2X' not in bundlecaps:
+    if bundlecaps is None or 'HG2Y' not in bundlecaps:
         if bundlecaps and not kwargs.get('cg', True):
             raise ValueError(_('request for bundle10 must include changegroup'))
 

mercurial/localrepo.py

@@ -109,7 +109,7 b' class localpeer(peer.peerrepository):'
                   format='HG10', **kwargs):
         cg = exchange.getbundle(self._repo, source, heads=heads,
                                 common=common, bundlecaps=bundlecaps, **kwargs)
-        if bundlecaps is not None and 'HG2X' in bundlecaps:
+        if bundlecaps is not None and 'HG2Y' in bundlecaps:
             # When requesting a bundle2, getbundle returns a stream to make the
             # wire level function happier. We need to build a proper object
             # from it in local peer.

mercurial/wireproto.py

@@ -359,7 +359,7 b' class wirepeer(peer.peerrepository):'
             opts[key] = value
         f = self._callcompressable("getbundle", **opts)
         bundlecaps = kwargs.get('bundlecaps')
-        if bundlecaps is not None and 'HG2X' in bundlecaps:
+        if bundlecaps is not None and 'HG2Y' in bundlecaps:
             return bundle2.unbundle20(self.ui, f)
         else:
             return changegroupmod.cg1unpacker(f, 'UN')

tests/test-bundle2-format.t

@@ -234,7 +234,7 b' Empty bundle'
 Test bundling
 
   $ hg bundle2
-  HG2X\x00\x00\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 Test unbundling
 
@@ -264,7 +264,7 b' Simplest possible parameters form'
 Test generation simple option
 
   $ hg bundle2 --param 'caution'
-  HG2X\x00\x07caution\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00\x07caution\x00\x00\x00\x00 (no-eol) (esc)
 
 Test unbundling
 
@@ -276,7 +276,7 b' Test unbundling'
 Test generation multiple option
 
   $ hg bundle2 --param 'caution' --param 'meal'
-  HG2X\x00\x0ccaution meal\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00\x0ccaution meal\x00\x00\x00\x00 (no-eol) (esc)
 
 Test unbundling
 
@@ -292,7 +292,7 b' advisory parameters, with value'
 Test generation
 
   $ hg bundle2 --param 'caution' --param 'meal=vegan' --param 'elephants'
-  HG2X\x00\x1ccaution meal=vegan elephants\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00\x1ccaution meal=vegan elephants\x00\x00\x00\x00 (no-eol) (esc)
 
 Test unbundling
 
@@ -310,7 +310,7 b' parameter with special char in value'
 Test generation
 
   $ hg bundle2 --param 'e|! 7/=babar%#==tutu' --param simple
-  HG2X\x00)e%7C%21%207/=babar%25%23%3D%3Dtutu simple\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00)e%7C%21%207/=babar%25%23%3D%3Dtutu simple\x00\x00\x00\x00 (no-eol) (esc)
 
 Test unbundling
 
@@ -334,7 +334,7 b' Test debug output'
 bundling debug
 
   $ hg bundle2 --debug --param 'e|! 7/=babar%#==tutu' --param simple ../out.hg2
-  start emission of HG2X stream
+  start emission of HG2Y stream
   bundle parameter: e%7C%21%207/=babar%25%23%3D%3Dtutu simple
   start of parts
   end of bundle
@@ -342,12 +342,12 b' bundling debug'
 file content is ok
 
   $ cat ../out.hg2
-  HG2X\x00)e%7C%21%207/=babar%25%23%3D%3Dtutu simple\x00\x00 (no-eol) (esc)
+  HG2Y\x00\x00\x00)e%7C%21%207/=babar%25%23%3D%3Dtutu simple\x00\x00\x00\x00 (no-eol) (esc)
 
 unbundling debug
 
   $ hg statbundle2 --debug < ../out.hg2
-  start processing of HG2X stream
+  start processing of HG2Y stream
   reading bundle2 stream parameters
   ignoring unknown parameter 'e|! 7/'
   ignoring unknown parameter 'simple'
@@ -381,7 +381,7 b' Test part'
 =================
 
   $ hg bundle2 --parts ../parts.hg2 --debug
-  start emission of HG2X stream
+  start emission of HG2Y stream
   bundle parameter: 
   start of parts
   bundle part: "test:empty"
@@ -394,11 +394,11 b' Test part'
   end of bundle
 
   $ cat ../parts.hg2
-  HG2X\x00\x00\x00\x11 (esc)
-  test:empty\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11 (esc)
-  test:empty\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x10	test:song\x00\x00\x00\x02\x00\x00\x00\x00\x00\xb2Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x00\x11 (esc)
+  test:empty\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11 (esc)
+  test:empty\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10	test:song\x00\x00\x00\x02\x00\x00\x00\x00\x00\xb2Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko (esc)
   Bokoro Dipoulito, Rondi Rondi Pepino, Pata Pata, Ko Ko Ko
-  Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko.\x00\x00\x00\x00\x00\x16\x0ftest:debugreply\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00+	test:math\x00\x00\x00\x04\x02\x01\x02\x04\x01\x04\x07\x03pi3.14e2.72cookingraw\x00\x00\x00\x0242\x00\x00\x00\x00\x00\x1d	test:song\x00\x00\x00\x05\x01\x00\x0b\x00randomparam\x00\x00\x00\x00\x00\x10	test:ping\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
+  Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko.\x00\x00\x00\x00\x00\x00\x00\x16\x0ftest:debugreply\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00+	test:math\x00\x00\x00\x04\x02\x01\x02\x04\x01\x04\x07\x03pi3.14e2.72cookingraw\x00\x00\x00\x0242\x00\x00\x00\x00\x00\x00\x00\x1d	test:song\x00\x00\x00\x05\x01\x00\x0b\x00randomparam\x00\x00\x00\x00\x00\x00\x00\x10	test:ping\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 
   $ hg statbundle2 < ../parts.hg2
@@ -434,7 +434,7 b' Test part'
   parts count:   7
 
   $ hg statbundle2 --debug < ../parts.hg2
-  start processing of HG2X stream
+  start processing of HG2Y stream
   reading bundle2 stream parameters
   options count: 0
   start extraction of bundle2 parts
@@ -513,7 +513,7 b' Test actual unbundling of test part'
 Process the bundle
 
   $ hg unbundle2 --debug < ../parts.hg2
-  start processing of HG2X stream
+  start processing of HG2Y stream
   reading bundle2 stream parameters
   start extraction of bundle2 parts
   part header size: 17
@@ -607,12 +607,12 b' unbundle with a reply'
 The reply is a bundle
 
   $ cat ../reply.hg2
-  HG2X\x00\x00\x00\x1f (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x00\x1f (esc)
   b2x:output\x00\x00\x00\x00\x00\x01\x0b\x01in-reply-to3\x00\x00\x00\xd9The choir starts singing: (esc)
       Patali Dirapata, Cromda Cromda Ripalo, Pata Pata, Ko Ko Ko
       Bokoro Dipoulito, Rondi Rondi Pepino, Pata Pata, Ko Ko Ko
       Emana Karassoli, Loucra Loucra Ponponto, Pata Pata, Ko Ko Ko.
-  \x00\x00\x00\x00\x00\x1f (esc)
+  \x00\x00\x00\x00\x00\x00\x00\x1f (esc)
   b2x:output\x00\x00\x00\x01\x00\x01\x0b\x01in-reply-to4\x00\x00\x00\xc9debugreply: capabilities: (esc)
   debugreply:     'city=!'
   debugreply:         'celeste,ville'
@@ -620,10 +620,10 b' The reply is a bundle'
   debugreply:         'babar'
   debugreply:         'celeste'
   debugreply:     'ping-pong'
-  \x00\x00\x00\x00\x00\x1e	test:pong\x00\x00\x00\x02\x01\x00\x0b\x01in-reply-to7\x00\x00\x00\x00\x00\x1f (esc)
+  \x00\x00\x00\x00\x00\x00\x00\x1e	test:pong\x00\x00\x00\x02\x01\x00\x0b\x01in-reply-to7\x00\x00\x00\x00\x00\x00\x00\x1f (esc)
   b2x:output\x00\x00\x00\x03\x00\x01\x0b\x01in-reply-to7\x00\x00\x00=received ping request (id 7) (esc)
   replying to ping request (id 7)
-  \x00\x00\x00\x00\x00\x00 (no-eol) (esc)
+  \x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 The reply is valid
 
@@ -711,7 +711,7 b' Support for changegroup'
   9520eea781bcca16c1e15acc0ba14335a0e8e5ba
   eea13746799a9e0bfd88f29d3c2e9dc9389f524f
   02de42196ebee42ef284b6780a87cdc96e8eaab6
-  start emission of HG2X stream
+  start emission of HG2Y stream
   bundle parameter: 
   start of parts
   bundle part: "b2x:changegroup"
@@ -729,7 +729,7 b' Support for changegroup'
   end of bundle
 
   $ cat ../rev.hg2
-  HG2X\x00\x00\x00\x16\x0fb2x:changegroup\x00\x00\x00\x00\x00\x00\x00\x00\x06\x13\x00\x00\x00\xa42\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j_\xdd\xd9\x89W\xc8\xa5JMCm\xfe\x1d\xa9\xd8\x7f!\xa1\xb9{\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00)6e1f4c47ecb533ffd0c8e52cdc88afb6cd39e20c (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x00\x16\x0fb2x:changegroup\x00\x00\x00\x00\x00\x00\x00\x00\x06\x13\x00\x00\x00\xa42\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j_\xdd\xd9\x89W\xc8\xa5JMCm\xfe\x1d\xa9\xd8\x7f!\xa1\xb9{\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\xafv\x86\xd4\x03\xcfE\xb5\xd9_-p\xce\xbe\xa5\x87\xac\x80j\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00)6e1f4c47ecb533ffd0c8e52cdc88afb6cd39e20c (esc)
   \x00\x00\x00f\x00\x00\x00h\x00\x00\x00\x02D (esc)
   \x00\x00\x00i\x00\x00\x00j\x00\x00\x00\x01D\x00\x00\x00\xa4\x95 \xee\xa7\x81\xbc\xca\x16\xc1\xe1Z\xcc\x0b\xa1C5\xa0\xe8\xe5\xba\xcd\x01\x0b\x8c\xd9\x98\xf3\x98\x1aZ\x81\x15\xf9O\x8d\xa4\xabP`\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x95 \xee\xa7\x81\xbc\xca\x16\xc1\xe1Z\xcc\x0b\xa1C5\xa0\xe8\xe5\xba\x00\x00\x00\x00\x00\x00\x00)\x00\x00\x00)4dece9c826f69490507b98c6383a3009b295837d (esc)
   \x00\x00\x00f\x00\x00\x00h\x00\x00\x00\x02E (esc)
@@ -750,7 +750,7 b' Support for changegroup'
   \x0cI\xd4\xa9\xc5\x01|\xf0pC\xf5NX\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x95 \xee\xa7\x81\xbc\xca\x16\xc1\xe1Z\xcc\x0b\xa1C5\xa0\xe8\xe5\xba\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02E (esc)
   \x00\x00\x00\x00\x00\x00\x00\x05H\x00\x00\x00b\x85\x00\x18\x9et\xa9\xe0G^\x82 \x93\xbc}\xb0\xd61\xae\xb0\xb4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\xdeB\x19n\xbe\xe4.\xf2\x84\xb6x (esc)
   \x87\xcd\xc9n\x8e\xaa\xb6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02H (esc)
-  \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
+  \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
   $ hg unbundle2 < ../rev.hg2
   adding changesets
@@ -768,12 +768,12 b' with reply'
   addchangegroup return: 1
 
   $ cat ../rev-reply.hg2
-  HG2X\x00\x00\x003\x15b2x:reply:changegroup\x00\x00\x00\x00\x00\x02\x0b\x01\x06\x01in-reply-to1return1\x00\x00\x00\x00\x00\x1f (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x003\x15b2x:reply:changegroup\x00\x00\x00\x00\x00\x02\x0b\x01\x06\x01in-reply-to1return1\x00\x00\x00\x00\x00\x00\x00\x1f (esc)
   b2x:output\x00\x00\x00\x01\x00\x01\x0b\x01in-reply-to1\x00\x00\x00dadding changesets (esc)
   adding manifests
   adding file changes
   added 0 changesets with 0 changes to 3 files
-  \x00\x00\x00\x00\x00\x00 (no-eol) (esc)
+  \x00\x00\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 Check handling of exception during generation.
 ----------------------------------------------
@@ -787,7 +787,7 b' Should still be a valid bundle'
 (is currently not right)
 
   $ cat ../genfailed.hg2
-  HG2X\x00\x00\x00\x11 (esc)
+  HG2Y\x00\x00\x00\x00\x00\x00\x00\x11 (esc)
   b2x:output\x00\x00\x00\x00\x00\x00 (no-eol) (esc)
 
 And its handling on the other size raise a clean exception
@@ -795,7 +795,7 b' And its handling on the other size raise'
 
   $ cat ../genfailed.hg2 | hg unbundle2
   0 unread bytes
-  abort: stream ended unexpectedly (got 0 bytes, expected 2)
+  abort: stream ended unexpectedly (got 0 bytes, expected 4)
   [255]