stream-clone: introduce the notion of an experimental "v3" version

We introduce a new experimental "v3" stream protocol, disabled by default. In
practice the "v3-exp" protocol introduced in this changeset is identical to v2,
but this changeset, lay the groundwork for having a new protocol:
configuration, capability exchange, test coverage, etc.

The actual protocol work will starts in the coming changesets.

hgext/remotefilelog/__init__.py

@@ -408,9 +408,7 b' def cloneshallow(orig, ui, repo, *args, '
             # bundle2 flavor of streamclones, so force us to use
             # v1 instead.
             if b'v2' in pullop.remotebundle2caps.get(b'stream', []):
-                pullop.remotebundle2caps[b'stream'] = [
-                    c for c in pullop.remotebundle2caps[b'stream'] if c != b'v2'
-                ]
+                pullop.remotebundle2caps[b'stream'] = []
             if bundle2:
                 return False, None
             supported, requirements = orig(pullop, bundle2=bundle2)

mercurial/bundle2.py

@@ -1671,6 +1671,10 b' def getrepocaps(repo, allowpushback=Fals'
     # Else always advertise support on client, because payload support
     # should always be advertised.
 
+    if repo.ui.configbool(b'experimental', b'stream-v3'):
+        if b'stream' in caps:
+            caps[b'stream'] += (b'v3-exp',)
+
     # b'rev-branch-cache is no longer advertised, but still supported
     # for legacy clients.
 
@@ -1892,8 +1896,16 b' def addpartbundlestream2(bundler, repo, '
         )
         hint = _(b'the client seems buggy')
         raise error.Abort(msg, hint=hint)
-    if not (b'v2' in bundler.capabilities[b'stream']):
-        raise error.Abort(_(b'the client does not support streamclone v2'))
+    client_supported = set(bundler.capabilities[b'stream'])
+    server_supported = set(getrepocaps(repo, role=b'client').get(b'stream', []))
+    common_supported = client_supported & server_supported
+    if not common_supported:
+        msg = _(b'no common supported version with the client: %s; %s')
+        str_server = b','.join(sorted(server_supported))
+        str_client = b','.join(sorted(client_supported))
+        msg %= (str_server, str_client)
+        raise error.Abort(msg)
+    version = max(common_supported)
 
     # Stream clones don't compress well. And compression undermines a
     # goal of stream clones, which is to be fast. Communicate the desire
@@ -1924,6 +1936,7 b' def addpartbundlestream2(bundler, repo, '
         elif repo.obsstore._version in remoteversions:
             includeobsmarkers = True
 
+    if version == b"v2":
     filecount, bytecount, it = streamclone.generatev2(
         repo, includepats, excludepats, includeobsmarkers
     )
@@ -1933,6 +1946,16 b' def addpartbundlestream2(bundler, repo, '
     part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
     part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
     part.addparam(b'requirements', requirements, mandatory=True)
+    elif version == b"v3-exp":
+        filecount, bytecount, it = streamclone.generatev2(
+            repo, includepats, excludepats, includeobsmarkers
+        )
+        requirements = streamclone.streamed_requirements(repo)
+        requirements = _formatrequirementsspec(requirements)
+        part = bundler.newpart(b'stream3', data=it)
+        part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
+        part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
+        part.addparam(b'requirements', requirements, mandatory=True)
 
 
 def buildobsmarkerspart(bundler, markers, mandatory=True):
@@ -2588,6 +2611,11 b' def handlestreamv2bundle(op, part):'
     streamclone.applybundlev2(repo, part, filecount, bytecount, requirements)
 
 
+@parthandler(b'stream3', (b'requirements', b'filecount', b'bytecount'))
+def handlestreamv3bundle(op, part):
+    return handlestreamv2bundle(op, part)
+
+
 def widen_bundle(
     bundler, repo, oldmatcher, newmatcher, common, known, cgversion, ellipses
 ):

mercurial/configitems.py

@@ -1298,6 +1298,11 b' coreconfigitem('
 )
 coreconfigitem(
     b'experimental',
+    b'stream-v3',
+    default=False,
+)
+coreconfigitem(
+    b'experimental',
     b'treemanifest',
     default=False,
 )

mercurial/exchange.py

@@ -2421,7 +2421,7 b' def getbundlechunks('
     return info, bundler.getchunks()
 
 
-@getbundle2partsgenerator(b'stream2')
+@getbundle2partsgenerator(b'stream')
 def _getbundlestream2(bundler, repo, *args, **kwargs):
     return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
 

mercurial/streamclone.py

@@ -758,6 +758,10 b' def generatev2(repo, includes, excludes,'
     return len(entries), totalfilesize, chunks
 
 
+def generatev3(repo, includes, excludes, includeobsmarkers):
+    return generatev2(repo, includes, excludes, includeobsmarkers)
+
+
 @contextlib.contextmanager
 def nested(*ctxs):
     this = ctxs[0]

tests/test-clone-stream.t

@@ -1,6 +1,6 b''
 #require serve no-reposimplestore no-chg
 
-#testcases stream-legacy stream-bundle2
+#testcases stream-legacy stream-bundle2-v2 stream-bundle2-v3
 
 #if stream-legacy
   $ cat << EOF >> $HGRCPATH
@@ -8,6 +8,12 b''
   > bundle2.stream = no
   > EOF
 #endif
+#if stream-bundle2-v3
+  $ cat << EOF >> $HGRCPATH
+  > [experimental]
+  > stream-v3 = yes
+  > EOF
+#endif
 
 Initialize repository
 
@@ -174,7 +180,75 b' Cannot stream clone when server.uncompre'
   0060: 69 73 20 66                                     |is f|
 
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg debugcapabilities http://localhost:$HGPORT
+  Main capabilities:
+    batch
+    branchmap
+    $USUAL_BUNDLE2_CAPS_SERVER$
+    changegroupsubset
+    compression=$BUNDLE2_COMPRESSIONS$
+    getbundle
+    httpheader=1024
+    httpmediatype=0.1rx,0.1tx,0.2tx
+    known
+    lookup
+    pushkey
+    unbundle=HG10GZ,HG10BZ,HG10UN
+    unbundlehash
+  Bundle2 capabilities:
+    HG20
+    bookmarks
+    changegroup
+      01
+      02
+      03
+    checkheads
+      related
+    digests
+      md5
+      sha1
+      sha512
+    error
+      abort
+      unsupportedcontent
+      pushraced
+      pushkey
+    hgtagsfnodes
+    listkeys
+    phases
+      heads
+    pushkey
+    remote-changegroup
+      http
+      https
+
+  $ hg clone --stream -U http://localhost:$HGPORT server-disabled
+  warning: stream clone requested but server has them disabled
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 1088 changes to 1088 files
+  new changesets 96ee1d7354c4:5223b5e3265f
+
+  $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=getbundle' content-type --bodyfile body --hgproto 0.2 --requestheader "x-hgarg-1=bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=0000000000000000000000000000000000000000&heads=c17445101a72edac06facd130d14808dfbd5c7c2&stream=1"
+  200 Script output follows
+  content-type: application/mercurial-0.2
+  
+
+  $ f --size body --hexdump --bytes 100
+  body: size=140
+  0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......|
+  0010: 73 0b 45 52 52 4f 52 3a 41 42 4f 52 54 00 00 00 |s.ERROR:ABORT...|
+  0020: 00 01 01 07 3c 04 16 6d 65 73 73 61 67 65 73 74 |....<..messagest|
+  0030: 72 65 61 6d 20 64 61 74 61 20 72 65 71 75 65 73 |ream data reques|
+  0040: 74 65 64 20 62 75 74 20 73 65 72 76 65 72 20 64 |ted but server d|
+  0050: 6f 65 73 20 6e 6f 74 20 61 6c 6c 6f 77 20 74 68 |oes not allow th|
+  0060: 69 73 20 66                                     |is f|
+
+#endif
+#if stream-bundle2-v3
   $ hg debugcapabilities http://localhost:$HGPORT
   Main capabilities:
     batch
@@ -262,7 +336,28 b' Basic clone'
   no changes found
   $ cat server/errors.txt
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --stream -U http://localhost:$HGPORT clone1
+  streaming all changes
+  1093 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1093 files to transfer, 98.9 KB of data (zstd !)
+  transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+
+  $ ls -1 clone1/.hg/cache
+  branch2-base
+  branch2-immutable
+  branch2-served
+  branch2-served.hidden
+  branch2-visible
+  branch2-visible-hidden
+  rbc-names-v1
+  rbc-revs-v1
+  tags2
+  tags2-served
+  $ cat server/errors.txt
+#endif
+#if stream-bundle2-v3
   $ hg clone --stream -U http://localhost:$HGPORT clone1
   streaming all changes
   1093 files to transfer, 102 KB of data (no-zstd !)
@@ -386,7 +481,15 b' getbundle requests with stream=1 are unc'
   searching for changes
   no changes found
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
+  streaming all changes
+  1093 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1093 files to transfer, 98.9 KB of data (zstd !)
+  transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+#endif
+#if stream-bundle2-v3
   $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed
   streaming all changes
   1093 files to transfer, 102 KB of data (no-zstd !)
@@ -425,7 +528,7 b' Clone with background file closing enabl'
   updating the branch cache
   (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
   $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
   using http://localhost:$HGPORT/
   sending capabilities command
@@ -452,6 +555,33 b' Clone with background file closing enabl'
   updating the branch cache
   (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob)
 #endif
+#if stream-bundle2-v3
+  $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding
+  using http://localhost:$HGPORT/
+  sending capabilities command
+  query 1; heads
+  sending batch command
+  streaming all changes
+  sending getbundle command
+  bundle2-input-bundle: with-transaction
+  bundle2-input-part: "stream3" (params: 3 mandatory) supported
+  applying stream bundle
+  1093 files to transfer, 102 KB of data (no-zstd !)
+  1093 files to transfer, 98.9 KB of data (zstd !)
+  starting 4 threads for background file closing
+  starting 4 threads for background file closing
+  updating the branch cache
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  bundle2-input-part: total payload size 118984 (no-zstd !)
+  transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+  bundle2-input-part: total payload size 116145 (zstd no-bigendian !)
+  bundle2-input-part: total payload size 116140 (zstd bigendian !)
+  bundle2-input-part: "listkeys" (params: 1 mandatory) supported
+  bundle2-input-bundle: 2 parts total
+  checking for updated bookmarks
+  updating the branch cache
+  (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob)
+#endif
 
 Cannot stream clone when there are secret changesets
 
@@ -484,7 +614,15 b' Streaming of secrets can be overridden b'
   searching for changes
   no changes found
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --stream -U http://localhost:$HGPORT secret-allowed
+  streaming all changes
+  1093 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1093 files to transfer, 98.9 KB of data (zstd !)
+  transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !)
+#endif
+#if stream-bundle2-v3
   $ hg clone --stream -U http://localhost:$HGPORT secret-allowed
   streaming all changes
   1093 files to transfer, 102 KB of data (no-zstd !)
@@ -638,7 +776,17 b' clone it'
   updating to branch default
   1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --stream http://localhost:$HGPORT with-bookmarks
+  streaming all changes
+  1096 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1096 files to transfer, 99.1 KB of data (zstd !)
+  transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+  updating to branch default
+  1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
+#if stream-bundle2-v3
   $ hg clone --stream http://localhost:$HGPORT with-bookmarks
   streaming all changes
   1096 files to transfer, 102 KB of data (no-zstd !)
@@ -674,7 +822,17 b' Clone as publishing'
   updating to branch default
   1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --stream http://localhost:$HGPORT phase-publish
+  streaming all changes
+  1096 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1096 files to transfer, 99.1 KB of data (zstd !)
+  transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+  updating to branch default
+  1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+#endif
+#if stream-bundle2-v3
   $ hg clone --stream http://localhost:$HGPORT phase-publish
   streaming all changes
   1096 files to transfer, 102 KB of data (no-zstd !)
@@ -720,7 +878,21 b' stream v1 unsuitable for non-publishing '
   1: public
   2: public
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+  $ hg clone --stream http://localhost:$HGPORT phase-no-publish
+  streaming all changes
+  1097 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1097 files to transfer, 99.1 KB of data (zstd !)
+  transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !)
+  updating to branch default
+  1088 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg -R phase-no-publish phase -r 'all()'
+  0: draft
+  1: draft
+  2: draft
+#endif
+#if stream-bundle2-v3
   $ hg clone --stream http://localhost:$HGPORT phase-no-publish
   streaming all changes
   1097 files to transfer, 102 KB of data (no-zstd !)
@@ -744,7 +916,57 b' With v1 of the stream protocol, changese'
 no obsolescence markers exchange in stream v1.
 
 #endif
-#if stream-bundle2
+#if stream-bundle2-v2
+
+Stream repository with obsolescence
+-----------------------------------
+
+Clone non-publishing with obsolescence
+
+  $ cat >> $HGRCPATH << EOF
+  > [experimental]
+  > evolution=all
+  > EOF
+
+  $ cd server
+  $ echo foo > foo
+  $ hg -q commit -m 'about to be pruned'
+  $ hg debugobsolete `hg log -r . -T '{node}'` -d '0 0' -u test --record-parents
+  1 new obsolescence markers
+  obsoleted 1 changesets
+  $ hg up null -q
+  $ hg log -T '{rev}: {phase}\n'
+  2: draft
+  1: draft
+  0: draft
+  $ hg serve -p $HGPORT -d --pid-file=hg.pid
+  $ cat hg.pid > $DAEMON_PIDS
+  $ cd ..
+
+  $ hg clone -U --stream http://localhost:$HGPORT with-obsolescence
+  streaming all changes
+  1098 files to transfer, 102 KB of data (no-zstd !)
+  transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !)
+  1098 files to transfer, 99.5 KB of data (zstd !)
+  transferred 99.5 KB in * seconds (* */sec) (glob) (zstd !)
+  $ hg -R with-obsolescence log -T '{rev}: {phase}\n'
+  2: draft
+  1: draft
+  0: draft
+  $ hg debugobsolete -R with-obsolescence
+  8c206a663911c1f97f2f9d7382e417ae55872cfa 0 {5223b5e3265f0df40bb743da62249413d74ac70f} (Thu Jan 01 00:00:00 1970 +0000) {'user': 'test'}
+  $ hg verify -R with-obsolescence -q
+
+  $ hg clone -U --stream --config experimental.evolution=0 http://localhost:$HGPORT with-obsolescence-no-evolution
+  streaming all changes
+  remote: abort: server has obsolescence markers, but client cannot receive them via stream clone
+  abort: pull failed on remote
+  [100]
+
+  $ killdaemons.py
+
+#endif
+#if stream-bundle2-v3
 
 Stream repository with obsolescence
 -----------------------------------