diff --git a/hgext/remotefilelog/__init__.py b/hgext/remotefilelog/__init__.py --- a/hgext/remotefilelog/__init__.py +++ b/hgext/remotefilelog/__init__.py @@ -408,9 +408,7 @@ def cloneshallow(orig, ui, repo, *args, # bundle2 flavor of streamclones, so force us to use # v1 instead. if b'v2' in pullop.remotebundle2caps.get(b'stream', []): - pullop.remotebundle2caps[b'stream'] = [ - c for c in pullop.remotebundle2caps[b'stream'] if c != b'v2' - ] + pullop.remotebundle2caps[b'stream'] = [] if bundle2: return False, None supported, requirements = orig(pullop, bundle2=bundle2) diff --git a/mercurial/bundle2.py b/mercurial/bundle2.py --- a/mercurial/bundle2.py +++ b/mercurial/bundle2.py @@ -1671,6 +1671,10 @@ def getrepocaps(repo, allowpushback=Fals # Else always advertise support on client, because payload support # should always be advertised. + if repo.ui.configbool(b'experimental', b'stream-v3'): + if b'stream' in caps: + caps[b'stream'] += (b'v3-exp',) + # b'rev-branch-cache is no longer advertised, but still supported # for legacy clients. @@ -1892,8 +1896,16 @@ def addpartbundlestream2(bundler, repo, ) hint = _(b'the client seems buggy') raise error.Abort(msg, hint=hint) - if not (b'v2' in bundler.capabilities[b'stream']): - raise error.Abort(_(b'the client does not support streamclone v2')) + client_supported = set(bundler.capabilities[b'stream']) + server_supported = set(getrepocaps(repo, role=b'client').get(b'stream', [])) + common_supported = client_supported & server_supported + if not common_supported: + msg = _(b'no common supported version with the client: %s; %s') + str_server = b','.join(sorted(server_supported)) + str_client = b','.join(sorted(client_supported)) + msg %= (str_server, str_client) + raise error.Abort(msg) + version = max(common_supported) # Stream clones don't compress well. And compression undermines a # goal of stream clones, which is to be fast. Communicate the desire @@ -1924,15 +1936,26 @@ def addpartbundlestream2(bundler, repo, elif repo.obsstore._version in remoteversions: includeobsmarkers = True - filecount, bytecount, it = streamclone.generatev2( - repo, includepats, excludepats, includeobsmarkers - ) - requirements = streamclone.streamed_requirements(repo) - requirements = _formatrequirementsspec(requirements) - part = bundler.newpart(b'stream2', data=it) - part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True) - part.addparam(b'filecount', b'%d' % filecount, mandatory=True) - part.addparam(b'requirements', requirements, mandatory=True) + if version == b"v2": + filecount, bytecount, it = streamclone.generatev2( + repo, includepats, excludepats, includeobsmarkers + ) + requirements = streamclone.streamed_requirements(repo) + requirements = _formatrequirementsspec(requirements) + part = bundler.newpart(b'stream2', data=it) + part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True) + part.addparam(b'filecount', b'%d' % filecount, mandatory=True) + part.addparam(b'requirements', requirements, mandatory=True) + elif version == b"v3-exp": + filecount, bytecount, it = streamclone.generatev2( + repo, includepats, excludepats, includeobsmarkers + ) + requirements = streamclone.streamed_requirements(repo) + requirements = _formatrequirementsspec(requirements) + part = bundler.newpart(b'stream3', data=it) + part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True) + part.addparam(b'filecount', b'%d' % filecount, mandatory=True) + part.addparam(b'requirements', requirements, mandatory=True) def buildobsmarkerspart(bundler, markers, mandatory=True): @@ -2588,6 +2611,11 @@ def handlestreamv2bundle(op, part): streamclone.applybundlev2(repo, part, filecount, bytecount, requirements) +@parthandler(b'stream3', (b'requirements', b'filecount', b'bytecount')) +def handlestreamv3bundle(op, part): + return handlestreamv2bundle(op, part) + + def widen_bundle( bundler, repo, oldmatcher, newmatcher, common, known, cgversion, ellipses ): diff --git a/mercurial/configitems.py b/mercurial/configitems.py --- a/mercurial/configitems.py +++ b/mercurial/configitems.py @@ -1298,6 +1298,11 @@ coreconfigitem( ) coreconfigitem( b'experimental', + b'stream-v3', + default=False, +) +coreconfigitem( + b'experimental', b'treemanifest', default=False, ) diff --git a/mercurial/exchange.py b/mercurial/exchange.py --- a/mercurial/exchange.py +++ b/mercurial/exchange.py @@ -2421,7 +2421,7 @@ def getbundlechunks( return info, bundler.getchunks() -@getbundle2partsgenerator(b'stream2') +@getbundle2partsgenerator(b'stream') def _getbundlestream2(bundler, repo, *args, **kwargs): return bundle2.addpartbundlestream2(bundler, repo, **kwargs) diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py --- a/mercurial/streamclone.py +++ b/mercurial/streamclone.py @@ -758,6 +758,10 @@ def generatev2(repo, includes, excludes, return len(entries), totalfilesize, chunks +def generatev3(repo, includes, excludes, includeobsmarkers): + return generatev2(repo, includes, excludes, includeobsmarkers) + + @contextlib.contextmanager def nested(*ctxs): this = ctxs[0] diff --git a/tests/test-clone-stream.t b/tests/test-clone-stream.t --- a/tests/test-clone-stream.t +++ b/tests/test-clone-stream.t @@ -1,6 +1,6 @@ #require serve no-reposimplestore no-chg -#testcases stream-legacy stream-bundle2 +#testcases stream-legacy stream-bundle2-v2 stream-bundle2-v3 #if stream-legacy $ cat << EOF >> $HGRCPATH @@ -8,6 +8,12 @@ > bundle2.stream = no > EOF #endif +#if stream-bundle2-v3 + $ cat << EOF >> $HGRCPATH + > [experimental] + > stream-v3 = yes + > EOF +#endif Initialize repository @@ -174,7 +180,75 @@ Cannot stream clone when server.uncompre 0060: 69 73 20 66 |is f| #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg debugcapabilities http://localhost:$HGPORT + Main capabilities: + batch + branchmap + $USUAL_BUNDLE2_CAPS_SERVER$ + changegroupsubset + compression=$BUNDLE2_COMPRESSIONS$ + getbundle + httpheader=1024 + httpmediatype=0.1rx,0.1tx,0.2tx + known + lookup + pushkey + unbundle=HG10GZ,HG10BZ,HG10UN + unbundlehash + Bundle2 capabilities: + HG20 + bookmarks + changegroup + 01 + 02 + 03 + checkheads + related + digests + md5 + sha1 + sha512 + error + abort + unsupportedcontent + pushraced + pushkey + hgtagsfnodes + listkeys + phases + heads + pushkey + remote-changegroup + http + https + + $ hg clone --stream -U http://localhost:$HGPORT server-disabled + warning: stream clone requested but server has them disabled + requesting all changes + adding changesets + adding manifests + adding file changes + added 3 changesets with 1088 changes to 1088 files + new changesets 96ee1d7354c4:5223b5e3265f + + $ get-with-headers.py $LOCALIP:$HGPORT '?cmd=getbundle' content-type --bodyfile body --hgproto 0.2 --requestheader "x-hgarg-1=bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=0000000000000000000000000000000000000000&heads=c17445101a72edac06facd130d14808dfbd5c7c2&stream=1" + 200 Script output follows + content-type: application/mercurial-0.2 + + + $ f --size body --hexdump --bytes 100 + body: size=140 + 0000: 04 6e 6f 6e 65 48 47 32 30 00 00 00 00 00 00 00 |.noneHG20.......| + 0010: 73 0b 45 52 52 4f 52 3a 41 42 4f 52 54 00 00 00 |s.ERROR:ABORT...| + 0020: 00 01 01 07 3c 04 16 6d 65 73 73 61 67 65 73 74 |....<..messagest| + 0030: 72 65 61 6d 20 64 61 74 61 20 72 65 71 75 65 73 |ream data reques| + 0040: 74 65 64 20 62 75 74 20 73 65 72 76 65 72 20 64 |ted but server d| + 0050: 6f 65 73 20 6e 6f 74 20 61 6c 6c 6f 77 20 74 68 |oes not allow th| + 0060: 69 73 20 66 |is f| + +#endif +#if stream-bundle2-v3 $ hg debugcapabilities http://localhost:$HGPORT Main capabilities: batch @@ -262,7 +336,28 @@ Basic clone no changes found $ cat server/errors.txt #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --stream -U http://localhost:$HGPORT clone1 + streaming all changes + 1093 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1093 files to transfer, 98.9 KB of data (zstd !) + transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !) + + $ ls -1 clone1/.hg/cache + branch2-base + branch2-immutable + branch2-served + branch2-served.hidden + branch2-visible + branch2-visible-hidden + rbc-names-v1 + rbc-revs-v1 + tags2 + tags2-served + $ cat server/errors.txt +#endif +#if stream-bundle2-v3 $ hg clone --stream -U http://localhost:$HGPORT clone1 streaming all changes 1093 files to transfer, 102 KB of data (no-zstd !) @@ -386,7 +481,15 @@ getbundle requests with stream=1 are unc searching for changes no changes found #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed + streaming all changes + 1093 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1093 files to transfer, 98.9 KB of data (zstd !) + transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !) +#endif +#if stream-bundle2-v3 $ hg clone --uncompressed -U http://localhost:$HGPORT clone1-uncompressed streaming all changes 1093 files to transfer, 102 KB of data (no-zstd !) @@ -425,7 +528,7 @@ Clone with background file closing enabl updating the branch cache (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob) #endif -#if stream-bundle2 +#if stream-bundle2-v2 $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding using http://localhost:$HGPORT/ sending capabilities command @@ -452,6 +555,33 @@ Clone with background file closing enabl updating the branch cache (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) #endif +#if stream-bundle2-v3 + $ hg --debug --config worker.backgroundclose=true --config worker.backgroundcloseminfilecount=1 clone --stream -U http://localhost:$HGPORT clone-background | grep -v adding + using http://localhost:$HGPORT/ + sending capabilities command + query 1; heads + sending batch command + streaming all changes + sending getbundle command + bundle2-input-bundle: with-transaction + bundle2-input-part: "stream3" (params: 3 mandatory) supported + applying stream bundle + 1093 files to transfer, 102 KB of data (no-zstd !) + 1093 files to transfer, 98.9 KB of data (zstd !) + starting 4 threads for background file closing + starting 4 threads for background file closing + updating the branch cache + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + bundle2-input-part: total payload size 118984 (no-zstd !) + transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !) + bundle2-input-part: total payload size 116145 (zstd no-bigendian !) + bundle2-input-part: total payload size 116140 (zstd bigendian !) + bundle2-input-part: "listkeys" (params: 1 mandatory) supported + bundle2-input-bundle: 2 parts total + checking for updated bookmarks + updating the branch cache + (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) +#endif Cannot stream clone when there are secret changesets @@ -484,7 +614,15 @@ Streaming of secrets can be overridden b searching for changes no changes found #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --stream -U http://localhost:$HGPORT secret-allowed + streaming all changes + 1093 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1093 files to transfer, 98.9 KB of data (zstd !) + transferred 98.9 KB in * seconds (* */sec) (glob) (zstd !) +#endif +#if stream-bundle2-v3 $ hg clone --stream -U http://localhost:$HGPORT secret-allowed streaming all changes 1093 files to transfer, 102 KB of data (no-zstd !) @@ -638,7 +776,17 @@ clone it updating to branch default 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --stream http://localhost:$HGPORT with-bookmarks + streaming all changes + 1096 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1096 files to transfer, 99.1 KB of data (zstd !) + transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !) + updating to branch default + 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved +#endif +#if stream-bundle2-v3 $ hg clone --stream http://localhost:$HGPORT with-bookmarks streaming all changes 1096 files to transfer, 102 KB of data (no-zstd !) @@ -674,7 +822,17 @@ Clone as publishing updating to branch default 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --stream http://localhost:$HGPORT phase-publish + streaming all changes + 1096 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1096 files to transfer, 99.1 KB of data (zstd !) + transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !) + updating to branch default + 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved +#endif +#if stream-bundle2-v3 $ hg clone --stream http://localhost:$HGPORT phase-publish streaming all changes 1096 files to transfer, 102 KB of data (no-zstd !) @@ -720,7 +878,21 @@ stream v1 unsuitable for non-publishing 1: public 2: public #endif -#if stream-bundle2 +#if stream-bundle2-v2 + $ hg clone --stream http://localhost:$HGPORT phase-no-publish + streaming all changes + 1097 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1097 files to transfer, 99.1 KB of data (zstd !) + transferred 99.1 KB in * seconds (* */sec) (glob) (zstd !) + updating to branch default + 1088 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg -R phase-no-publish phase -r 'all()' + 0: draft + 1: draft + 2: draft +#endif +#if stream-bundle2-v3 $ hg clone --stream http://localhost:$HGPORT phase-no-publish streaming all changes 1097 files to transfer, 102 KB of data (no-zstd !) @@ -744,7 +916,57 @@ With v1 of the stream protocol, changese no obsolescence markers exchange in stream v1. #endif -#if stream-bundle2 +#if stream-bundle2-v2 + +Stream repository with obsolescence +----------------------------------- + +Clone non-publishing with obsolescence + + $ cat >> $HGRCPATH << EOF + > [experimental] + > evolution=all + > EOF + + $ cd server + $ echo foo > foo + $ hg -q commit -m 'about to be pruned' + $ hg debugobsolete `hg log -r . -T '{node}'` -d '0 0' -u test --record-parents + 1 new obsolescence markers + obsoleted 1 changesets + $ hg up null -q + $ hg log -T '{rev}: {phase}\n' + 2: draft + 1: draft + 0: draft + $ hg serve -p $HGPORT -d --pid-file=hg.pid + $ cat hg.pid > $DAEMON_PIDS + $ cd .. + + $ hg clone -U --stream http://localhost:$HGPORT with-obsolescence + streaming all changes + 1098 files to transfer, 102 KB of data (no-zstd !) + transferred 102 KB in * seconds (* */sec) (glob) (no-zstd !) + 1098 files to transfer, 99.5 KB of data (zstd !) + transferred 99.5 KB in * seconds (* */sec) (glob) (zstd !) + $ hg -R with-obsolescence log -T '{rev}: {phase}\n' + 2: draft + 1: draft + 0: draft + $ hg debugobsolete -R with-obsolescence + 8c206a663911c1f97f2f9d7382e417ae55872cfa 0 {5223b5e3265f0df40bb743da62249413d74ac70f} (Thu Jan 01 00:00:00 1970 +0000) {'user': 'test'} + $ hg verify -R with-obsolescence -q + + $ hg clone -U --stream --config experimental.evolution=0 http://localhost:$HGPORT with-obsolescence-no-evolution + streaming all changes + remote: abort: server has obsolescence markers, but client cannot receive them via stream clone + abort: pull failed on remote + [100] + + $ killdaemons.py + +#endif +#if stream-bundle2-v3 Stream repository with obsolescence -----------------------------------