narrow: use narrow_widen wireproto command to widen in case of ellipses

Few releases ago, we introduce narrow_widen wireproto command to be used to widen
narrow repositories. Before this patch, that was used in non-ellipses cases
only. In ellipses cases, we still do exchange.pull() which can pull more data
than required.

After this patch, the client will first check whether server supports doing
ellipses widening using wireproto command or not by checking server's wireproto
capability. If the server is upto date and support latest ellipses capability,
we call the wireproto command. Otherwise we fallback to exchange.pull() like
before.

The compat code make sure that things works even if one of the client or server
is old. The initial version of this patch does not had this compat code. It's
added to help Google release things smoothly internally. I plan to drop the
compat code before the upcoming major release.

Due to change to wireproto command, the code looks a bit dirty, next patches
will clean that up.

Differential Revision: https://phab.mercurial-scm.org/D6436

hgext/narrow/narrowcommands.py

@@ -146,7 +146,7 def pullbundle2extraprepare(orig, pullop
         kwargs['excludepats'] = exclude
     # calculate known nodes only in ellipses cases because in non-ellipses cases
     # we have all the nodes
-    if wireprototypes.ELLIPSESCAP in pullop.remote.capabilities():
+    if wireprototypes.ELLIPSESCAP1 in pullop.remote.capabilities():
         kwargs['known'] = [node.hex(ctx.node()) for ctx in
                            repo.set('::%ln', pullop.common)
                            if ctx.node() != node.nullid]
@@ -253,7 +253,14 def _widen(ui, repo, remote, commoninc,
     # then send that information to server whether we want ellipses or not.
     # Theoretically a non-ellipses repo should be able to use narrow
     # functionality from an ellipses enabled server
-    ellipsesremote = wireprototypes.ELLIPSESCAP in remote.capabilities()
+    remotecap = remote.capabilities()
+    ellipsesremote = any(cap in remotecap
+                         for cap in wireprototypes.SUPPORTED_ELLIPSESCAP)
+
+    # check whether we are talking to a server which supports old version of
+    # ellipses capabilities
+    isoldellipses = (ellipsesremote and wireprototypes.ELLIPSESCAP1 in
+                     remotecap and wireprototypes.ELLIPSESCAP not in remotecap)
 
     def pullbundle2extraprepare_widen(orig, pullop, kwargs):
         orig(pullop, kwargs)
@@ -279,8 +286,31 def _widen(ui, repo, remote, commoninc,
             with ds.parentchange():
                 ds.setparents(node.nullid, node.nullid)
             with wrappedextraprepare:
-                with repo.ui.configoverride(overrides, 'widen'):
+                if isoldellipses:
                     exchange.pull(repo, remote, heads=common)
+                else:
+                    known = [node.hex(ctx.node()) for ctx in
+                                       repo.set('::%ln', common)
+                                       if ctx.node() != node.nullid]
+
+                    with remote.commandexecutor() as e:
+                        bundle = e.callcommand('narrow_widen', {
+                            'oldincludes': oldincludes,
+                            'oldexcludes': oldexcludes,
+                            'newincludes': newincludes,
+                            'newexcludes': newexcludes,
+                            'cgversion': '03',
+                            'commonheads': common,
+                            'known': known,
+                            'ellipses': True,
+                        }).result()
+                    trmanager = exchange.transactionmanager(repo, 'widen',
+                                                            remote.url())
+                    with trmanager:
+                        op = bundle2.bundleoperation(repo,
+                                trmanager.transaction, source='widen')
+                        bundle2.processbundle(repo, bundle, op=op)
+
             with ds.parentchange():
                 ds.setparents(p1, p2)
         else:

hgext/narrow/narrowwirepeer.py

@@ -13,12 +13,15 from mercurial import (
     extensions,
     hg,
     narrowspec,
+    node as nodemod,
     pycompat,
     wireprototypes,
     wireprotov1peer,
     wireprotov1server,
 )
 
+from . import narrowbundle2
+
 def uisetup():
     wireprotov1peer.wirepeer.narrow_widen = peernarrowwiden
 
@@ -69,21 +72,26 def narrow_widen(repo, proto, oldinclude
         narrowspec.validatepatterns(set(newexcludes))
 
         common = wireprototypes.decodelist(commonheads)
-        known = None
-        if known:
-            known = wireprototypes.decodelist(known)
+        known = wireprototypes.decodelist(known)
+        known = {nodemod.bin(n) for n in known}
         if ellipses == '0':
             ellipses = False
         else:
             ellipses = bool(ellipses)
         cgversion = cgversion
-        newmatch = narrowspec.match(repo.root, include=newincludes,
-                                    exclude=newexcludes)
-        oldmatch = narrowspec.match(repo.root, include=oldincludes,
-                                    exclude=oldexcludes)
 
-        bundler = bundle2.widen_bundle(repo, oldmatch, newmatch, common, known,
-                                             cgversion, ellipses)
+        if not ellipses:
+            newmatch = narrowspec.match(repo.root, include=newincludes,
+                                        exclude=newexcludes)
+            oldmatch = narrowspec.match(repo.root, include=oldincludes,
+                                        exclude=oldexcludes)
+            bundler = bundle2.widen_bundle(repo, oldmatch, newmatch, common,
+                                           known, cgversion, ellipses)
+        else:
+            bundler = bundle2.bundle20(repo.ui)
+            narrowbundle2.generateellipsesbundle2(bundler, repo, oldincludes,
+                    oldexcludes, newincludes, newexcludes, cgversion, common,
+                    list(common), known, None)
     except error.Abort as exc:
         bundler = bundle2.bundle20(repo.ui)
         manargs = [('message', pycompat.bytestr(exc))]

mercurial/wireprototypes.py

@@ -30,7 +30,9 SSHV2 = 'exp-ssh-v2-0003'
 HTTP_WIREPROTO_V2 = 'exp-http-v2-0003'
 
 NARROWCAP = 'exp-narrow-1'
-ELLIPSESCAP = 'exp-ellipses-1'
+ELLIPSESCAP1 = 'exp-ellipses-1'
+ELLIPSESCAP = 'exp-ellipses-2'
+SUPPORTED_ELLIPSESCAP = (ELLIPSESCAP1, ELLIPSESCAP)
 
 # All available wire protocol transports.
 TRANSPORTS = {

tests/test-narrow-patterns.t

@@ -135,13 +135,11 widen the narrow checkout
   $ hg tracked --removeexclude dir1/dirA
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 9 changesets with 6 changes to 6 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:dir1
   I path:dir2
@@ -195,13 +193,11 widen narrow spec again, but exclude a f
   deleting data/dir1/dirA/bar.i (reporevlogstore !)
   deleting data/dir1/dirA/bar/0eca1d0cbdaea4651d1d04d71976a6d2d9bfaae5 (reposimplestore !)
   deleting data/dir1/dirA/bar/index (reposimplestore !)
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 11 changesets with 7 changes to 7 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:dir1
   I path:dir2
@@ -253,13 +249,11 widen narrow spec yet again, excluding a
   deleting data/dir1/dirA/foo.i (reporevlogstore !)
   deleting data/dir1/dirA/foo/162caeb3d55dceb1fee793aa631ac8c73fcb8b5e (reposimplestore !)
   deleting data/dir1/dirA/foo/index (reposimplestore !)
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 13 changesets with 8 changes to 8 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:dir1
   I path:dir2
@@ -312,13 +306,11 include a directory that was previously
   $ hg tracked --removeexclude dir1/dirA
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 13 changesets with 9 changes to 9 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:dir1
   I path:dir2
@@ -389,13 +381,11 clone a narrow portion of the master, su
   $ hg tracked --addinclude dir1
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow2/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 10 changesets with 6 changes to 6 files
-  new changesets *:* (glob)
   $ find * | sort
   dir1
   dir1/bar

tests/test-narrow-trackedcmd.t

@@ -145,13 +145,11 Testing the --import-rules flag of `hg t
   looking for local changes to affected paths
   deleting data/inside/f.i
   deleting meta/inside/00manifest.i (tree !)
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 0 changes to 0 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:outisde
   X path:inside
@@ -166,13 +164,11 Testing the --import-rules flag with --a
   $ hg tracked --import-rules specs --addinclude 'wider/'
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 1 changes to 1 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:outisde
   I path:wider
@@ -211,13 +207,11 Testing with passing a out of wdir file
   $ hg tracked --import-rules ../nspecs
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 0 changes to 0 files
-  new changesets *:* (glob)
 
   $ cd ..
 

tests/test-narrow-widen.t

@@ -95,13 +95,11 added upstream revisions.
   $ hg tracked --addinclude widest/f
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 2 changes to 2 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:inside
   I path:widest/f
@@ -154,13 +152,11 widen the narrow spec to include the wid
   $ hg tracked --addinclude wider
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 8 changesets with 7 changes to 3 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:inside
   I path:wider
@@ -261,13 +257,11 make narrow clone with every third node.
   $ hg tracked --addinclude d1
   comparing with ssh://user@dummy/upstream
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow2/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 9 changesets with 5 changes to 5 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:d0
   I path:d1
@@ -342,7 +336,6 Widening that fails can be recovered fro
   $ hg --config hooks.pretxnchangegroup.bad=false tracked --addinclude d1
   comparing with ssh://user@dummy/upstream
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/interrupted/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests

tests/test-narrow.t

@@ -290,13 +290,11 Can widen the empty clone
   $ hg tracked --addinclude d0
   comparing with ssh://user@dummy/master
   searching for changes
-  no changes found
   saved backup bundle to $TESTTMP/narrow-empty/.hg/strip-backup/*-widen.hg (glob)
   adding changesets
   adding manifests
   adding file changes
   added 3 changesets with 1 changes to 1 files
-  new changesets *:* (glob)
   $ hg tracked
   I path:d0
   $ hg files