push: add a way to allow concurrent pushes on unrelated heads

Client has a mechanism for the server to check that nothing changed server side
since the client prepared a push. That check is wide and any head changed on
the server will lead to an aborted push. We introduce a way for the client to
send a less strict checking. That logic will check that no heads impacted by
the push have been affected. If other unrelated heads (including named branches
heads) have been affected, the push will proceed.

This is very helpful for repositories with high developers traffic on different
heads, a common setup.

That behavior is currently controlled by an experimental option. The config
should live in the "server" section but bike-shedding of the name will happen
in the next changesets. Servers advertise this capability through a new bundle2
capability 'checkeads', using the value 'related'.

The 'test-push-race.t' is updated to check that new capabilities on the
documented cases.

mercurial/bundle2.py

@@ -1323,6 +1323,8 b' def getrepocaps(repo, allowpushback=Fals'
         caps['obsmarkers'] = supportedformat
     if allowpushback:
         caps['pushback'] = ()
+    if not repo.ui.configbool('experimental', 'checkheads-strict', True):
+        caps['checkheads'] = ('related',)
     return caps
 
 def bundle2caps(remote):
@@ -1603,6 +1605,35 b' def handlecheckheads(op, inpart):'
         raise error.PushRaced('repository changed while pushing - '
                               'please try again')
 
+@parthandler('check:updated-heads')
+def handlecheckupdatedheads(op, inpart):
+    """check for race on the heads touched by a push
+
+    This is similar to 'check:heads' but focus on the heads actually updated
+    during the push. If other activities happen on unrelated heads, it is
+    ignored.
+
+    This allow server with high traffic to avoid push contention as long as
+    unrelated parts of the graph are involved."""
+    h = inpart.read(20)
+    heads = []
+    while len(h) == 20:
+        heads.append(h)
+        h = inpart.read(20)
+    assert not h
+    # trigger a transaction so that we are guaranteed to have the lock now.
+    if op.ui.configbool('experimental', 'bundle2lazylocking'):
+        op.gettransaction()
+
+    currentheads = set()
+    for ls in op.repo.branchmap().itervalues():
+        currentheads.update(ls)
+
+    for h in heads:
+        if h not in currentheads:
+            raise error.PushRaced('repository changed while pushing - '
+                                  'please try again')
+
 @parthandler('output')
 def handleoutput(op, inpart):
     """forward output captured on the server to the client"""

mercurial/discovery.py

@@ -332,6 +332,7 b' def checkheads(pushop):'
         headssum = _headssummary(pushop)
     else:
         headssum = _oldheadssummary(repo, remoteheads, outgoing, inc)
+    pushop.pushbranchmap = headssum
     newbranches = [branch for branch, heads in headssum.iteritems()
                    if heads[0] is None]
     # 1. Check for new branches on the remote.

mercurial/exchange.py

@@ -323,8 +323,21 b' class pushoperation(object):'
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
-        # all remote heads before the push
+        # all remote topological heads before the push
         self.remoteheads = None
+        # Details of the remote branch pre and post push
+        #
+        # mapping: {'branch': ([remoteheads],
+        #                      [newheads],
+        #                      [unsyncedheads],
+        #                      [discardedheads])}
+        # - branch: the branch name
+        # - remoteheads: the list of remote heads known locally
+        #                None if the branch is new
+        # - newheads: the new remote heads (known locally) with outgoing pushed
+        # - unsyncedheads: the list of remote heads unknown locally.
+        # - discardedheads: the list of remote heads made obsolete by the push
+        self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # phases changes that must be pushed along side the changesets
@@ -712,8 +725,23 b' def _pushb2ctxcheckheads(pushop, bundler'
 
     Exists as an independent function to aid extensions
     """
-    if not pushop.force:
-        bundler.newpart('check:heads', data=iter(pushop.remoteheads))
+    # * 'force' do not check for push race,
+    # * if we don't push anything, there are nothing to check.
+    if not pushop.force and pushop.outgoing.missingheads:
+        allowunrelated = 'related' in bundler.capabilities.get('checkheads', ())
+        if not allowunrelated:
+            bundler.newpart('check:heads', data=iter(pushop.remoteheads))
+        else:
+            affected = set()
+            for branch, heads in pushop.pushbranchmap.iteritems():
+                remoteheads, newheads, unsyncedheads, discardedheads = heads
+                if remoteheads is not None:
+                    remote = set(remoteheads)
+                    affected |= set(discardedheads) & remote
+                    affected |= remote - set(newheads)
+            if affected:
+                data = iter(sorted(affected))
+                bundler.newpart('check:updated-heads', data=data)
 
 @b2partsgenerator('changeset')
 def _pushb2ctx(pushop, bundler):

tests/test-push-race.t

@@ -102,6 +102,21 b' A set of extension and shell functions e'
   > graph = log -G --rev 'sort(all(), "topo")'
   > EOF
 
+We tests multiple cases:
+* strict: no race detected,
+* unrelated: race on unrelated heads are allowed.
+
+#testcases strict unrelated
+
+#if unrelated
+
+  $ cat >> $HGRCPATH << EOF
+  > [experimental]
+  > checkheads-strict = no
+  > EOF
+
+#endif
+
 Setup
 -----
 
@@ -265,6 +280,7 b' Pushing'
 
 Check the result of the push
 
+#if strict
   $ cat ./push-log
   pushing to ssh://user@dummy/server
   searching for changes
@@ -282,6 +298,34 b' Check the result of the push'
   |/
   @  842e2fac6304 C-ROOT (default)
   
+#endif
+#if unrelated
+
+(The two heads are unrelated, push should be allowed)
+
+  $ cat ./push-log
+  pushing to ssh://user@dummy/server
+  searching for changes
+  wrote ready: $TESTTMP/readyfile
+  waiting on: $TESTTMP/watchfile
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files
+
+  $ hg -R server graph
+  o  59e76faf78bd C-D (default)
+  |
+  o  a9149a1428e2 C-B (default)
+  |
+  | o  51c544a58128 C-C (default)
+  | |
+  | o  98217d5a1659 C-A (default)
+  |/
+  @  842e2fac6304 C-ROOT (default)
+  
+#endif
+
 Pushing while someone creates a new head
 -----------------------------------------
 
@@ -295,6 +339,8 b' Pushing a new changeset while someone cr'
 
 (resync-all)
 
+#if strict
+
   $ hg -R ./server pull ./client-racy
   pulling from ./client-racy
   searching for changes
@@ -303,6 +349,17 b' Pushing a new changeset while someone cr'
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
+
+#endif
+#if unrelated
+
+  $ hg -R ./server pull ./client-racy
+  pulling from ./client-racy
+  searching for changes
+  no changes found
+
+#endif
+
   $ hg -R ./client-other pull
   pulling from ssh://user@dummy/server
   searching for changes
@@ -367,6 +424,8 b' Pushing'
 
 Check the result of the push
 
+#if strict
+
   $ cat ./push-log
   pushing to ssh://user@dummy/server
   searching for changes
@@ -389,6 +448,39 b' Check the result of the push'
   @  842e2fac6304 C-ROOT (default)
   
 
+#endif
+
+#if unrelated
+
+(The racing new head do not affect existing heads, push should go through)
+
+  $ cat ./push-log
+  pushing to ssh://user@dummy/server
+  searching for changes
+  wrote ready: $TESTTMP/readyfile
+  waiting on: $TESTTMP/watchfile
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files
+
+  $ hg -R server graph
+  o  d9e379a8c432 C-F (default)
+  |
+  o  51c544a58128 C-C (default)
+  |
+  | o  d603e2c0cdd7 C-E (default)
+  |/
+  o  98217d5a1659 C-A (default)
+  |
+  | o  59e76faf78bd C-D (default)
+  | |
+  | o  a9149a1428e2 C-B (default)
+  |/
+  @  842e2fac6304 C-ROOT (default)
+  
+#endif
+
 Pushing touching different named branch (same topo): new branch raced
 ---------------------------------------------------------------------
 
@@ -402,6 +494,8 b' Pushing two children on the same head, o'
 
 (resync-all)
 
+#if strict
+
   $ hg -R ./server pull ./client-racy
   pulling from ./client-racy
   searching for changes
@@ -410,6 +504,17 b' Pushing two children on the same head, o'
   adding file changes
   added 1 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
+
+#endif
+#if unrelated
+
+  $ hg -R ./server pull ./client-racy
+  pulling from ./client-racy
+  searching for changes
+  no changes found
+
+#endif
+
   $ hg -R ./client-other pull
   pulling from ssh://user@dummy/server
   searching for changes
@@ -480,6 +585,7 b' Pushing'
 
 Check the result of the push
 
+#if strict
   $ cat ./push-log
   pushing to ssh://user@dummy/server
   searching for changes
@@ -505,6 +611,43 b' Check the result of the push'
   |/
   @  842e2fac6304 C-ROOT (default)
   
+#endif
+#if unrelated
+
+(unrelated named branches are unrelated)
+
+  $ cat ./push-log
+  pushing to ssh://user@dummy/server
+  searching for changes
+  wrote ready: $TESTTMP/readyfile
+  waiting on: $TESTTMP/watchfile
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files (+1 heads)
+
+  $ hg -R server graph
+  o  833be552cfe6 C-H (my-first-test-branch)
+  |
+  | o  75d69cba5402 C-G (default)
+  |/
+  o  d9e379a8c432 C-F (default)
+  |
+  o  51c544a58128 C-C (default)
+  |
+  | o  d603e2c0cdd7 C-E (default)
+  |/
+  o  98217d5a1659 C-A (default)
+  |
+  | o  59e76faf78bd C-D (default)
+  | |
+  | o  a9149a1428e2 C-B (default)
+  |/
+  @  842e2fac6304 C-ROOT (default)
+  
+#endif
+
+The racing new head do not affect existing heads, push should go through
 
 pushing touching different named branch (same topo): old branch raced
 ---------------------------------------------------------------------
@@ -519,6 +662,8 b' Pushing two children on the same head, o'
 
 (resync-all)
 
+#if strict
+
   $ hg -R ./server pull ./client-racy
   pulling from ./client-racy
   searching for changes
@@ -527,6 +672,17 b' Pushing two children on the same head, o'
   adding file changes
   added 1 changesets with 1 changes to 1 files (+1 heads)
   (run 'hg heads .' to see heads, 'hg merge' to merge)
+
+#endif
+#if unrelated
+
+  $ hg -R ./server pull ./client-racy
+  pulling from ./client-racy
+  searching for changes
+  no changes found
+
+#endif
+
   $ hg -R ./client-other pull
   pulling from ssh://user@dummy/server
   searching for changes
@@ -600,6 +756,8 b' Pushing'
 
 Check the result of the push
 
+#if strict
+
   $ cat ./push-log
   pushing to ssh://user@dummy/server
   searching for changes
@@ -630,6 +788,48 b' Check the result of the push'
   @  842e2fac6304 C-ROOT (default)
   
 
+#endif
+
+#if unrelated
+
+(unrelated named branches are unrelated)
+
+  $ cat ./push-log
+  pushing to ssh://user@dummy/server
+  searching for changes
+  wrote ready: $TESTTMP/readyfile
+  waiting on: $TESTTMP/watchfile
+  remote: adding changesets
+  remote: adding manifests
+  remote: adding file changes
+  remote: added 1 changesets with 1 changes to 1 files (+1 heads)
+
+  $ hg -R server graph
+  o  89420bf00fae C-J (default)
+  |
+  | o  b35ed749f288 C-I (my-second-test-branch)
+  |/
+  o  75d69cba5402 C-G (default)
+  |
+  | o  833be552cfe6 C-H (my-first-test-branch)
+  |/
+  o  d9e379a8c432 C-F (default)
+  |
+  o  51c544a58128 C-C (default)
+  |
+  | o  d603e2c0cdd7 C-E (default)
+  |/
+  o  98217d5a1659 C-A (default)
+  |
+  | o  59e76faf78bd C-D (default)
+  | |
+  | o  a9149a1428e2 C-B (default)
+  |/
+  @  842e2fac6304 C-ROOT (default)
+  
+
+#endif
+
 pushing racing push touch multiple heads
 ----------------------------------------
 
@@ -644,6 +844,8 b' There are multiple heads, but the racing'
 
 (resync-all)
 
+#if strict
+
   $ hg -R ./server pull ./client-racy
   pulling from ./client-racy
   searching for changes
@@ -652,6 +854,18 b' There are multiple heads, but the racing'
   adding file changes
   added 1 changesets with 1 changes to 1 files (+1 heads)
   (run 'hg heads .' to see heads, 'hg merge' to merge)
+
+#endif
+
+#if unrelated
+
+  $ hg -R ./server pull ./client-racy
+  pulling from ./client-racy
+  searching for changes
+  no changes found
+
+#endif
+
   $ hg -R ./client-other pull
   pulling from ssh://user@dummy/server
   searching for changes