subrepo: rewrite handling of subrepo state at commit (issue2403)

When the contents of .hgsubstate are stale (either because they've
manually been tweaked or partial updates have confused it), we get
confused about whether it actually needs committing.

So instead, we actively consult the parent's substate and compare it
the actual current state when deciding whether it needs committing.

Side effect: lots of "committing subrepo" messages that didn't
correspond with real commits disappear.

This change is fairly invasive for a fairly obscure condition, so it's
kept on the default branch.

mercurial/localrepo.py

@@ -1096,37 +1096,58 b' class localrepository(repo.repository):'
 
             # check subrepos
             subs = []
-            removedsubs = set()
+            commitsubs = set()
+            newstate = wctx.substate.copy()
+            # only manage subrepos and .hgsubstate if .hgsub is present
             if '.hgsub' in wctx:
-                # only manage subrepos and .hgsubstate if .hgsub is present
+                # we'll decide whether to track this ourselves, thanks
+                if '.hgsubstate' in changes[0]:
+                    changes[0].remove('.hgsubstate')
+                if '.hgsubstate' in changes[2]:
+                    changes[2].remove('.hgsubstate')
+
+                # compare current state to last committed state
+                # build new substate based on last committed state
+                oldstate = wctx.p1().substate
+                for s in sorted(newstate.keys()):
+                    if not match(s):
+                        # ignore working copy, use old state if present
+                        if s in oldstate:
+                            newstate[s] = oldstate[s]
+                            continue
+                        if not force:
+                            raise util.Abort(
+                                _("commit with new subrepo %s excluded") % s)
+                    if wctx.sub(s).dirty(True):
+                        if not self.ui.configbool('ui', 'commitsubrepos'):
+                            raise util.Abort(
+                                _("uncommitted changes in subrepo %s") % s,
+                                hint=_("use --subrepos for recursive commit"))
+                        subs.append(s)
+                        commitsubs.add(s)
+                    else:
+                        bs = wctx.sub(s).basestate()
+                        newstate[s] = (newstate[s][0], bs, newstate[s][2])
+                        if oldstate.get(s, (None, None, None))[1] != bs:
+                            subs.append(s)
+
+                # check for removed subrepos
                 for p in wctx.parents():
-                    removedsubs.update(s for s in p.substate if match(s))
-                for s in wctx.substate:
-                    removedsubs.discard(s)
-                    if match(s) and wctx.sub(s).dirty():
-                        subs.append(s)
-                if (subs or removedsubs):
+                    r = [s for s in p.substate if s not in newstate]
+                    subs += [s for s in r if match(s)]
+                if subs:
                     if (not match('.hgsub') and
                         '.hgsub' in (wctx.modified() + wctx.added())):
                         raise util.Abort(
                             _("can't commit subrepos without .hgsub"))
-                    if '.hgsubstate' not in changes[0]:
-                        changes[0].insert(0, '.hgsubstate')
-                        if '.hgsubstate' in changes[2]:
-                            changes[2].remove('.hgsubstate')
+                    changes[0].insert(0, '.hgsubstate')
+
             elif '.hgsub' in changes[2]:
                 # clean up .hgsubstate when .hgsub is removed
                 if ('.hgsubstate' in wctx and
                     '.hgsubstate' not in changes[0] + changes[1] + changes[2]):
                     changes[2].insert(0, '.hgsubstate')
 
-            if subs and not self.ui.configbool('ui', 'commitsubrepos', False):
-                changedsubs = [s for s in subs if wctx.sub(s).dirty(True)]
-                if changedsubs:
-                    raise util.Abort(_("uncommitted changes in subrepo %s")
-                                     % changedsubs[0],
-                                     hint=_("use --subrepos for recursive commit"))
-
             # make sure all explicit patterns are matched
             if not force and match.files():
                 matched = set(changes[0] + changes[1] + changes[2])
@@ -1162,16 +1183,15 b' class localrepository(repo.repository):'
                 cctx._text = editor(self, cctx, subs)
             edited = (text != cctx._text)
 
-            # commit subs
-            if subs or removedsubs:
-                state = wctx.substate.copy()
-                for s in sorted(subs):
+            # commit subs and write new state
+            if subs:
+                for s in sorted(commitsubs):
                     sub = wctx.sub(s)
                     self.ui.status(_('committing subrepository %s\n') %
                         subrepo.subrelpath(sub))
                     sr = sub.commit(cctx._text, user, date)
-                    state[s] = (state[s][0], sr)
-                subrepo.writestate(self, state)
+                    newstate[s] = (newstate[s][0], sr)
+                subrepo.writestate(self, newstate)
 
             # Save commit message in case this transaction gets rolled back
             # (e.g. by a pretxncommit hook).  Leave the content alone on

tests/test-check-code-hg.t

@@ -474,9 +474,6 b''
    >     except:
    warning: naked except clause
   mercurial/localrepo.py:0:
-   >                                      hint=_("use --subrepos for recursive commit"))
-   warning: line over 80 characters
-  mercurial/localrepo.py:0:
    >                         # we return an integer indicating remote head count change
    warning: line over 80 characters
   mercurial/localrepo.py:0:

tests/test-mq-subrepo-svn.t

@@ -37,7 +37,6 b' qnew on repo w/svn subrepo'
   $ hg status -S -X '**/format'
   A .hgsub
   $ hg qnew -m0 0.diff
-  committing subrepository sub
   $ cd sub
   $ echo a > a
   $ svn add a

tests/test-mq-subrepo.t

@@ -105,7 +105,6 b' handle subrepos safely on qnew'
   % update substate when adding .hgsub w/clean updated subrepo
   A .hgsub
   % qnew -m0 0.diff
-  committing subrepository sub
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -121,7 +120,6 b' handle subrepos safely on qnew'
   % update substate when modifying .hgsub w/clean updated subrepo
   M .hgsub
   % qnew -m1 1.diff
-  committing subrepository sub2
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -166,7 +164,6 b' handle subrepos safely on qrefresh'
   % update substate when adding .hgsub w/clean updated subrepo
   A .hgsub
   % qrefresh
-  committing subrepository sub
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -183,7 +180,6 b' handle subrepos safely on qrefresh'
   % update substate when modifying .hgsub w/clean updated subrepo
   M .hgsub
   % qrefresh
-  committing subrepository sub2
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -225,7 +221,6 b' handle subrepos safely on qpush/qpop'
   $ echo sub = sub > .hgsub
   $ hg add .hgsub
   $ hg qnew -m0 0.diff
-  committing subrepository sub
   $ hg debugsub
   path sub
    source   sub
@@ -277,7 +272,6 b' handle subrepos safely on qrecord'
   diff --git a/.hgsub b/.hgsub
   new file mode 100644
   examine changes to '.hgsub'? [Ynsfdaq?] 
-  committing subrepository sub
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -310,7 +304,6 b' handle subrepos safely on qrecord'
    sub = sub
   +sub2 = sub2
   record this change to '.hgsub'? [Ynsfdaq?] 
-  committing subrepository sub2
   path sub
    source   sub
    revision b2fdb12cd82b021c3b7053d67802e77b6eeaee31
@@ -360,4 +353,3 b' correctly handle subrepos with patch que'
   $ echo sub = sub >> .hgsub
   $ hg add .hgsub
   $ hg qnew 0.diff
-  committing subrepository sub

tests/test-subrepo-deep-nested-change.t

@@ -18,7 +18,6 b" Preparing the 'sub1' repo which depends "
   adding sub1/.hgsub (glob)
   adding sub1/sub1 (glob)
   $ hg commit -R sub1 -m "sub1 import"
-  committing subrepository sub2
 
 Preparing the 'main' repo which depends on the subrepo 'sub1'
 
@@ -33,7 +32,6 b" Preparing the 'main' repo which depends "
   adding main/.hgsub (glob)
   adding main/main (glob)
   $ hg commit -R main -m "main import"
-  committing subrepository sub1
 
 Cleaning both repositories, just as a clone -U
 

tests/test-subrepo-git.t

@@ -34,7 +34,6 b' add subrepo clone'
   $ git clone -q ../gitroot s
   $ hg add .hgsub
   $ hg commit -m 'new git subrepo'
-  committing subrepository s
   $ hg debugsub
   path s
    source   ../gitroot
@@ -55,7 +54,6 b' record a new commit from upstream from a'
   $ hg status --subrepos
   M s/g
   $ hg commit -m 'update git subrepo'
-  committing subrepository s
   $ hg debugsub
   path s
    source   ../gitroot
@@ -222,7 +220,6 b' sync to upstream git, distribute changes'
   $ git pull -q >/dev/null 2>/dev/null
   $ cd ..
   $ hg commit -m 'git upstream sync'
-  committing subrepository s
   $ hg debugsub
   path s
    source   ../gitroot
@@ -287,7 +284,6 b' create nested repo'
   $ echo inner = inner > .hgsub
   $ hg add .hgsub
   $ hg commit -m 'nested sub'
-  committing subrepository inner
 
 nested commit
 
@@ -339,27 +335,32 b" Don't crash if the .hgsubstate entry is "
   $ hg update 1 -q
   $ hg rm .hgsubstate
   $ hg commit .hgsubstate -m 'no substate'
-  created new head
+  nothing changed
+  [1]
   $ hg tag -l nosubstate
   $ hg manifest
   .hgsub
+  .hgsubstate
   a
 
   $ hg status -S
+  R .hgsubstate
   $ hg sum | grep commit
-  commit: 1 subrepos
+  commit: 1 removed, 1 subrepos (new branch head)
 
   $ hg commit -m 'restore substate'
-  committing subrepository s
+  nothing changed
+  [1]
   $ hg manifest
   .hgsub
   .hgsubstate
   a
   $ hg sum | grep commit
-  commit: (clean)
+  commit: 1 removed, 1 subrepos (new branch head)
 
   $ hg update -qC nosubstate
   $ ls s
+  g
 
 issue3109: false positives in git diff-index
 

tests/test-subrepo-missing.t

@@ -7,12 +7,10 b''
   $ echo 'subrepo = subrepo' > .hgsub
   $ hg ci -Am addsubrepo
   adding .hgsub
-  committing subrepository subrepo
   $ echo b > subrepo/b
   $ hg -R subrepo ci -Am addb
   adding b
   $ hg ci -m updatedsub
-  committing subrepository subrepo
 
 delete .hgsub and revert it
 

tests/test-subrepo-recursion.t

@@ -79,11 +79,9 b' The --subrepos flag overwrite the config'
 
   $ cd ..
   $ hg commit -m 0-2-1
-  committing subrepository bar
 
   $ cd ..
   $ hg commit -m 1-2-1
-  committing subrepository foo
 
 Change working directory:
 

tests/test-subrepo-relative-path.t

@@ -20,7 +20,6 b" Preparing the 'main' repo which depends "
   adding main/.hgsub (glob)
   adding main/main (glob)
   $ hg commit -R main -m "main import"
-  committing subrepository sub
 
 Cleaning both repositories, just as a clone -U
 

tests/test-subrepo-svn.t

@@ -69,8 +69,6 b' add first svn sub with leading whitespac'
   $ svn co --quiet "$SVNREPO"/src subdir/s
   $ hg add .hgsub
   $ hg ci -m1
-  committing subrepository s
-  committing subrepository subdir/s
 
 make sure we avoid empty commits (issue2445)
 
@@ -432,7 +430,6 b' are unknown directories being replaced b'
   $ echo "s =        [svn]       $SVNREPO/src" >> .hgsub
   $ hg add .hgsub
   $ hg ci -m addsub
-  committing subrepository s
   $ echo a > a
   $ hg ci -Am adda
   adding a
@@ -440,7 +437,6 b' are unknown directories being replaced b'
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ svn up -qr6 s
   $ hg ci -m updatesub
-  committing subrepository s
   created new head
   $ echo pyc > s/dir/epsilon.pyc
   $ hg up 1
@@ -462,14 +458,12 b' test having obstructions when switching '
   $ echo "obstruct =        [svn]       $SVNREPO/externals" >> .hgsub
   $ svn co -r5 --quiet "$SVNREPO"/externals obstruct
   $ hg commit -m 'Start making obstructed working copy'
-  committing subrepository obstruct
   $ hg book other
   $ hg co -r 'p1(tip)'
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo "obstruct =        [svn]       $SVNREPO/src" >> .hgsub
   $ svn co -r5 --quiet "$SVNREPO"/src obstruct
   $ hg commit -m 'Other branch which will be obstructed'
-  committing subrepository obstruct
   created new head
 
 Switching back to the head where we have another path mapped to the
@@ -530,12 +524,10 b' First, create that condition in the repo'
   Checked out revision 10.
   $ echo "recreated =        [svn]       $SVNREPO/branch" >> .hgsub
   $ hg ci -m addsub
-  committing subrepository recreated
   $ cd recreated
   $ svn up -q
   $ cd ..
   $ hg ci -m updatesub
-  committing subrepository recreated
   $ hg up -r-2
   D    *recreated/somethingnew (glob)
   A    *recreated/somethingold (glob)

tests/test-subrepo.t

@@ -37,7 +37,6 b' Issue2232: committing a subrepo without '
   commit: 1 added, 1 subrepos
   update: (current)
   $ hg ci -m1
-  committing subrepository s
 
 Revert can't (yet) revert subrepos:
 
@@ -105,7 +104,6 b' bump sub rev (and check it is ignored by'
   $ echo b > s/a
   $ hg -R s ci -ms1
   $ hg --config ui.commitsubrepos=no ci -m3
-  committing subrepository s
 
 leave sub dirty (and check ui.commitsubrepos=no aborts the commit)
 
@@ -455,7 +453,6 b" shouldn't need merging"
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg ci -Am1
   adding .hgsub
-  committing subrepository s
   $ hg branch br
   marked working directory as branch br
   (branches are permanent and global, did you want a bookmark?)
@@ -464,7 +461,6 b" shouldn't need merging"
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg ci -Am1
   adding b
-  committing subrepository s
   $ hg up default
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ echo c > c
@@ -483,7 +479,6 b" shouldn't need merging"
   $ echo d > d
   $ hg ci -Am1
   adding d
-  committing subrepository s
   $ hg up 3
   2 files updated, 0 files merged, 1 files removed, 0 files unresolved
   $ hg -R s up 5
@@ -491,7 +486,6 b" shouldn't need merging"
   $ echo e > e
   $ hg ci -Am1
   adding e
-  committing subrepository s
 
   $ hg up 5
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
@@ -519,8 +513,6 b' test subrepo delete from .hgsubstate'
   $ hg -R testdelete add
   adding testdelete/.hgsub (glob)
   $ hg -R testdelete ci -m "nested 1 & 2 added"
-  committing subrepository nested
-  committing subrepository nested2
   $ echo nested = nested > testdelete/.hgsub
   $ hg -R testdelete ci -m "nested 2 deleted"
   $ cat testdelete/.hgsubstate
@@ -550,8 +542,6 b' test repository cloning'
   $ hg -R main add
   adding main/.hgsub (glob)
   $ hg -R main ci -m "add subrepos"
-  committing subrepository nested_absolute
-  committing subrepository nested_relative
   $ cd ..
   $ hg clone mercurial/main mercurial2/main
   updating to branch default
@@ -574,7 +564,6 b' Issue1977: multirepo push should fail if'
   $ echo s = s > repo/.hgsub
   $ hg -R repo ci -Am1
   adding .hgsub
-  committing subrepository s
   $ hg clone repo repo2
   updating to branch default
   cloning subrepo s from $TESTTMP/sub/repo/s (glob)
@@ -590,7 +579,6 b' Issue1977: multirepo push should fail if'
   $ hg -R repo2/s ci -m3
   created new head
   $ hg -R repo2 ci -m3
-  committing subrepository s
   $ hg -q -R repo2 push
   abort: push creates new remote head 9d66565e64e1!
   (did you forget to merge? use push -f to force)
@@ -701,7 +689,6 b' subrepository:'
   $ echo subrepo-2 = subrepo-2 >> .hgsub
   $ hg add .hgsub
   $ hg ci -m 'Added subrepos'
-  committing subrepository subrepo-1
   committing subrepository subrepo-2
   $ hg st subrepo-2/file
 
@@ -859,17 +846,16 b" Test that removing .hgsubstate doesn't b"
 
   $ hg rm -f .hgsubstate
   $ hg ci -mrm
-  committing subrepository s
-  committing subrepository t
-  created new head
+  nothing changed
+  [1]
   $ hg log -vr tip
-  changeset:   14:3941e0aa5236
+  changeset:   13:925c17564ef8
   tag:         tip
-  parent:      11:365661e5936a
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
+  files:       .hgsubstate
   description:
-  rm
+  13
   
   
 
@@ -877,9 +863,11 b' Test that removing .hgsub removes .hgsub'
 
   $ hg rm .hgsub
   $ hg ci -mrm2
+  created new head
   $ hg log -vr tip
-  changeset:   15:8b31de9d13d1
+  changeset:   14:2400bccd50af
   tag:         tip
+  parent:      11:365661e5936a
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   files:       .hgsub .hgsubstate
@@ -890,13 +878,13 b' Test that removing .hgsub removes .hgsub'
 Test issue3153: diff -S with deleted subrepos
 
   $ hg diff --nodates -S -c .
-  diff -r 3941e0aa5236 -r 8b31de9d13d1 .hgsub
+  diff -r 365661e5936a -r 2400bccd50af .hgsub
   --- a/.hgsub
   +++ /dev/null
   @@ -1,2 +0,0 @@
   -s = s
   -t = t
-  diff -r 3941e0aa5236 -r 8b31de9d13d1 .hgsubstate
+  diff -r 365661e5936a -r 2400bccd50af .hgsubstate
   --- a/.hgsubstate
   +++ /dev/null
   @@ -1,2 +0,0 @@
@@ -911,7 +899,6 b' Test behavior of add for explicit path i'
   $ hg add .hgsub
   $ hg init s
   $ hg ci -m0
-  committing subrepository s
 Adding with an explicit path in a subrepo adds the file
   $ echo c1 > f1
   $ echo c2 > s/f2
@@ -925,7 +912,6 b' Adding with an explicit path in a subrep'
   $ hg ci -R s -m0
   $ hg ci -Am1
   adding f1
-  committing subrepository s
 Adding with an explicit path in a subrepo with -S has the same behavior
   $ echo c3 > f3
   $ echo c4 > s/f4
@@ -939,7 +925,6 b' Adding with an explicit path in a subrep'
   $ hg ci -R s -m1
   $ hg ci -Ama2
   adding f3
-  committing subrepository s
 Adding without a path or pattern silently ignores subrepos
   $ echo c5 > f5
   $ echo c6 > s/f6
@@ -958,7 +943,6 b' Adding without a path or pattern silentl'
   adding f6
   adding f7
   $ hg ci -m3
-  committing subrepository s
 Adding without a path or pattern with -S also adds files in subrepos
   $ echo c8 > f8
   $ echo c9 > s/f9
@@ -977,7 +961,6 b' Adding without a path or pattern with -S'
   A s/f9
   $ hg ci -R s -m3
   $ hg ci -m4
-  committing subrepository s
 Adding with a pattern silently ignores subrepos
   $ echo c11 > fm11
   $ echo c12 > fn12
@@ -1000,7 +983,6 b' Adding with a pattern silently ignores s'
   adding fn14
   $ hg ci -Am5
   adding fn12
-  committing subrepository s
 Adding with a pattern with -S also adds matches in subrepos
   $ echo c15 > fm15
   $ echo c16 > fn16
@@ -1023,7 +1005,6 b' Adding with a pattern with -S also adds '
   adding fn18
   $ hg ci -Am6
   adding fn16
-  committing subrepository s
 
 Test behavior of forget for explicit path in subrepo:
 Forgetting an explicit path in a subrepo untracks the file