git: fix index handling of removed files during commit (issue6398)

Other changes in this series also changed the behavior here in
positive ways, but this was the final step that actually fixed things.

Differential Revision: https://phab.mercurial-scm.org/D8999

hgext/git/dirstate.py

@@ -1,339 +1,341 b''
 from __future__ import absolute_import
 
 import contextlib
 import errno
 import os
 
 from mercurial import (
     error,
     extensions,
     match as matchmod,
     node as nodemod,
     pycompat,
     scmutil,
     util,
 )
 from mercurial.interfaces import (
     dirstate as intdirstate,
     util as interfaceutil,
 )
 
 from . import gitutil
 
 pygit2 = gitutil.get_pygit2()
 
 
 def readpatternfile(orig, filepath, warn, sourceinfo=False):
     if not (b'info/exclude' in filepath or filepath.endswith(b'.gitignore')):
         return orig(filepath, warn, sourceinfo=False)
     result = []
     warnings = []
     with open(filepath, b'rb') as fp:
         for l in fp:
             l = l.strip()
             if not l or l.startswith(b'#'):
                 continue
             if l.startswith(b'!'):
                 warnings.append(b'unsupported ignore pattern %s' % l)
                 continue
             if l.startswith(b'/'):
                 result.append(b'rootglob:' + l[1:])
             else:
                 result.append(b'relglob:' + l)
     return result, warnings
 
 
 extensions.wrapfunction(matchmod, b'readpatternfile', readpatternfile)
 
 
 _STATUS_MAP = {}
 if pygit2:
     _STATUS_MAP = {
         pygit2.GIT_STATUS_CONFLICTED: b'm',
         pygit2.GIT_STATUS_CURRENT: b'n',
         pygit2.GIT_STATUS_IGNORED: b'?',
         pygit2.GIT_STATUS_INDEX_DELETED: b'r',
         pygit2.GIT_STATUS_INDEX_MODIFIED: b'n',
         pygit2.GIT_STATUS_INDEX_NEW: b'a',
         pygit2.GIT_STATUS_INDEX_RENAMED: b'a',
         pygit2.GIT_STATUS_INDEX_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_DELETED: b'r',
         pygit2.GIT_STATUS_WT_MODIFIED: b'n',
         pygit2.GIT_STATUS_WT_NEW: b'?',
         pygit2.GIT_STATUS_WT_RENAMED: b'a',
         pygit2.GIT_STATUS_WT_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_UNREADABLE: b'?',
         pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_WT_MODIFIED: 'm',
     }
 
 
 @interfaceutil.implementer(intdirstate.idirstate)
 class gitdirstate(object):
     def __init__(self, ui, root, gitrepo):
         self._ui = ui
         self._root = os.path.dirname(root)
         self.git = gitrepo
         self._plchangecallbacks = {}
 
     def p1(self):
         try:
             return self.git.head.peel().id.raw
         except pygit2.GitError:
             # Typically happens when peeling HEAD fails, as in an
             # empty repository.
             return nodemod.nullid
 
     def p2(self):
         # TODO: MERGE_HEAD? something like that, right?
         return nodemod.nullid
 
     def setparents(self, p1, p2=nodemod.nullid):
         assert p2 == nodemod.nullid, b'TODO merging support'
         self.git.head.set_target(gitutil.togitnode(p1))
 
     @util.propertycache
     def identity(self):
         return util.filestat.frompath(
             os.path.join(self._root, b'.git', b'index')
         )
 
     def branch(self):
         return b'default'
 
     def parents(self):
         # TODO how on earth do we find p2 if a merge is in flight?
         return self.p1(), nodemod.nullid
 
     def __iter__(self):
         return (pycompat.fsencode(f.path) for f in self.git.index)
 
     def items(self):
         for ie in self.git.index:
             yield ie.path, None  # value should be a dirstatetuple
 
     # py2,3 compat forward
     iteritems = items
 
     def __getitem__(self, filename):
         try:
             gs = self.git.status_file(filename)
         except KeyError:
             return b'?'
         return _STATUS_MAP[gs]
 
     def __contains__(self, filename):
         try:
             gs = self.git.status_file(filename)
             return _STATUS_MAP[gs] != b'?'
         except KeyError:
             return False
 
     def status(self, match, subrepos, ignored, clean, unknown):
         listignored, listclean, listunknown = ignored, clean, unknown
         # TODO handling of clean files - can we get that from git.status()?
         modified, added, removed, deleted, unknown, ignored, clean = (
             [],
             [],
             [],
             [],
             [],
             [],
             [],
         )
         gstatus = self.git.status()
         for path, status in gstatus.items():
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             if status == pygit2.GIT_STATUS_IGNORED:
                 if path.endswith(b'/'):
                     continue
                 ignored.append(path)
             elif status in (
                 pygit2.GIT_STATUS_WT_MODIFIED,
                 pygit2.GIT_STATUS_INDEX_MODIFIED,
                 pygit2.GIT_STATUS_WT_MODIFIED
                 | pygit2.GIT_STATUS_INDEX_MODIFIED,
             ):
                 modified.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_NEW:
                 added.append(path)
             elif status == pygit2.GIT_STATUS_WT_NEW:
                 unknown.append(path)
             elif status == pygit2.GIT_STATUS_WT_DELETED:
                 deleted.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_DELETED:
                 removed.append(path)
             else:
                 raise error.Abort(
                     b'unhandled case: status for %r is %r' % (path, status)
                 )
 
         if listclean:
             observed = set(
                 modified + added + removed + deleted + unknown + ignored
             )
             index = self.git.index
             index.read()
             for entry in index:
                 path = pycompat.fsencode(entry.path)
                 if not match(path):
                     continue
                 if path in observed:
                     continue  # already in some other set
                 if path[-1] == b'/':
                     continue  # directory
                 clean.append(path)
 
         # TODO are we really always sure of status here?
         return (
             False,
             scmutil.status(
                 modified, added, removed, deleted, unknown, ignored, clean
             ),
         )
 
     def flagfunc(self, buildfallback):
         # TODO we can do better
         return buildfallback()
 
     def getcwd(self):
         # TODO is this a good way to do this?
         return os.path.dirname(
             os.path.dirname(pycompat.fsencode(self.git.path))
         )
 
     def normalize(self, path):
         normed = util.normcase(path)
         assert normed == path, b"TODO handling of case folding: %s != %s" % (
             normed,
             path,
         )
         return path
 
     @property
     def _checklink(self):
         return util.checklink(os.path.dirname(pycompat.fsencode(self.git.path)))
 
     def copies(self):
         # TODO support copies?
         return {}
 
     # # TODO what the heck is this
     _filecache = set()
 
     def pendingparentchange(self):
         # TODO: we need to implement the context manager bits and
         # correctly stage/revert index edits.
         return False
 
     def write(self, tr):
         # TODO: call parent change callbacks
 
         if tr:
 
             def writeinner(category):
                 self.git.index.write()
 
             tr.addpending(b'gitdirstate', writeinner)
         else:
             self.git.index.write()
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         # TODO core dirstate does something about slashes here
         assert isinstance(f, bytes)
         r = util.pathto(self._root, cwd, f)
         return r
 
     def matches(self, match):
         for x in self.git.index:
             p = pycompat.fsencode(x.path)
             if match(p):
                 yield p
 
     def normal(self, f, parentfiledata=None):
         """Mark a file normal and clean."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def normallookup(self, f):
         """Mark a file normal, but possibly dirty."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         # TODO: we need to use .status() and not iterate the index,
         # because the index doesn't force a re-walk and so `hg add` of
         # a new file without an intervening call to status will
         # silently do nothing.
         r = {}
         cwd = self.getcwd()
         for path, status in self.git.status().items():
             if path.startswith('.hg/'):
                 continue
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             # TODO construct the stat info from the status object?
             try:
                 s = os.stat(os.path.join(cwd, path))
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 continue
             r[path] = s
         return r
 
     def savebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def restorebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def add(self, f):
         index = self.git.index
         index.read()
         index.add(pycompat.fsdecode(f))
         index.write()
 
     def drop(self, f):
         index = self.git.index
         index.read()
-        index.remove(pycompat.fsdecode(f))
+        fs = pycompat.fsdecode(f)
+        if fs in index:
+            index.remove(fs)
         index.write()
 
     def remove(self, f):
         index = self.git.index
         index.read()
         index.remove(pycompat.fsdecode(f))
         index.write()
 
     def copied(self, path):
         # TODO: track copies?
         return None
 
     def prefetch_parents(self):
         # TODO
         pass
 
     @contextlib.contextmanager
     def parentchange(self):
         # TODO: track this maybe?
         yield
 
     def addparentchangecallback(self, category, callback):
         # TODO: should this be added to the dirstate interface?
         self._plchangecallbacks[category] = callback
 
     def clearbackup(self, tr, backupname):
         # TODO
         pass
 
     def setbranch(self, branch):
         raise error.Abort(
             b'git repos do not support branches. try using bookmarks'
         )

tests/test-git-interop.t

@@ -1,272 +1,277 b''
 #require pygit2
 
 Setup:
   $ GIT_AUTHOR_NAME='test'; export GIT_AUTHOR_NAME
   > GIT_AUTHOR_EMAIL='[e-mail]'; export GIT_AUTHOR_EMAIL
   > GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0000"; export GIT_AUTHOR_DATE
   > GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"; export GIT_COMMITTER_NAME
   > GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"; export GIT_COMMITTER_EMAIL
   > GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"; export GIT_COMMITTER_DATE
   > count=10
   > gitcommit() {
   >    GIT_AUTHOR_DATE="2007-01-01 00:00:$count +0000";
   >    GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE"
   >    git commit "$@" >/dev/null 2>/dev/null || echo "git commit error"
   >    count=`expr $count + 1`
   >  }
 
 
 Test auto-loading extension works:
   $ mkdir nogit
   $ cd nogit
   $ mkdir .hg
   $ echo git >> .hg/requires
   $ hg status
   abort: repository specified git format in .hg/requires but has no .git directory
   [255]
   $ git init
   Initialized empty Git repository in $TESTTMP/nogit/.git/
 This status invocation shows some hg gunk because we didn't use
 `hg init --git`, which fixes up .git/info/exclude for us.
   $ hg status
   ? .hg/cache/git-commits.sqlite
   ? .hg/cache/git-commits.sqlite-shm
   ? .hg/cache/git-commits.sqlite-wal
   ? .hg/requires
   $ cd ..
 
 Now globally enable extension for the rest of the test:
   $ cat <<EOF >> $HGRCPATH
   > [extensions]
   > git=
   > [git]
   > log-index-cache-miss = yes
   > EOF
 
 Make a new repo with git:
   $ mkdir foo
   $ cd foo
   $ git init
   Initialized empty Git repository in $TESTTMP/foo/.git/
 Ignore the .hg directory within git:
   $ echo .hg >> .git/info/exclude
   $ echo alpha > alpha
   $ git add alpha
   $ gitcommit -am 'Add alpha'
   $ echo beta > beta
   $ git add beta
   $ gitcommit -am 'Add beta'
   $ echo gamma > gamma
   $ git status
   On branch master
   Untracked files:
     (use "git add <file>..." to include in what will be committed)
   	gamma
   
   nothing added to commit but untracked files present (use "git add" to track)
 
 Without creating the .hg, hg status fails:
   $ hg status
   abort: no repository found in '$TESTTMP/foo' (.hg not found)!
   [255]
 But if you run hg init --git, it works:
   $ hg init --git
   $ hg id --traceback
   heads mismatch, rebuilding dagcache
   3d9be8deba43 tip master
   $ hg status
   ? gamma
 Log works too:
   $ hg log
   changeset:   1:3d9be8deba43
   bookmark:    master
   tag:         tip
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:11 2007 +0000
   summary:     Add beta
   
   changeset:   0:c5864c9d16fb
   user:        test <[e-mail]>
   date:        Mon Jan 01 00:00:10 2007 +0000
   summary:     Add alpha
   
 
 
 and bookmarks:
   $ hg bookmarks
    * master                    1:3d9be8deba43
 
 diff even works transparently in both systems:
   $ echo blah >> alpha
   $ git diff
   diff --git a/alpha b/alpha
   index 4a58007..faed1b7 100644
   --- a/alpha
   +++ b/alpha
   @@ -1* +1,2 @@ (glob)
    alpha
   +blah
   $ hg diff --git
   diff --git a/alpha b/alpha
   --- a/alpha
   +++ b/alpha
   @@ -1,1 +1,2 @@
    alpha
   +blah
 
 Remove a file, it shows as such:
   $ rm alpha
   $ hg status
   ! alpha
   ? gamma
 
 Revert works:
   $ hg revert alpha --traceback
   $ hg status
   ? gamma
   $ git status
   On branch master
   Untracked files:
     (use "git add <file>..." to include in what will be committed)
   	gamma
   
   nothing added to commit but untracked files present (use "git add" to track)
 
 Add shows sanely in both:
   $ hg add gamma
   $ hg status
   A gamma
   $ hg files
   alpha
   beta
   gamma
   $ git ls-files
   alpha
   beta
   gamma
   $ git status
   On branch master
   Changes to be committed:
     (use "git restore --staged <file>..." to unstage)
   	new file:   gamma
   
 
 forget does what it should as well:
   $ hg forget gamma
   $ hg status
   ? gamma
   $ git status
   On branch master
   Untracked files:
     (use "git add <file>..." to include in what will be committed)
   	gamma
   
   nothing added to commit but untracked files present (use "git add" to track)
 
 clean up untracked file
   $ rm gamma
 
 hg log FILE
 
   $ echo a >> alpha
   $ hg ci -m 'more alpha' --traceback --date '1583522787 18000'
   $ echo b >> beta
   $ hg ci -m 'more beta'
   heads mismatch, rebuilding dagcache
   $ echo a >> alpha
   $ hg ci -m 'even more alpha'
   heads mismatch, rebuilding dagcache
   $ hg log -G alpha
   heads mismatch, rebuilding dagcache
   @  changeset:   4:6626247b7dc8
   :  bookmark:    master
   :  tag:         tip
   :  user:        test <test>
   :  date:        Thu Jan 01 00:00:00 1970 +0000
   :  summary:     even more alpha
   :
   o  changeset:   2:a1983dd7fb19
   :  user:        test <test>
   :  date:        Fri Mar 06 14:26:27 2020 -0500
   :  summary:     more alpha
   :
   o  changeset:   0:c5864c9d16fb
      user:        test <[e-mail]>
      date:        Mon Jan 01 00:00:10 2007 +0000
      summary:     Add alpha
   
   $ hg log -G beta
   o  changeset:   3:d8ee22687733
   :  user:        test <test>
   :  date:        Thu Jan 01 00:00:00 1970 +0000
   :  summary:     more beta
   :
   o  changeset:   1:3d9be8deba43
   |  user:        test <[e-mail]>
   ~  date:        Mon Jan 01 00:00:11 2007 +0000
      summary:     Add beta
   
 
   $ hg log -r "children(3d9be8deba43)" -T"{node|short} {children}\n"
   a1983dd7fb19 3:d8ee22687733
 
 hg annotate
 
   $ hg annotate alpha
   0: alpha
   2: a
   4: a
   $ hg annotate beta
   1: beta
   3: b
 
 
 Files in subdirectories. TODO: case-folding support, make this `A`
 instead of `a`.
 
   $ mkdir a
   $ echo "This is file mu." > a/mu
   $ hg ci -A -m 'Introduce file a/mu'
   adding a/mu
 
 Both hg and git agree a/mu is part of the repo
 
   $ git ls-files
   a/mu
   alpha
   beta
   $ hg files
   a/mu
   alpha
   beta
 
 hg and git status both clean
 
   $ git status
   On branch master
   nothing to commit, working tree clean
   $ hg status
   heads mismatch, rebuilding dagcache
 
 
 node|shortest works correctly
   $ hg log -T '{node}\n' | sort
   3d9be8deba43482be2c81a4cb4be1f10d85fa8bc
   6626247b7dc8f231b183b8a4761c89139baca2ad
   a1983dd7fb19cbd83ad5a1c2fc8bf3d775dea12f
   ae1ab744f95bfd5b07cf573baef98a778058537b
   c5864c9d16fb3431fe2c175ff84dc6accdbb2c18
   d8ee22687733a1991813560b15128cd9734f4b48
   $ hg log -r ae1ab744f95bfd5b07cf573baef98a778058537b --template "{shortest(node,1)}\n"
   ae
 
 This coveres changelog.findmissing()
   $ hg merge --preview 3d9be8deba43
 
 This covers manifest.diff()
   $ hg diff -c 3d9be8deba43
   diff -r c5864c9d16fb -r 3d9be8deba43 beta
   --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
   +++ b/beta	Mon Jan 01 00:00:11 2007 +0000
   @@ -0,0 +1,1 @@
   +beta
+
+
+Deleting files should also work (this was issue6398)
+  $ hg rm beta
+  $ hg ci -m 'remove beta'