git: adapt to some recent dirstate API changes

There are still old methods like add() and drop().  I don't see anything that
looks equivalent, so there's likely more work to do.  But this allows diff and
commit to work again on the simple webpage repo for thg.

Differential Revision: https://phab.mercurial-scm.org/D12567

hgext/git/__init__.py

@@ -1,343 +1,350 b''
 """grant Mercurial the ability to operate on Git repositories. (EXPERIMENTAL)
 
 This is currently super experimental. It probably will consume your
 firstborn a la Rumpelstiltskin, etc.
 """
 
 
 import os
 
 from mercurial.i18n import _
 
 from mercurial import (
     commands,
     error,
     extensions,
     localrepo,
     pycompat,
     registrar,
+    requirements as requirementsmod,
     scmutil,
     store,
     util,
 )
 
 from . import (
     dirstate,
     gitlog,
     gitutil,
     index,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 # git.log-index-cache-miss: internal knob for testing
 configitem(
     b"git",
     b"log-index-cache-miss",
     default=False,
 )
 
 getversion = gitutil.pygit2_version
 
 
 # TODO: extract an interface for this in core
 class gitstore:  # store.basicstore):
     def __init__(self, path, vfstype):
         self.vfs = vfstype(path)
         self.opener = self.vfs
         self.path = self.vfs.base
         self.createmode = store._calcmode(self.vfs)
         # above lines should go away in favor of:
         # super(gitstore, self).__init__(path, vfstype)
 
         self.git = gitutil.get_pygit2().Repository(
             os.path.normpath(os.path.join(path, b'..', b'.git'))
         )
         self._progress_factory = lambda *args, **kwargs: None
         self._logfn = lambda x: None
 
     @util.propertycache
     def _db(self):
         # We lazy-create the database because we want to thread a
         # progress callback down to the indexing process if it's
         # required, and we don't have a ui handle in makestore().
         return index.get_index(self.git, self._logfn, self._progress_factory)
 
     def join(self, f):
         """Fake store.join method for git repositories.
 
         For the most part, store.join is used for @storecache
         decorators to invalidate caches when various files
         change. We'll map the ones we care about, and ignore the rest.
         """
         if f in (b'00changelog.i', b'00manifest.i'):
             # This is close enough: in order for the changelog cache
             # to be invalidated, HEAD will have to change.
             return os.path.join(self.path, b'HEAD')
         elif f == b'lock':
             # TODO: we probably want to map this to a git lock, I
             # suspect index.lock. We should figure out what the
             # most-alike file is in git-land. For now we're risking
             # bad concurrency errors if another git client is used.
             return os.path.join(self.path, b'hgit-bogus-lock')
         elif f in (b'obsstore', b'phaseroots', b'narrowspec', b'bookmarks'):
             return os.path.join(self.path, b'..', b'.hg', f)
         raise NotImplementedError(b'Need to pick file for %s.' % f)
 
     def changelog(self, trypending, concurrencychecker):
         # TODO we don't have a plan for trypending in hg's git support yet
         return gitlog.changelog(self.git, self._db)
 
     def manifestlog(self, repo, storenarrowmatch):
         # TODO handle storenarrowmatch and figure out if we need the repo arg
         return gitlog.manifestlog(self.git, self._db)
 
     def invalidatecaches(self):
         pass
 
     def write(self, tr=None):
         # normally this handles things like fncache writes, which we don't have
         pass
 
 
 def _makestore(orig, requirements, storebasepath, vfstype):
     if b'git' in requirements:
         if not os.path.exists(os.path.join(storebasepath, b'..', b'.git')):
             raise error.Abort(
                 _(
                     b'repository specified git format in '
                     b'.hg/requires but has no .git directory'
                 )
             )
         # Check for presence of pygit2 only here. The assumption is that we'll
         # run this code iff we'll later need pygit2.
         if gitutil.get_pygit2() is None:
             raise error.Abort(
                 _(
                     b'the git extension requires the Python '
                     b'pygit2 library to be installed'
                 )
             )
 
         return gitstore(storebasepath, vfstype)
     return orig(requirements, storebasepath, vfstype)
 
 
 class gitfilestorage:
     def file(self, path):
         if path[0:1] == b'/':
             path = path[1:]
         return gitlog.filelog(self.store.git, self.store._db, path)
 
 
 def _makefilestorage(orig, requirements, features, **kwargs):
     store = kwargs['store']
     if isinstance(store, gitstore):
         return gitfilestorage
     return orig(requirements, features, **kwargs)
 
 
 def _setupdothg(ui, path):
     dothg = os.path.join(path, b'.hg')
     if os.path.exists(dothg):
         ui.warn(_(b'git repo already initialized for hg\n'))
     else:
         os.mkdir(os.path.join(path, b'.hg'))
         # TODO is it ok to extend .git/info/exclude like this?
         with open(
             os.path.join(path, b'.git', b'info', b'exclude'), 'ab'
         ) as exclude:
             exclude.write(b'\n.hg\n')
     with open(os.path.join(dothg, b'requires'), 'wb') as f:
         f.write(b'git\n')
 
 
 _BMS_PREFIX = 'refs/heads/'
 
 
 class gitbmstore:
     def __init__(self, gitrepo):
         self.gitrepo = gitrepo
         self._aclean = True
         self._active = gitrepo.references['HEAD']  # git head, not mark
 
     def __contains__(self, name):
         return (
             _BMS_PREFIX + pycompat.fsdecode(name)
         ) in self.gitrepo.references
 
     def __iter__(self):
         for r in self.gitrepo.listall_references():
             if r.startswith(_BMS_PREFIX):
                 yield pycompat.fsencode(r[len(_BMS_PREFIX) :])
 
     def __getitem__(self, k):
         return (
             self.gitrepo.references[_BMS_PREFIX + pycompat.fsdecode(k)]
             .peel()
             .id.raw
         )
 
     def get(self, k, default=None):
         try:
             if k in self:
                 return self[k]
             return default
         except gitutil.get_pygit2().InvalidSpecError:
             return default
 
     @property
     def active(self):
         h = self.gitrepo.references['HEAD']
         if not isinstance(h.target, str) or not h.target.startswith(
             _BMS_PREFIX
         ):
             return None
         return pycompat.fsencode(h.target[len(_BMS_PREFIX) :])
 
     @active.setter
     def active(self, mark):
         githead = mark is not None and (_BMS_PREFIX + mark) or None
         if githead is not None and githead not in self.gitrepo.references:
             raise AssertionError(b'bookmark %s does not exist!' % mark)
 
         self._active = githead
         self._aclean = False
 
     def _writeactive(self):
         if self._aclean:
             return
         self.gitrepo.references.create('HEAD', self._active, True)
         self._aclean = True
 
     def names(self, node):
         r = []
         for ref in self.gitrepo.listall_references():
             if not ref.startswith(_BMS_PREFIX):
                 continue
             if self.gitrepo.references[ref].peel().id.raw != node:
                 continue
             r.append(pycompat.fsencode(ref[len(_BMS_PREFIX) :]))
         return r
 
     # Cleanup opportunity: this is *identical* to core's bookmarks store.
     def expandname(self, bname):
         if bname == b'.':
             if self.active:
                 return self.active
             raise error.RepoLookupError(_(b"no active bookmark"))
         return bname
 
     def applychanges(self, repo, tr, changes):
         """Apply a list of changes to bookmarks"""
         # TODO: this should respect transactions, but that's going to
         # require enlarging the gitbmstore to know how to do in-memory
         # temporary writes and read those back prior to transaction
         # finalization.
         for name, node in changes:
             if node is None:
                 self.gitrepo.references.delete(
                     _BMS_PREFIX + pycompat.fsdecode(name)
                 )
             else:
                 self.gitrepo.references.create(
                     _BMS_PREFIX + pycompat.fsdecode(name),
                     gitutil.togitnode(node),
                     force=True,
                 )
 
     def checkconflict(self, mark, force=False, target=None):
         githead = _BMS_PREFIX + mark
         cur = self.gitrepo.references['HEAD']
         if githead in self.gitrepo.references and not force:
             if target:
                 if self.gitrepo.references[githead] == target and target == cur:
                     # re-activating a bookmark
                     return []
                 # moving a bookmark - forward?
                 raise NotImplementedError
             raise error.Abort(
                 _(b"bookmark '%s' already exists (use -f to force)") % mark
             )
         if len(mark) > 3 and not force:
             try:
                 shadowhash = scmutil.isrevsymbol(self._repo, mark)
             except error.LookupError:  # ambiguous identifier
                 shadowhash = False
             if shadowhash:
                 self._repo.ui.warn(
                     _(
                         b"bookmark %s matches a changeset hash\n"
                         b"(did you leave a -r out of an 'hg bookmark' "
                         b"command?)\n"
                     )
                     % mark
                 )
         return []
 
 
 def init(orig, ui, dest=b'.', **opts):
     if opts.get('git', False):
         path = util.abspath(dest)
         # TODO: walk up looking for the git repo
         _setupdothg(ui, path)
         return 0
     return orig(ui, dest=dest, **opts)
 
 
 def reposetup(ui, repo):
     if repo.local() and isinstance(repo.store, gitstore):
         orig = repo.__class__
         repo.store._progress_factory = repo.ui.makeprogress
         if ui.configbool(b'git', b'log-index-cache-miss'):
             repo.store._logfn = repo.ui.warn
 
         class gitlocalrepo(orig):
             def _makedirstate(self):
+                v2_req = requirementsmod.DIRSTATE_V2_REQUIREMENT
+                use_dirstate_v2 = v2_req in self.requirements
+
                 # TODO narrow support here
                 return dirstate.gitdirstate(
-                    self.ui, self.vfs.base, self.store.git
+                    self.ui,
+                    self.vfs,
+                    self.store.git,
+                    use_dirstate_v2,
                 )
 
             def commit(self, *args, **kwargs):
                 ret = orig.commit(self, *args, **kwargs)
                 if ret is None:
                     # there was nothing to commit, so we should skip
                     # the index fixup logic we'd otherwise do.
                     return None
                 tid = self.store.git[gitutil.togitnode(ret)].tree.id
                 # DANGER! This will flush any writes staged to the
                 # index in Git, but we're sidestepping the index in a
                 # way that confuses git when we commit. Alas.
                 self.store.git.index.read_tree(tid)
                 self.store.git.index.write()
                 return ret
 
             @property
             def _bookmarks(self):
                 return gitbmstore(self.store.git)
 
         repo.__class__ = gitlocalrepo
     return repo
 
 
 def _featuresetup(ui, supported):
     # don't die on seeing a repo with the git requirement
     supported |= {b'git'}
 
 
 def extsetup(ui):
     extensions.wrapfunction(localrepo, b'makestore', _makestore)
     extensions.wrapfunction(localrepo, b'makefilestorage', _makefilestorage)
     # Inject --git flag for `hg init`
     entry = extensions.wrapcommand(commands.table, b'init', init)
     entry[1].extend(
         [(b'', b'git', None, b'setup up a git repository instead of hg')]
     )
     localrepo.featuresetupfuncs.add(_featuresetup)

hgext/git/dirstate.py

@@ -1,367 +1,404 b''
 import contextlib
 import errno
 import os
 
 from mercurial.node import sha1nodeconstants
 from mercurial import (
+    dirstatemap,
     error,
     extensions,
     match as matchmod,
     pycompat,
     scmutil,
     util,
 )
+from mercurial.dirstateutils import (
+    timestamp,
+)
 from mercurial.interfaces import (
     dirstate as intdirstate,
     util as interfaceutil,
 )
 
 from . import gitutil
 
+
+DirstateItem = dirstatemap.DirstateItem
+propertycache = util.propertycache
 pygit2 = gitutil.get_pygit2()
 
 
 def readpatternfile(orig, filepath, warn, sourceinfo=False):
     if not (b'info/exclude' in filepath or filepath.endswith(b'.gitignore')):
         return orig(filepath, warn, sourceinfo=False)
     result = []
     warnings = []
     with open(filepath, b'rb') as fp:
         for l in fp:
             l = l.strip()
             if not l or l.startswith(b'#'):
                 continue
             if l.startswith(b'!'):
                 warnings.append(b'unsupported ignore pattern %s' % l)
                 continue
             if l.startswith(b'/'):
                 result.append(b'rootglob:' + l[1:])
             else:
                 result.append(b'relglob:' + l)
     return result, warnings
 
 
 extensions.wrapfunction(matchmod, b'readpatternfile', readpatternfile)
 
 
 _STATUS_MAP = {}
 if pygit2:
     _STATUS_MAP = {
         pygit2.GIT_STATUS_CONFLICTED: b'm',
         pygit2.GIT_STATUS_CURRENT: b'n',
         pygit2.GIT_STATUS_IGNORED: b'?',
         pygit2.GIT_STATUS_INDEX_DELETED: b'r',
         pygit2.GIT_STATUS_INDEX_MODIFIED: b'n',
         pygit2.GIT_STATUS_INDEX_NEW: b'a',
         pygit2.GIT_STATUS_INDEX_RENAMED: b'a',
         pygit2.GIT_STATUS_INDEX_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_DELETED: b'r',
         pygit2.GIT_STATUS_WT_MODIFIED: b'n',
         pygit2.GIT_STATUS_WT_NEW: b'?',
         pygit2.GIT_STATUS_WT_RENAMED: b'a',
         pygit2.GIT_STATUS_WT_TYPECHANGE: b'n',
         pygit2.GIT_STATUS_WT_UNREADABLE: b'?',
         pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_WT_MODIFIED: b'm',
     }
 
 
 @interfaceutil.implementer(intdirstate.idirstate)
 class gitdirstate:
-    def __init__(self, ui, root, gitrepo):
+    def __init__(self, ui, vfs, gitrepo, use_dirstate_v2):
         self._ui = ui
-        self._root = os.path.dirname(root)
+        self._root = os.path.dirname(vfs.base)
+        self._opener = vfs
         self.git = gitrepo
         self._plchangecallbacks = {}
         # TODO: context.poststatusfixup is bad and uses this attribute
         self._dirty = False
+        self._mapcls = dirstatemap.dirstatemap
+        self._use_dirstate_v2 = use_dirstate_v2
+
+    @propertycache
+    def _map(self):
+        """Return the dirstate contents (see documentation for dirstatemap)."""
+        self._map = self._mapcls(
+            self._ui,
+            self._opener,
+            self._root,
+            sha1nodeconstants,
+            self._use_dirstate_v2,
+        )
+        return self._map
 
     def p1(self):
         try:
             return self.git.head.peel().id.raw
         except pygit2.GitError:
             # Typically happens when peeling HEAD fails, as in an
             # empty repository.
             return sha1nodeconstants.nullid
 
     def p2(self):
         # TODO: MERGE_HEAD? something like that, right?
         return sha1nodeconstants.nullid
 
     def setparents(self, p1, p2=None):
         if p2 is None:
             p2 = sha1nodeconstants.nullid
         assert p2 == sha1nodeconstants.nullid, b'TODO merging support'
         self.git.head.set_target(gitutil.togitnode(p1))
 
     @util.propertycache
     def identity(self):
         return util.filestat.frompath(
             os.path.join(self._root, b'.git', b'index')
         )
 
     def branch(self):
         return b'default'
 
     def parents(self):
         # TODO how on earth do we find p2 if a merge is in flight?
         return self.p1(), sha1nodeconstants.nullid
 
     def __iter__(self):
         return (pycompat.fsencode(f.path) for f in self.git.index)
 
     def items(self):
         for ie in self.git.index:
             yield ie.path, None  # value should be a DirstateItem
 
     # py2,3 compat forward
     iteritems = items
 
     def __getitem__(self, filename):
         try:
             gs = self.git.status_file(filename)
         except KeyError:
             return b'?'
         return _STATUS_MAP[gs]
 
     def __contains__(self, filename):
         try:
             gs = self.git.status_file(filename)
             return _STATUS_MAP[gs] != b'?'
         except KeyError:
             return False
 
     def status(self, match, subrepos, ignored, clean, unknown):
         listclean = clean
         # TODO handling of clean files - can we get that from git.status()?
         modified, added, removed, deleted, unknown, ignored, clean = (
             [],
             [],
             [],
             [],
             [],
             [],
             [],
         )
+
+        try:
+            mtime_boundary = timestamp.get_fs_now(self._opener)
+        except OSError:
+            # In largefiles or readonly context
+            mtime_boundary = None
+
         gstatus = self.git.status()
         for path, status in gstatus.items():
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             if status == pygit2.GIT_STATUS_IGNORED:
                 if path.endswith(b'/'):
                     continue
                 ignored.append(path)
             elif status in (
                 pygit2.GIT_STATUS_WT_MODIFIED,
                 pygit2.GIT_STATUS_INDEX_MODIFIED,
                 pygit2.GIT_STATUS_WT_MODIFIED
                 | pygit2.GIT_STATUS_INDEX_MODIFIED,
             ):
                 modified.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_NEW:
                 added.append(path)
             elif status == pygit2.GIT_STATUS_WT_NEW:
                 unknown.append(path)
             elif status == pygit2.GIT_STATUS_WT_DELETED:
                 deleted.append(path)
             elif status == pygit2.GIT_STATUS_INDEX_DELETED:
                 removed.append(path)
             else:
                 raise error.Abort(
                     b'unhandled case: status for %r is %r' % (path, status)
                 )
 
         if listclean:
             observed = set(
                 modified + added + removed + deleted + unknown + ignored
             )
             index = self.git.index
             index.read()
             for entry in index:
                 path = pycompat.fsencode(entry.path)
                 if not match(path):
                     continue
                 if path in observed:
                     continue  # already in some other set
                 if path[-1] == b'/':
                     continue  # directory
                 clean.append(path)
 
         # TODO are we really always sure of status here?
         return (
             False,
             scmutil.status(
                 modified, added, removed, deleted, unknown, ignored, clean
             ),
+            mtime_boundary,
         )
 
     def flagfunc(self, buildfallback):
         # TODO we can do better
         return buildfallback()
 
     def getcwd(self):
         # TODO is this a good way to do this?
         return os.path.dirname(
             os.path.dirname(pycompat.fsencode(self.git.path))
         )
 
+    def get_entry(self, path):
+        """return a DirstateItem for the associated path"""
+        entry = self._map.get(path)
+        if entry is None:
+            return DirstateItem()
+        return entry
+
     def normalize(self, path):
         normed = util.normcase(path)
         assert normed == path, b"TODO handling of case folding: %s != %s" % (
             normed,
             path,
         )
         return path
 
     @property
     def _checklink(self):
         return util.checklink(os.path.dirname(pycompat.fsencode(self.git.path)))
 
     def copies(self):
         # TODO support copies?
         return {}
 
     # # TODO what the heck is this
     _filecache = set()
 
     def pendingparentchange(self):
         # TODO: we need to implement the context manager bits and
         # correctly stage/revert index edits.
         return False
 
     def write(self, tr):
         # TODO: call parent change callbacks
 
         if tr:
 
             def writeinner(category):
                 self.git.index.write()
 
             tr.addpending(b'gitdirstate', writeinner)
         else:
             self.git.index.write()
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         # TODO core dirstate does something about slashes here
         assert isinstance(f, bytes)
         r = util.pathto(self._root, cwd, f)
         return r
 
     def matches(self, match):
         for x in self.git.index:
             p = pycompat.fsencode(x.path)
             if match(p):
                 yield p
 
     def set_clean(self, f, parentfiledata):
         """Mark a file normal and clean."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def set_possibly_dirty(self, f):
         """Mark a file normal, but possibly dirty."""
         # TODO: for now we just let libgit2 re-stat the file. We can
         # clearly do better.
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         # TODO: we need to use .status() and not iterate the index,
         # because the index doesn't force a re-walk and so `hg add` of
         # a new file without an intervening call to status will
         # silently do nothing.
         r = {}
         cwd = self.getcwd()
         for path, status in self.git.status().items():
             if path.startswith('.hg/'):
                 continue
             path = pycompat.fsencode(path)
             if not match(path):
                 continue
             # TODO construct the stat info from the status object?
             try:
                 s = os.stat(os.path.join(cwd, path))
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 continue
             r[path] = s
         return r
 
     def savebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def restorebackup(self, tr, backupname):
         # TODO: figure out a strategy for saving index backups.
         pass
 
     def set_tracked(self, f):
         uf = pycompat.fsdecode(f)
         if uf in self.git.index:
             return False
         index = self.git.index
         index.read()
         index.add(uf)
         index.write()
         return True
 
     def add(self, f):
         index = self.git.index
         index.read()
         index.add(pycompat.fsdecode(f))
         index.write()
 
     def drop(self, f):
         index = self.git.index
         index.read()
         fs = pycompat.fsdecode(f)
         if fs in index:
             index.remove(fs)
             index.write()
 
     def set_untracked(self, f):
         index = self.git.index
         index.read()
         fs = pycompat.fsdecode(f)
         if fs in index:
             index.remove(fs)
             index.write()
             return True
         return False
 
     def remove(self, f):
         index = self.git.index
         index.read()
         index.remove(pycompat.fsdecode(f))
         index.write()
 
     def copied(self, path):
         # TODO: track copies?
         return None
 
     def prefetch_parents(self):
         # TODO
         pass
 
     def update_file(self, *args, **kwargs):
         # TODO
         pass
 
     @contextlib.contextmanager
     def parentchange(self):
         # TODO: track this maybe?
         yield
 
     def addparentchangecallback(self, category, callback):
         # TODO: should this be added to the dirstate interface?
         self._plchangecallbacks[category] = callback
 
     def clearbackup(self, tr, backupname):
         # TODO
         pass
 
     def setbranch(self, branch):
         raise error.Abort(
             b'git repos do not support branches. try using bookmarks'
         )