rust-dirstate: fall back to v1 if reading v2 failed

This will help us not fail when a v1 dirstate is present on disk while a v2
was expected (which could happen with a racy/interrupted upgrade).

mercurial/dirstatemap.py

@@ -1,786 +1,852 b''
 # dirstatemap.py
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
+import struct
 from .i18n import _
 
 from . import (
     error,
     pathutil,
     policy,
     testing,
     txnutil,
     util,
 )
 
 from .dirstateutils import (
     docket as docketmod,
     v2,
 )
 
 parsers = policy.importmod('parsers')
 rustmod = policy.importrust('dirstate')
 
 propertycache = util.propertycache
 
 if rustmod is None:
     DirstateItem = parsers.DirstateItem
 else:
     DirstateItem = rustmod.DirstateItem
 
 rangemask = 0x7FFFFFFF
 
 WRITE_MODE_AUTO = 0
 WRITE_MODE_FORCE_NEW = 1
 WRITE_MODE_FORCE_APPEND = 2
 
 
 V2_MAX_READ_ATTEMPTS = 5
 
 
 class _dirstatemapcommon:
     """
     Methods that are identical for both implementations of the dirstatemap
     class, with and without Rust extensions enabled.
     """
 
     # please pytype
 
     _map = None
     copymap = None
 
     def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
         self._use_dirstate_v2 = use_dirstate_v2
         self._nodeconstants = nodeconstants
         self._ui = ui
         self._opener = opener
         self._root = root
         self._filename = b'dirstate'
         self._nodelen = 20  # Also update Rust code when changing this!
         self._parents = None
         self._dirtyparents = False
         self._docket = None
         write_mode = ui.config(b"devel", b"dirstate.v2.data_update_mode")
         if write_mode == b"auto":
             self._write_mode = WRITE_MODE_AUTO
         elif write_mode == b"force-append":
             self._write_mode = WRITE_MODE_FORCE_APPEND
         elif write_mode == b"force-new":
             self._write_mode = WRITE_MODE_FORCE_NEW
         else:
             # unknown value, fallback to default
             self._write_mode = WRITE_MODE_AUTO
 
         # for consistent view between _pl() and _read() invocations
         self._pendingmode = None
 
     def _set_identity(self):
         self.identity = self._get_current_identity()
 
     def _get_current_identity(self):
         try:
             return util.cachestat(self._opener.join(self._filename))
         except FileNotFoundError:
             return None
 
     def may_need_refresh(self):
         if 'identity' not in vars(self):
             # no existing identity, we need a refresh
             return True
         if self.identity is None:
             return True
         if not self.identity.cacheable():
             # We cannot trust the entry
             # XXX this is a problem on windows, NFS, or other inode less system
             return True
         current_identity = self._get_current_identity()
         if current_identity is None:
             return True
         if not current_identity.cacheable():
             # We cannot trust the entry
             # XXX this is a problem on windows, NFS, or other inode less system
             return True
         return current_identity != self.identity
 
     def preload(self):
         """Loads the underlying data, if it's not already loaded"""
         self._map
 
     def get(self, key, default=None):
         return self._map.get(key, default)
 
     def __len__(self):
         return len(self._map)
 
     def __iter__(self):
         return iter(self._map)
 
     def __contains__(self, key):
         return key in self._map
 
     def __getitem__(self, item):
         return self._map[item]
 
     ### disk interaction
 
     def _opendirstatefile(self):
         fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
         if self._pendingmode is not None and self._pendingmode != mode:
             fp.close()
             raise error.Abort(
                 _(b'working directory state may be changed parallelly')
             )
         self._pendingmode = mode
         return fp
 
     def _readdirstatefile(self, size=-1):
         try:
             with self._opendirstatefile() as fp:
                 return fp.read(size)
         except FileNotFoundError:
             # File doesn't exist, so the current state is empty
             return b''
 
     @property
     def docket(self):
         if not self._docket:
             if not self._use_dirstate_v2:
                 raise error.ProgrammingError(
                     b'dirstate only has a docket in v2 format'
                 )
             self._set_identity()
-            self._docket = docketmod.DirstateDocket.parse(
-                self._readdirstatefile(), self._nodeconstants
-            )
+            try:
+                self._docket = docketmod.DirstateDocket.parse(
+                    self._readdirstatefile(), self._nodeconstants
+                )
+            except struct.error:
+                self._ui.debug(b"failed to read dirstate-v2 data")
+                raise error.CorruptedDirstate(
+                    b"failed to read dirstate-v2 data"
+                )
         return self._docket
 
     def _read_v2_data(self):
         data = None
         attempts = 0
         while attempts < V2_MAX_READ_ATTEMPTS:
             attempts += 1
             try:
                 # TODO: use mmap when possible
                 data = self._opener.read(self.docket.data_filename())
             except FileNotFoundError:
                 # read race detected between docket and data file
                 # reload the docket and retry
                 self._docket = None
         if data is None:
             assert attempts >= V2_MAX_READ_ATTEMPTS
             msg = b"dirstate read race happened %d times in a row"
             msg %= attempts
             raise error.Abort(msg)
         return self._opener.read(self.docket.data_filename())
 
     def write_v2_no_append(self, tr, st, meta, packed):
-        old_docket = self.docket
+        try:
+            old_docket = self.docket
+        except error.CorruptedDirstate:
+            # This means we've identified a dirstate-v1 file on-disk when we
+            # were expecting a dirstate-v2 docket. We've managed to recover
+            # from that unexpected situation, and now we want to write back a
+            # dirstate-v2 file to make the on-disk situation right again.
+            #
+            # This shouldn't be triggered since `self.docket` is cached and
+            # we would have called parents() or read() first, but it's here
+            # just in case.
+            old_docket = None
+
         new_docket = docketmod.DirstateDocket.with_new_uuid(
             self.parents(), len(packed), meta
         )
-        if old_docket.uuid == new_docket.uuid:
+        if old_docket is not None and old_docket.uuid == new_docket.uuid:
             raise error.ProgrammingError(b'dirstate docket name collision')
         data_filename = new_docket.data_filename()
         self._opener.write(data_filename, packed)
         # tell the transaction that we are adding a new file
         if tr is not None:
             tr.addbackup(data_filename, location=b'plain')
         # Write the new docket after the new data file has been
         # written. Because `st` was opened with `atomictemp=True`,
         # the actual `.hg/dirstate` file is only affected on close.
         st.write(new_docket.serialize())
         st.close()
         # Remove the old data file after the new docket pointing to
         # the new data file was written.
-        if old_docket.uuid:
+        if old_docket is not None and old_docket.uuid:
             data_filename = old_docket.data_filename()
             if tr is not None:
                 tr.addbackup(data_filename, location=b'plain')
             unlink = lambda _tr=None: self._opener.unlink(data_filename)
             if tr:
                 category = b"dirstate-v2-clean-" + old_docket.uuid
                 tr.addpostclose(category, unlink)
             else:
                 unlink()
         self._docket = new_docket
 
     ### reading/setting parents
 
     def parents(self):
         if not self._parents:
             if self._use_dirstate_v2:
-                self._parents = self.docket.parents
+                try:
+                    self.docket
+                except error.CorruptedDirstate as e:
+                    # fall back to dirstate-v1 if we fail to read v2
+                    self._v1_parents(e)
+                else:
+                    self._parents = self.docket.parents
             else:
-                read_len = self._nodelen * 2
-                st = self._readdirstatefile(read_len)
-                l = len(st)
-                if l == read_len:
-                    self._parents = (
-                        st[: self._nodelen],
-                        st[self._nodelen : 2 * self._nodelen],
-                    )
-                elif l == 0:
-                    self._parents = (
-                        self._nodeconstants.nullid,
-                        self._nodeconstants.nullid,
-                    )
-                else:
-                    raise error.Abort(
-                        _(b'working directory state appears damaged!')
-                    )
+                self._v1_parents()
 
         return self._parents
 
+    def _v1_parents(self, from_v2_exception=None):
+        read_len = self._nodelen * 2
+        st = self._readdirstatefile(read_len)
+        l = len(st)
+        if l == read_len:
+            self._parents = (
+                st[: self._nodelen],
+                st[self._nodelen : 2 * self._nodelen],
+            )
+        elif l == 0:
+            self._parents = (
+                self._nodeconstants.nullid,
+                self._nodeconstants.nullid,
+            )
+        else:
+            hint = None
+            if from_v2_exception is not None:
+                hint = _(b"falling back to dirstate-v1 from v2 also failed")
+            raise error.Abort(
+                _(b'working directory state appears damaged!'), hint
+            )
+
 
 class dirstatemap(_dirstatemapcommon):
     """Map encapsulating the dirstate's contents.
 
     The dirstate contains the following state:
 
     - `identity` is the identity of the dirstate file, which can be used to
       detect when changes have occurred to the dirstate file.
 
     - `parents` is a pair containing the parents of the working copy. The
       parents are updated by calling `setparents`.
 
     - the state map maps filenames to tuples of (state, mode, size, mtime),
       where state is a single character representing 'normal', 'added',
       'removed', or 'merged'. It is read by treating the dirstate as a
       dict.  File state is updated by calling various methods (see each
       documentation for details):
 
       - `reset_state`,
       - `set_tracked`
       - `set_untracked`
       - `set_clean`
       - `set_possibly_dirty`
 
     - `copymap` maps destination filenames to their source filename.
 
     The dirstate also provides the following views onto the state:
 
     - `filefoldmap` is a dict mapping normalized filenames to the denormalized
       form that they appear as in the dirstate.
 
     - `dirfoldmap` is a dict mapping normalized directory names to the
       denormalized form that they appear as in the dirstate.
     """
 
     ### Core data storage and access
 
     @propertycache
     def _map(self):
         self._map = {}
         self.read()
         return self._map
 
     @propertycache
     def copymap(self):
         self.copymap = {}
         self._map
         return self.copymap
 
     def clear(self):
         self._map.clear()
         self.copymap.clear()
         self.setparents(self._nodeconstants.nullid, self._nodeconstants.nullid)
         util.clearcachedproperty(self, b"_dirs")
         util.clearcachedproperty(self, b"_alldirs")
         util.clearcachedproperty(self, b"filefoldmap")
         util.clearcachedproperty(self, b"dirfoldmap")
 
     def items(self):
         return self._map.items()
 
     # forward for python2,3 compat
     iteritems = items
 
     def debug_iter(self, all):
         """
         Return an iterator of (filename, state, mode, size, mtime) tuples
 
         `all` is unused when Rust is not enabled
         """
         for (filename, item) in self.items():
             yield (filename, item.state, item.mode, item.size, item.mtime)
 
     def keys(self):
         return self._map.keys()
 
     ### reading/setting parents
 
     def setparents(self, p1, p2, fold_p2=False):
         self._parents = (p1, p2)
         self._dirtyparents = True
         copies = {}
         if fold_p2:
             for f, s in self._map.items():
                 # Discard "merged" markers when moving away from a merge state
                 if s.p2_info:
                     source = self.copymap.pop(f, None)
                     if source:
                         copies[f] = source
                     s.drop_merge_data()
         return copies
 
     ### disk interaction
 
     def read(self):
         testing.wait_on_cfg(self._ui, b'dirstate.pre-read-file')
         if self._use_dirstate_v2:
-
-            if not self.docket.uuid:
-                return
-            testing.wait_on_cfg(self._ui, b'dirstate.post-docket-read-file')
-            st = self._read_v2_data()
+            try:
+                self.docket
+            except error.CorruptedDirstate:
+                # fall back to dirstate-v1 if we fail to read v2
+                self._set_identity()
+                st = self._readdirstatefile()
+            else:
+                if not self.docket.uuid:
+                    return
+                testing.wait_on_cfg(self._ui, b'dirstate.post-docket-read-file')
+                st = self._read_v2_data()
         else:
             self._set_identity()
             st = self._readdirstatefile()
 
         if not st:
             return
 
         # TODO: adjust this estimate for dirstate-v2
         if util.safehasattr(parsers, b'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. This trades wasting some memory for avoiding costly
             # resizes. Each entry have a prefix of 17 bytes followed by one or
             # two path names. Studies on various large-scale real-world repositories
             # found 54 bytes a reasonable upper limit for the average path names.
             # Copy entries are ignored for the sake of this estimate.
             self._map = parsers.dict_new_presized(len(st) // 71)
 
         # Python's garbage collector triggers a GC each time a certain number
         # of container objects (the number being defined by
         # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
         # for each file in the dirstate. The C version then immediately marks
         # them as not to be tracked by the collector. However, this has no
         # effect on when GCs are triggered, only on what objects the GC looks
         # into. This means that O(number of files) GCs are unavoidable.
         # Depending on when in the process's lifetime the dirstate is parsed,
         # this can get very expensive. As a workaround, disable GC while
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
         if self._use_dirstate_v2:
-            p = self.docket.parents
-            meta = self.docket.tree_metadata
-            parse_dirstate = util.nogc(v2.parse_dirstate)
-            parse_dirstate(self._map, self.copymap, st, meta)
+            try:
+                self.docket
+            except error.CorruptedDirstate:
+                # fall back to dirstate-v1 if we fail to parse v2
+                parse_dirstate = util.nogc(parsers.parse_dirstate)
+                p = parse_dirstate(self._map, self.copymap, st)
+            else:
+                p = self.docket.parents
+                meta = self.docket.tree_metadata
+                parse_dirstate = util.nogc(v2.parse_dirstate)
+                parse_dirstate(self._map, self.copymap, st, meta)
         else:
             parse_dirstate = util.nogc(parsers.parse_dirstate)
             p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
 
         # Avoid excess attribute lookups by fast pathing certain checks
         self.__contains__ = self._map.__contains__
         self.__getitem__ = self._map.__getitem__
         self.get = self._map.get
 
     def write(self, tr, st):
         if self._use_dirstate_v2:
             packed, meta = v2.pack_dirstate(self._map, self.copymap)
             self.write_v2_no_append(tr, st, meta, packed)
         else:
             packed = parsers.pack_dirstate(
                 self._map, self.copymap, self.parents()
             )
             st.write(packed)
             st.close()
         self._dirtyparents = False
 
     @propertycache
     def identity(self):
         self._map
         return self.identity
 
     ### code related to maintaining and accessing "extra" property
     # (e.g. "has_dir")
 
     def _dirs_incr(self, filename, old_entry=None):
         """increment the dirstate counter if applicable"""
         if (
             old_entry is None or old_entry.removed
         ) and "_dirs" in self.__dict__:
             self._dirs.addpath(filename)
         if old_entry is None and "_alldirs" in self.__dict__:
             self._alldirs.addpath(filename)
 
     def _dirs_decr(self, filename, old_entry=None, remove_variant=False):
         """decrement the dirstate counter if applicable"""
         if old_entry is not None:
             if "_dirs" in self.__dict__ and not old_entry.removed:
                 self._dirs.delpath(filename)
             if "_alldirs" in self.__dict__ and not remove_variant:
                 self._alldirs.delpath(filename)
         elif remove_variant and "_alldirs" in self.__dict__:
             self._alldirs.addpath(filename)
         if "filefoldmap" in self.__dict__:
             normed = util.normcase(filename)
             self.filefoldmap.pop(normed, None)
 
     @propertycache
     def filefoldmap(self):
         """Returns a dictionary mapping normalized case paths to their
         non-normalized versions.
         """
         try:
             makefilefoldmap = parsers.make_file_foldmap
         except AttributeError:
             pass
         else:
             return makefilefoldmap(
                 self._map, util.normcasespec, util.normcasefallback
             )
 
         f = {}
         normcase = util.normcase
         for name, s in self._map.items():
             if not s.removed:
                 f[normcase(name)] = name
         f[b'.'] = b'.'  # prevents useless util.fspath() invocation
         return f
 
     @propertycache
     def dirfoldmap(self):
         f = {}
         normcase = util.normcase
         for name in self._dirs:
             f[normcase(name)] = name
         return f
 
     def hastrackeddir(self, d):
         """
         Returns True if the dirstate contains a tracked (not removed) file
         in this directory.
         """
         return d in self._dirs
 
     def hasdir(self, d):
         """
         Returns True if the dirstate contains a file (tracked or removed)
         in this directory.
         """
         return d in self._alldirs
 
     @propertycache
     def _dirs(self):
         return pathutil.dirs(self._map, only_tracked=True)
 
     @propertycache
     def _alldirs(self):
         return pathutil.dirs(self._map)
 
     ### code related to manipulation of entries and copy-sources
 
     def reset_state(
         self,
         filename,
         wc_tracked=False,
         p1_tracked=False,
         p2_info=False,
         has_meaningful_mtime=True,
         parentfiledata=None,
     ):
         """Set a entry to a given state, diregarding all previous state
 
         This is to be used by the part of the dirstate API dedicated to
         adjusting the dirstate after a update/merge.
 
         note: calling this might result to no entry existing at all if the
         dirstate map does not see any point at having one for this file
         anymore.
         """
         # copy information are now outdated
         # (maybe new information should be in directly passed to this function)
         self.copymap.pop(filename, None)
 
         if not (p1_tracked or p2_info or wc_tracked):
             old_entry = self._map.get(filename)
             self._drop_entry(filename)
             self._dirs_decr(filename, old_entry=old_entry)
             return
 
         old_entry = self._map.get(filename)
         self._dirs_incr(filename, old_entry)
         entry = DirstateItem(
             wc_tracked=wc_tracked,
             p1_tracked=p1_tracked,
             p2_info=p2_info,
             has_meaningful_mtime=has_meaningful_mtime,
             parentfiledata=parentfiledata,
         )
         self._map[filename] = entry
 
     def set_tracked(self, filename):
         new = False
         entry = self.get(filename)
         if entry is None:
             self._dirs_incr(filename)
             entry = DirstateItem(
                 wc_tracked=True,
             )
 
             self._map[filename] = entry
             new = True
         elif not entry.tracked:
             self._dirs_incr(filename, entry)
             entry.set_tracked()
             self._refresh_entry(filename, entry)
             new = True
         else:
             # XXX This is probably overkill for more case, but we need this to
             # fully replace the `normallookup` call with `set_tracked` one.
             # Consider smoothing this in the future.
             entry.set_possibly_dirty()
             self._refresh_entry(filename, entry)
         return new
 
     def set_untracked(self, f):
         """Mark a file as no longer tracked in the dirstate map"""
         entry = self.get(f)
         if entry is None:
             return False
         else:
             self._dirs_decr(f, old_entry=entry, remove_variant=not entry.added)
             if not entry.p2_info:
                 self.copymap.pop(f, None)
             entry.set_untracked()
             self._refresh_entry(f, entry)
             return True
 
     def set_clean(self, filename, mode, size, mtime):
         """mark a file as back to a clean state"""
         entry = self[filename]
         size = size & rangemask
         entry.set_clean(mode, size, mtime)
         self._refresh_entry(filename, entry)
         self.copymap.pop(filename, None)
 
     def set_possibly_dirty(self, filename):
         """record that the current state of the file on disk is unknown"""
         entry = self[filename]
         entry.set_possibly_dirty()
         self._refresh_entry(filename, entry)
 
     def _refresh_entry(self, f, entry):
         """record updated state of an entry"""
         if not entry.any_tracked:
             self._map.pop(f, None)
 
     def _drop_entry(self, f):
         """remove any entry for file f
 
         This should also drop associated copy information
 
         The fact we actually need to drop it is the responsability of the caller"""
         self._map.pop(f, None)
         self.copymap.pop(f, None)
 
 
 if rustmod is not None:
 
     class dirstatemap(_dirstatemapcommon):
 
         ### Core data storage and access
 
         @propertycache
         def _map(self):
             """
             Fills the Dirstatemap when called.
             """
             # ignore HG_PENDING because identity is used only for writing
             self._set_identity()
 
             testing.wait_on_cfg(self._ui, b'dirstate.pre-read-file')
             if self._use_dirstate_v2:
-                self.docket  # load the data if needed
-                inode = (
-                    self.identity.stat.st_ino
-                    if self.identity is not None
-                    and self.identity.stat is not None
-                    else None
-                )
-                testing.wait_on_cfg(self._ui, b'dirstate.post-docket-read-file')
-                if not self.docket.uuid:
-                    data = b''
-                    self._map = rustmod.DirstateMap.new_empty()
+                try:
+                    self.docket
+                except error.CorruptedDirstate as e:
+                    # fall back to dirstate-v1 if we fail to read v2
+                    parents = self._v1_map(e)
                 else:
-                    data = self._read_v2_data()
-                    self._map = rustmod.DirstateMap.new_v2(
-                        data,
-                        self.docket.data_size,
-                        self.docket.tree_metadata,
-                        self.docket.uuid,
-                        inode,
+                    parents = self.docket.parents
+                    inode = (
+                        self.identity.stat.st_ino
+                        if self.identity is not None
+                        and self.identity.stat is not None
+                        else None
+                    )
+                    testing.wait_on_cfg(
+                        self._ui, b'dirstate.post-docket-read-file'
                     )
-                parents = self.docket.parents
+                    if not self.docket.uuid:
+                        data = b''
+                        self._map = rustmod.DirstateMap.new_empty()
+                    else:
+                        data = self._read_v2_data()
+                        self._map = rustmod.DirstateMap.new_v2(
+                            data,
+                            self.docket.data_size,
+                            self.docket.tree_metadata,
+                            self.docket.uuid,
+                            inode,
+                        )
+                    parents = self.docket.parents
             else:
-                self._set_identity()
-                inode = (
-                    self.identity.stat.st_ino
-                    if self.identity is not None
-                    and self.identity.stat is not None
-                    else None
-                )
-                self._map, parents = rustmod.DirstateMap.new_v1(
-                    self._readdirstatefile(), inode
-                )
+                parents = self._v1_map()
 
             if parents and not self._dirtyparents:
                 self.setparents(*parents)
 
             self.__contains__ = self._map.__contains__
             self.__getitem__ = self._map.__getitem__
             self.get = self._map.get
             return self._map
 
+        def _v1_map(self, from_v2_exception=None):
+            self._set_identity()
+            inode = (
+                self.identity.stat.st_ino
+                if self.identity is not None and self.identity.stat is not None
+                else None
+            )
+            try:
+                self._map, parents = rustmod.DirstateMap.new_v1(
+                    self._readdirstatefile(), inode
+                )
+            except OSError as e:
+                if from_v2_exception is not None:
+                    raise e from from_v2_exception
+                raise
+            return parents
+
         @property
         def copymap(self):
             return self._map.copymap()
 
         def debug_iter(self, all):
             """
             Return an iterator of (filename, state, mode, size, mtime) tuples
 
             `all`: also include with `state == b' '` dirstate tree nodes that
             don't have an associated `DirstateItem`.
 
             """
             return self._map.debug_iter(all)
 
         def clear(self):
             self._map.clear()
             self.setparents(
                 self._nodeconstants.nullid, self._nodeconstants.nullid
             )
             util.clearcachedproperty(self, b"_dirs")
             util.clearcachedproperty(self, b"_alldirs")
             util.clearcachedproperty(self, b"dirfoldmap")
 
         def items(self):
             return self._map.items()
 
         # forward for python2,3 compat
         iteritems = items
 
         def keys(self):
             return iter(self._map)
 
         ### reading/setting parents
 
         def setparents(self, p1, p2, fold_p2=False):
             self._parents = (p1, p2)
             self._dirtyparents = True
             copies = {}
             if fold_p2:
                 copies = self._map.setparents_fixup()
             return copies
 
         ### disk interaction
 
         @propertycache
         def identity(self):
             self._map
             return self.identity
 
         def write(self, tr, st):
             if not self._use_dirstate_v2:
                 p1, p2 = self.parents()
                 packed = self._map.write_v1(p1, p2)
                 st.write(packed)
                 st.close()
                 self._dirtyparents = False
                 return
 
+            write_mode = self._write_mode
+            try:
+                docket = self.docket
+            except error.CorruptedDirstate:
+                # fall back to dirstate-v1 if we fail to parse v2
+                docket = None
+
             # We can only append to an existing data file if there is one
-            write_mode = self._write_mode
-            if self.docket.uuid is None:
+            if docket is None or docket.uuid is None:
                 write_mode = WRITE_MODE_FORCE_NEW
             packed, meta, append = self._map.write_v2(write_mode)
             if append:
                 docket = self.docket
                 data_filename = docket.data_filename()
                 # We mark it for backup to make sure a future `hg rollback` (or
                 # `hg recover`?) call find the data it needs to restore a
                 # working repository.
                 #
                 # The backup can use a hardlink because the format is resistant
                 # to trailing "dead" data.
                 if tr is not None:
                     tr.addbackup(data_filename, location=b'plain')
                 with self._opener(data_filename, b'r+b') as fp:
                     fp.seek(docket.data_size)
                     assert fp.tell() == docket.data_size
                     written = fp.write(packed)
                     if written is not None:  # py2 may return None
                         assert written == len(packed), (written, len(packed))
                 docket.data_size += len(packed)
                 docket.parents = self.parents()
                 docket.tree_metadata = meta
                 st.write(docket.serialize())
                 st.close()
             else:
                 self.write_v2_no_append(tr, st, meta, packed)
             # Reload from the newly-written file
             util.clearcachedproperty(self, b"_map")
             self._dirtyparents = False
 
         ### code related to maintaining and accessing "extra" property
         # (e.g. "has_dir")
 
         @propertycache
         def filefoldmap(self):
             """Returns a dictionary mapping normalized case paths to their
             non-normalized versions.
             """
             return self._map.filefoldmapasdict()
 
         def hastrackeddir(self, d):
             return self._map.hastrackeddir(d)
 
         def hasdir(self, d):
             return self._map.hasdir(d)
 
         @propertycache
         def dirfoldmap(self):
             f = {}
             normcase = util.normcase
             for name in self._map.tracked_dirs():
                 f[normcase(name)] = name
             return f
 
         ### code related to manipulation of entries and copy-sources
 
         def set_tracked(self, f):
             return self._map.set_tracked(f)
 
         def set_untracked(self, f):
             return self._map.set_untracked(f)
 
         def set_clean(self, filename, mode, size, mtime):
             self._map.set_clean(filename, mode, size, mtime)
 
         def set_possibly_dirty(self, f):
             self._map.set_possibly_dirty(f)
 
         def reset_state(
             self,
             filename,
             wc_tracked=False,
             p1_tracked=False,
             p2_info=False,
             has_meaningful_mtime=True,
             parentfiledata=None,
         ):
             return self._map.reset_state(
                 filename,
                 wc_tracked,
                 p1_tracked,
                 p2_info,
                 has_meaningful_mtime,
                 parentfiledata,
             )

mercurial/error.py

@@ -1,674 +1,681 b''
 # error.py - Mercurial exceptions
 #
 # Copyright 2005-2008 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Mercurial exceptions.
 
 This allows us to catch exceptions at higher levels without forcing
 imports.
 """
 
 
 import difflib
 
 # Do not import anything but pycompat here, please
 from . import pycompat
 
 if pycompat.TYPE_CHECKING:
     from typing import (
         Any,
         AnyStr,
         Iterable,
         List,
         Optional,
         Sequence,
         Union,
     )
 
 
 def _tobytes(exc):
     # type: (...) -> bytes
     """Byte-stringify exception in the same way as BaseException_str()"""
     if not exc.args:
         return b''
     if len(exc.args) == 1:
         return pycompat.bytestr(exc.args[0])
     return b'(%s)' % b', '.join(b"'%s'" % pycompat.bytestr(a) for a in exc.args)
 
 
 class Hint:
     """Mix-in to provide a hint of an error
 
     This should come first in the inheritance list to consume a hint and
     pass remaining arguments to the exception class.
     """
 
     def __init__(self, *args, **kw):
         self.hint = kw.pop('hint', None)  # type: Optional[bytes]
         super(Hint, self).__init__(*args, **kw)
 
 
 class Error(Hint, Exception):
     """Base class for Mercurial errors."""
 
     coarse_exit_code = None
     detailed_exit_code = None
 
     def __init__(self, message, hint=None):
         # type: (bytes, Optional[bytes]) -> None
         self.message = message
         self.hint = hint
         # Pass the message into the Exception constructor to help extensions
         # that look for exc.args[0].
         Exception.__init__(self, message)
 
     def __bytes__(self):
         return self.message
 
     def __str__(self):
         # type: () -> str
         # the output would be unreadable if the message was translated,
         # but do not replace it with encoding.strfromlocal(), which
         # may raise another exception.
         return pycompat.sysstr(self.__bytes__())
 
     def format(self):
         # type: () -> bytes
         from .i18n import _
 
         message = _(b"abort: %s\n") % self.message
         if self.hint:
             message += _(b"(%s)\n") % self.hint
         return message
 
 
 class Abort(Error):
     """Raised if a command needs to print an error and exit."""
 
 
 class StorageError(Error):
     """Raised when an error occurs in a storage layer.
 
     Usually subclassed by a storage-specific exception.
     """
 
     detailed_exit_code = 50
 
 
 class RevlogError(StorageError):
     pass
 
 
 class SidedataHashError(RevlogError):
     def __init__(self, key, expected, got):
         # type: (int, bytes, bytes) -> None
         self.hint = None
         self.sidedatakey = key
         self.expecteddigest = expected
         self.actualdigest = got
 
 
 class FilteredIndexError(IndexError):
     __bytes__ = _tobytes
 
 
 class LookupError(RevlogError, KeyError):
     def __init__(self, name, index, message):
         # type: (bytes, bytes, bytes) -> None
         self.name = name
         self.index = index
         # this can't be called 'message' because at least some installs of
         # Python 2.6+ complain about the 'message' property being deprecated
         self.lookupmessage = message
         if isinstance(name, bytes) and len(name) == 20:
             from .node import hex
 
             name = hex(name)
         # if name is a binary node, it can be None
         RevlogError.__init__(
             self, b'%s@%s: %s' % (index, pycompat.bytestr(name), message)
         )
 
     def __bytes__(self):
         return RevlogError.__bytes__(self)
 
     def __str__(self):
         return RevlogError.__str__(self)
 
 
 class AmbiguousPrefixLookupError(LookupError):
     pass
 
 
 class FilteredLookupError(LookupError):
     pass
 
 
 class ManifestLookupError(LookupError):
     pass
 
 
 class CommandError(Exception):
     """Exception raised on errors in parsing the command line."""
 
     def __init__(self, command, message):
         # type: (Optional[bytes], bytes) -> None
         self.command = command
         self.message = message
         super(CommandError, self).__init__()
 
     __bytes__ = _tobytes
 
 
 class UnknownCommand(Exception):
     """Exception raised if command is not in the command table."""
 
     def __init__(self, command, all_commands=None):
         # type: (bytes, Optional[List[bytes]]) -> None
         self.command = command
         self.all_commands = all_commands
         super(UnknownCommand, self).__init__()
 
     __bytes__ = _tobytes
 
 
 class AmbiguousCommand(Exception):
     """Exception raised if command shortcut matches more than one command."""
 
     def __init__(self, prefix, matches):
         # type: (bytes, List[bytes]) -> None
         self.prefix = prefix
         self.matches = matches
         super(AmbiguousCommand, self).__init__()
 
     __bytes__ = _tobytes
 
 
 class WorkerError(Exception):
     """Exception raised when a worker process dies."""
 
     def __init__(self, status_code):
         # type: (int) -> None
         self.status_code = status_code
         # Pass status code to superclass just so it becomes part of __bytes__
         super(WorkerError, self).__init__(status_code)
 
     __bytes__ = _tobytes
 
 
 class InterventionRequired(Abort):
     """Exception raised when a command requires human intervention."""
 
     coarse_exit_code = 1
     detailed_exit_code = 240
 
     def format(self):
         # type: () -> bytes
         from .i18n import _
 
         message = _(b"%s\n") % self.message
         if self.hint:
             message += _(b"(%s)\n") % self.hint
         return message
 
 
 class ConflictResolutionRequired(InterventionRequired):
     """Exception raised when a continuable command required merge conflict resolution."""
 
     def __init__(self, opname):
         # type: (bytes) -> None
         from .i18n import _
 
         self.opname = opname
         InterventionRequired.__init__(
             self,
             _(
                 b"unresolved conflicts (see 'hg resolve', then 'hg %s --continue')"
             )
             % opname,
         )
 
 
 class InputError(Abort):
     """Indicates that the user made an error in their input.
 
     Examples: Invalid command, invalid flags, invalid revision.
     """
 
     detailed_exit_code = 10
 
 
 class StateError(Abort):
     """Indicates that the operation might work if retried in a different state.
 
     Examples: Unresolved merge conflicts, unfinished operations.
     """
 
     detailed_exit_code = 20
 
 
 class CanceledError(Abort):
     """Indicates that the user canceled the operation.
 
     Examples: Close commit editor with error status, quit chistedit.
     """
 
     detailed_exit_code = 250
 
 
 class SecurityError(Abort):
     """Indicates that some aspect of security failed.
 
     Examples: Bad server credentials, expired local credentials for network
     filesystem, mismatched GPG signature, DoS protection.
     """
 
     detailed_exit_code = 150
 
 
 class HookLoadError(Abort):
     """raised when loading a hook fails, aborting an operation
 
     Exists to allow more specialized catching."""
 
 
 class HookAbort(Abort):
     """raised when a validation hook fails, aborting an operation
 
     Exists to allow more specialized catching."""
 
     detailed_exit_code = 40
 
 
 class ConfigError(Abort):
     """Exception raised when parsing config files"""
 
     detailed_exit_code = 30
 
     def __init__(self, message, location=None, hint=None):
         # type: (bytes, Optional[bytes], Optional[bytes]) -> None
         super(ConfigError, self).__init__(message, hint=hint)
         self.location = location
 
     def format(self):
         # type: () -> bytes
         from .i18n import _
 
         if self.location is not None:
             message = _(b"config error at %s: %s\n") % (
                 pycompat.bytestr(self.location),
                 self.message,
             )
         else:
             message = _(b"config error: %s\n") % self.message
         if self.hint:
             message += _(b"(%s)\n") % self.hint
         return message
 
 
 class UpdateAbort(Abort):
     """Raised when an update is aborted for destination issue"""
 
 
 class MergeDestAbort(Abort):
     """Raised when an update is aborted for destination issues"""
 
 
 class NoMergeDestAbort(MergeDestAbort):
     """Raised when an update is aborted because there is nothing to merge"""
 
 
 class ManyMergeDestAbort(MergeDestAbort):
     """Raised when an update is aborted because destination is ambiguous"""
 
 
 class ResponseExpected(Abort):
     """Raised when an EOF is received for a prompt"""
 
     def __init__(self):
         from .i18n import _
 
         Abort.__init__(self, _(b'response expected'))
 
 
 class RemoteError(Abort):
     """Exception raised when interacting with a remote repo fails"""
 
     detailed_exit_code = 100
 
 
 class OutOfBandError(RemoteError):
     """Exception raised when a remote repo reports failure"""
 
     def __init__(self, message=None, hint=None):
         # type: (Optional[bytes], Optional[bytes]) -> None
         from .i18n import _
 
         if message:
             # Abort.format() adds a trailing newline
             message = _(b"remote error:\n%s") % message.rstrip(b'\n')
         else:
             message = _(b"remote error")
         super(OutOfBandError, self).__init__(message, hint=hint)
 
 
 class ParseError(Abort):
     """Raised when parsing config files and {rev,file}sets (msg[, pos])"""
 
     detailed_exit_code = 10
 
     def __init__(self, message, location=None, hint=None):
         # type: (bytes, Optional[Union[bytes, int]], Optional[bytes]) -> None
         super(ParseError, self).__init__(message, hint=hint)
         self.location = location
 
     def format(self):
         # type: () -> bytes
         from .i18n import _
 
         if self.location is not None:
             message = _(b"hg: parse error at %s: %s\n") % (
                 pycompat.bytestr(self.location),
                 self.message,
             )
         else:
             message = _(b"hg: parse error: %s\n") % self.message
         if self.hint:
             message += _(b"(%s)\n") % self.hint
         return message
 
 
 class PatchError(Exception):
     __bytes__ = _tobytes
 
 
 class PatchParseError(PatchError):
     __bytes__ = _tobytes
 
 
 class PatchApplicationError(PatchError):
     __bytes__ = _tobytes
 
 
 def getsimilar(symbols, value):
     # type: (Iterable[bytes], bytes) -> List[bytes]
     sim = lambda x: difflib.SequenceMatcher(None, value, x).ratio()
     # The cutoff for similarity here is pretty arbitrary. It should
     # probably be investigated and tweaked.
     return [s for s in symbols if sim(s) > 0.6]
 
 
 def similarity_hint(similar):
     # type: (List[bytes]) -> Optional[bytes]
     from .i18n import _
 
     if len(similar) == 1:
         return _(b"did you mean %s?") % similar[0]
     elif similar:
         ss = b", ".join(sorted(similar))
         return _(b"did you mean one of %s?") % ss
     else:
         return None
 
 
 class UnknownIdentifier(ParseError):
     """Exception raised when a {rev,file}set references an unknown identifier"""
 
     def __init__(self, function, symbols):
         # type: (bytes, Iterable[bytes]) -> None
         from .i18n import _
 
         similar = getsimilar(symbols, function)
         hint = similarity_hint(similar)
 
         ParseError.__init__(
             self, _(b"unknown identifier: %s") % function, hint=hint
         )
 
 
 class RepoError(Hint, Exception):
     __bytes__ = _tobytes
 
 
 class RepoLookupError(RepoError):
     pass
 
 
 class FilteredRepoLookupError(RepoLookupError):
     pass
 
 
 class CapabilityError(RepoError):
     pass
 
 
 class RequirementError(RepoError):
     """Exception raised if .hg/requires has an unknown entry."""
 
 
 class StdioError(IOError):
     """Raised if I/O to stdout or stderr fails"""
 
     def __init__(self, err):
         # type: (IOError) -> None
         IOError.__init__(self, err.errno, err.strerror)
 
     # no __bytes__() because error message is derived from the standard IOError
 
 
 class UnsupportedMergeRecords(Abort):
     def __init__(self, recordtypes):
         # type: (Iterable[bytes]) -> None
         from .i18n import _
 
         self.recordtypes = sorted(recordtypes)
         s = b' '.join(self.recordtypes)
         Abort.__init__(
             self,
             _(b'unsupported merge state records: %s') % s,
             hint=_(
                 b'see https://mercurial-scm.org/wiki/MergeStateRecords for '
                 b'more information'
             ),
         )
 
 
 class UnknownVersion(Abort):
     """generic exception for aborting from an encounter with an unknown version"""
 
     def __init__(self, msg, hint=None, version=None):
         # type: (bytes, Optional[bytes], Optional[bytes]) -> None
         self.version = version
         super(UnknownVersion, self).__init__(msg, hint=hint)
 
 
 class LockError(IOError):
     def __init__(self, errno, strerror, filename, desc):
         # TODO: figure out if this should be bytes or str
         # _type: (int, str, str, bytes) -> None
         IOError.__init__(self, errno, strerror, filename)
         self.desc = desc
 
     # no __bytes__() because error message is derived from the standard IOError
 
 
 class LockHeld(LockError):
     def __init__(self, errno, filename, desc, locker):
         LockError.__init__(self, errno, b'Lock held', filename, desc)
         self.locker = locker
 
 
 class LockUnavailable(LockError):
     pass
 
 
 # LockError is for errors while acquiring the lock -- this is unrelated
 class LockInheritanceContractViolation(RuntimeError):
     __bytes__ = _tobytes
 
 
 class ResponseError(Exception):
     """Raised to print an error with part of output and exit."""
 
     __bytes__ = _tobytes
 
 
 # derived from KeyboardInterrupt to simplify some breakout code
 class SignalInterrupt(KeyboardInterrupt):
     """Exception raised on SIGTERM and SIGHUP."""
 
 
 class SignatureError(Exception):
     __bytes__ = _tobytes
 
 
 class PushRaced(RuntimeError):
     """An exception raised during unbundling that indicate a push race"""
 
     __bytes__ = _tobytes
 
 
 class ProgrammingError(Hint, RuntimeError):
     """Raised if a mercurial (core or extension) developer made a mistake"""
 
     def __init__(self, msg, *args, **kwargs):
         # type: (AnyStr, Any, Any) -> None
         # On Python 3, turn the message back into a string since this is
         # an internal-only error that won't be printed except in a
         # stack traces.
         msg = pycompat.sysstr(msg)
         super(ProgrammingError, self).__init__(msg, *args, **kwargs)
 
     __bytes__ = _tobytes
 
 
 class WdirUnsupported(Exception):
     """An exception which is raised when 'wdir()' is not supported"""
 
     __bytes__ = _tobytes
 
 
 # bundle2 related errors
 class BundleValueError(ValueError):
     """error raised when bundle2 cannot be processed"""
 
     __bytes__ = _tobytes
 
 
 class BundleUnknownFeatureError(BundleValueError):
     def __init__(self, parttype=None, params=(), values=()):
         self.parttype = parttype
         self.params = params
         self.values = values
         if self.parttype is None:
             msg = b'Stream Parameter'
         else:
             msg = parttype
         entries = self.params
         if self.params and self.values:
             assert len(self.params) == len(self.values)
             entries = []
             for idx, par in enumerate(self.params):
                 val = self.values[idx]
                 if val is None:
                     entries.append(val)
                 else:
                     entries.append(b"%s=%r" % (par, pycompat.maybebytestr(val)))
         if entries:
             msg = b'%s - %s' % (msg, b', '.join(entries))
         ValueError.__init__(self, msg)  # TODO: convert to str?
 
 
 class ReadOnlyPartError(RuntimeError):
     """error raised when code tries to alter a part being generated"""
 
     __bytes__ = _tobytes
 
 
 class PushkeyFailed(Abort):
     """error raised when a pushkey part failed to update a value"""
 
     def __init__(
         self, partid, namespace=None, key=None, new=None, old=None, ret=None
     ):
         self.partid = partid
         self.namespace = namespace
         self.key = key
         self.new = new
         self.old = old
         self.ret = ret
         # no i18n expected to be processed into a better message
         Abort.__init__(
             self, b'failed to update value for "%s/%s"' % (namespace, key)
         )
 
 
 class CensoredNodeError(StorageError):
     """error raised when content verification fails on a censored node
 
     Also contains the tombstone data substituted for the uncensored data.
     """
 
     def __init__(self, filename, node, tombstone):
         # type: (bytes, bytes, bytes) -> None
         from .node import short
 
         StorageError.__init__(self, b'%s:%s' % (filename, short(node)))
         self.tombstone = tombstone
 
 
 class CensoredBaseError(StorageError):
     """error raised when a delta is rejected because its base is censored
 
     A delta based on a censored revision must be formed as single patch
     operation which replaces the entire base with new content. This ensures
     the delta may be applied by clones which have not censored the base.
     """
 
 
 class InvalidBundleSpecification(Exception):
     """error raised when a bundle specification is invalid.
 
     This is used for syntax errors as opposed to support errors.
     """
 
     __bytes__ = _tobytes
 
 
 class UnsupportedBundleSpecification(Exception):
     """error raised when a bundle specification is not supported."""
 
     __bytes__ = _tobytes
 
 
 class CorruptedState(Exception):
     """error raised when a command is not able to read its state from file"""
 
     __bytes__ = _tobytes
 
 
+class CorruptedDirstate(Exception):
+    """error raised the dirstate appears corrupted on-disk. It may be due to
+    a dirstate version mismatch (i.e. expecting v2 and finding v1 on disk)."""
+
+    __bytes__ = _tobytes
+
+
 class PeerTransportError(Abort):
     """Transport-level I/O error when communicating with a peer repo."""
 
 
 class InMemoryMergeConflictsError(Exception):
     """Exception raised when merge conflicts arose during an in-memory merge."""
 
     __bytes__ = _tobytes
 
 
 class WireprotoCommandError(Exception):
     """Represents an error during execution of a wire protocol command.
 
     Should only be thrown by wire protocol version 2 commands.
 
     The error is a formatter string and an optional iterable of arguments.
     """
 
     def __init__(self, message, args=None):
         # type: (bytes, Optional[Sequence[bytes]]) -> None
         self.message = message
         self.messageargs = args

rust/hg-core/src/repo.rs

@@ -1,745 +1,782 b''
 use crate::changelog::Changelog;
 use crate::config::{Config, ConfigError, ConfigParseError};
 use crate::dirstate::DirstateParents;
 use crate::dirstate_tree::dirstate_map::DirstateMapWriteMode;
 use crate::dirstate_tree::on_disk::Docket as DirstateDocket;
 use crate::dirstate_tree::owning::OwningDirstateMap;
 use crate::errors::HgResultExt;
 use crate::errors::{HgError, IoResultExt};
 use crate::lock::{try_with_lock_no_wait, LockError};
 use crate::manifest::{Manifest, Manifestlog};
 use crate::revlog::filelog::Filelog;
 use crate::revlog::RevlogError;
 use crate::utils::debug::debug_wait_for_file_or_print;
 use crate::utils::files::get_path_from_bytes;
 use crate::utils::hg_path::HgPath;
 use crate::utils::SliceExt;
 use crate::vfs::{is_dir, is_file, Vfs};
 use crate::{requirements, NodePrefix};
 use crate::{DirstateError, Revision};
 use std::cell::{Ref, RefCell, RefMut};
 use std::collections::HashSet;
 use std::io::Seek;
 use std::io::SeekFrom;
 use std::io::Write as IoWrite;
 use std::path::{Path, PathBuf};
 
 const V2_MAX_READ_ATTEMPTS: usize = 5;
 
 type DirstateMapIdentity = (Option<u64>, Option<Vec<u8>>, usize);
 
 /// A repository on disk
 pub struct Repo {
     working_directory: PathBuf,
     dot_hg: PathBuf,
     store: PathBuf,
     requirements: HashSet<String>,
     config: Config,
     dirstate_parents: LazyCell<DirstateParents>,
     dirstate_map: LazyCell<OwningDirstateMap>,
     changelog: LazyCell<Changelog>,
     manifestlog: LazyCell<Manifestlog>,
 }
 
 #[derive(Debug, derive_more::From)]
 pub enum RepoError {
     NotFound {
         at: PathBuf,
     },
     #[from]
     ConfigParseError(ConfigParseError),
     #[from]
     Other(HgError),
 }
 
 impl From<ConfigError> for RepoError {
     fn from(error: ConfigError) -> Self {
         match error {
             ConfigError::Parse(error) => error.into(),
             ConfigError::Other(error) => error.into(),
         }
     }
 }
 
 impl Repo {
     /// tries to find nearest repository root in current working directory or
     /// its ancestors
     pub fn find_repo_root() -> Result<PathBuf, RepoError> {
         let current_directory = crate::utils::current_dir()?;
         // ancestors() is inclusive: it first yields `current_directory`
         // as-is.
         for ancestor in current_directory.ancestors() {
             if is_dir(ancestor.join(".hg"))? {
                 return Ok(ancestor.to_path_buf());
             }
         }
         Err(RepoError::NotFound {
             at: current_directory,
         })
     }
 
     /// Find a repository, either at the given path (which must contain a `.hg`
     /// sub-directory) or by searching the current directory and its
     /// ancestors.
     ///
     /// A method with two very different "modes" like this usually a code smell
     /// to make two methods instead, but in this case an `Option` is what rhg
     /// sub-commands get from Clap for the `-R` / `--repository` CLI argument.
     /// Having two methods would just move that `if` to almost all callers.
     pub fn find(
         config: &Config,
         explicit_path: Option<PathBuf>,
     ) -> Result<Self, RepoError> {
         if let Some(root) = explicit_path {
             if is_dir(root.join(".hg"))? {
                 Self::new_at_path(root, config)
             } else if is_file(&root)? {
                 Err(HgError::unsupported("bundle repository").into())
             } else {
                 Err(RepoError::NotFound { at: root })
             }
         } else {
             let root = Self::find_repo_root()?;
             Self::new_at_path(root, config)
         }
     }
 
     /// To be called after checking that `.hg` is a sub-directory
     fn new_at_path(
         working_directory: PathBuf,
         config: &Config,
     ) -> Result<Self, RepoError> {
         let dot_hg = working_directory.join(".hg");
 
         let mut repo_config_files =
             vec![dot_hg.join("hgrc"), dot_hg.join("hgrc-not-shared")];
 
         let hg_vfs = Vfs { base: &dot_hg };
         let mut reqs = requirements::load_if_exists(hg_vfs)?;
         let relative =
             reqs.contains(requirements::RELATIVE_SHARED_REQUIREMENT);
         let shared =
             reqs.contains(requirements::SHARED_REQUIREMENT) || relative;
 
         // From `mercurial/localrepo.py`:
         //
         // if .hg/requires contains the sharesafe requirement, it means
         // there exists a `.hg/store/requires` too and we should read it
         // NOTE: presence of SHARESAFE_REQUIREMENT imply that store requirement
         // is present. We never write SHARESAFE_REQUIREMENT for a repo if store
         // is not present, refer checkrequirementscompat() for that
         //
         // However, if SHARESAFE_REQUIREMENT is not present, it means that the
         // repository was shared the old way. We check the share source
         // .hg/requires for SHARESAFE_REQUIREMENT to detect whether the
         // current repository needs to be reshared
         let share_safe = reqs.contains(requirements::SHARESAFE_REQUIREMENT);
 
         let store_path;
         if !shared {
             store_path = dot_hg.join("store");
         } else {
             let bytes = hg_vfs.read("sharedpath")?;
             let mut shared_path =
                 get_path_from_bytes(bytes.trim_end_matches(|b| b == b'\n'))
                     .to_owned();
             if relative {
                 shared_path = dot_hg.join(shared_path)
             }
             if !is_dir(&shared_path)? {
                 return Err(HgError::corrupted(format!(
                     ".hg/sharedpath points to nonexistent directory {}",
                     shared_path.display()
                 ))
                 .into());
             }
 
             store_path = shared_path.join("store");
 
             let source_is_share_safe =
                 requirements::load(Vfs { base: &shared_path })?
                     .contains(requirements::SHARESAFE_REQUIREMENT);
 
             if share_safe != source_is_share_safe {
                 return Err(HgError::unsupported("share-safe mismatch").into());
             }
 
             if share_safe {
                 repo_config_files.insert(0, shared_path.join("hgrc"))
             }
         }
         if share_safe {
             reqs.extend(requirements::load(Vfs { base: &store_path })?);
         }
 
         let repo_config = if std::env::var_os("HGRCSKIPREPO").is_none() {
             config.combine_with_repo(&repo_config_files)?
         } else {
             config.clone()
         };
 
         let repo = Self {
             requirements: reqs,
             working_directory,
             store: store_path,
             dot_hg,
             config: repo_config,
             dirstate_parents: LazyCell::new(),
             dirstate_map: LazyCell::new(),
             changelog: LazyCell::new(),
             manifestlog: LazyCell::new(),
         };
 
         requirements::check(&repo)?;
 
         Ok(repo)
     }
 
     pub fn working_directory_path(&self) -> &Path {
         &self.working_directory
     }
 
     pub fn requirements(&self) -> &HashSet<String> {
         &self.requirements
     }
 
     pub fn config(&self) -> &Config {
         &self.config
     }
 
     /// For accessing repository files (in `.hg`), except for the store
     /// (`.hg/store`).
     pub fn hg_vfs(&self) -> Vfs<'_> {
         Vfs { base: &self.dot_hg }
     }
 
     /// For accessing repository store files (in `.hg/store`)
     pub fn store_vfs(&self) -> Vfs<'_> {
         Vfs { base: &self.store }
     }
 
     /// For accessing the working copy
     pub fn working_directory_vfs(&self) -> Vfs<'_> {
         Vfs {
             base: &self.working_directory,
         }
     }
 
     pub fn try_with_wlock_no_wait<R>(
         &self,
         f: impl FnOnce() -> R,
     ) -> Result<R, LockError> {
         try_with_lock_no_wait(self.hg_vfs(), "wlock", f)
     }
 
     /// Whether this repo should use dirstate-v2.
     /// The presence of `dirstate-v2` in the requirements does not mean that
     /// the on-disk dirstate is necessarily in version 2. In most cases,
     /// a dirstate-v2 file will indeed be found, but in rare cases (like the
     /// upgrade mechanism being cut short), the on-disk version will be a
     /// v1 file.
-    /// Semantically, having a requirement only means that a client should be
-    /// able to understand the repo *if* it uses the requirement, but not that
-    /// the requirement is actually used.
+    /// Semantically, having a requirement only means that a client cannot
+    /// properly understand or properly update the repo if it lacks the support
+    /// for the required feature, but not that that feature is actually used
+    /// in all occasions.
     pub fn use_dirstate_v2(&self) -> bool {
         self.requirements
             .contains(requirements::DIRSTATE_V2_REQUIREMENT)
     }
 
     pub fn has_sparse(&self) -> bool {
         self.requirements.contains(requirements::SPARSE_REQUIREMENT)
     }
 
     pub fn has_narrow(&self) -> bool {
         self.requirements.contains(requirements::NARROW_REQUIREMENT)
     }
 
     pub fn has_nodemap(&self) -> bool {
         self.requirements
             .contains(requirements::NODEMAP_REQUIREMENT)
     }
 
     fn dirstate_file_contents(&self) -> Result<Vec<u8>, HgError> {
         Ok(self
             .hg_vfs()
             .read("dirstate")
             .io_not_found_as_none()?
             .unwrap_or_default())
     }
 
     fn dirstate_identity(&self) -> Result<Option<u64>, HgError> {
         use std::os::unix::fs::MetadataExt;
         Ok(self
             .hg_vfs()
             .symlink_metadata("dirstate")
             .io_not_found_as_none()?
             .map(|meta| meta.ino()))
     }
 
     pub fn dirstate_parents(&self) -> Result<DirstateParents, HgError> {
         Ok(*self
             .dirstate_parents
             .get_or_init(|| self.read_dirstate_parents())?)
     }
 
     fn read_dirstate_parents(&self) -> Result<DirstateParents, HgError> {
         let dirstate = self.dirstate_file_contents()?;
         let parents = if dirstate.is_empty() {
             DirstateParents::NULL
         } else if self.use_dirstate_v2() {
-            let docket =
-                crate::dirstate_tree::on_disk::read_docket(&dirstate)?;
-            docket.parents()
+            let docket_res =
+                crate::dirstate_tree::on_disk::read_docket(&dirstate);
+            match docket_res {
+                Ok(docket) => docket.parents(),
+                Err(_) => {
+                    log::info!(
+                        "Parsing dirstate docket failed, \
+                        falling back to dirstate-v1"
+                    );
+                    *crate::dirstate::parsers::parse_dirstate_parents(
+                        &dirstate,
+                    )?
+                }
+            }
         } else {
             *crate::dirstate::parsers::parse_dirstate_parents(&dirstate)?
         };
         self.dirstate_parents.set(parents);
         Ok(parents)
     }
 
     /// Returns the information read from the dirstate docket necessary to
     /// check if the data file has been updated/deleted by another process
     /// since we last read the dirstate.
     /// Namely, the inode, data file uuid and the data size.
     fn get_dirstate_data_file_integrity(
         &self,
     ) -> Result<DirstateMapIdentity, HgError> {
         assert!(
             self.use_dirstate_v2(),
             "accessing dirstate data file ID without dirstate-v2"
         );
         // Get the identity before the contents since we could have a race
         // between the two. Having an identity that is too old is fine, but
         // one that is younger than the content change is bad.
         let identity = self.dirstate_identity()?;
         let dirstate = self.dirstate_file_contents()?;
         if dirstate.is_empty() {
             self.dirstate_parents.set(DirstateParents::NULL);
             Ok((identity, None, 0))
         } else {
-            let docket =
-                crate::dirstate_tree::on_disk::read_docket(&dirstate)?;
-            self.dirstate_parents.set(docket.parents());
-            Ok((identity, Some(docket.uuid.to_owned()), docket.data_size()))
+            let docket_res =
+                crate::dirstate_tree::on_disk::read_docket(&dirstate);
+            match docket_res {
+                Ok(docket) => {
+                    self.dirstate_parents.set(docket.parents());
+                    Ok((
+                        identity,
+                        Some(docket.uuid.to_owned()),
+                        docket.data_size(),
+                    ))
+                }
+                Err(_) => {
+                    log::info!(
+                        "Parsing dirstate docket failed, \
+                        falling back to dirstate-v1"
+                    );
+                    let parents =
+                        *crate::dirstate::parsers::parse_dirstate_parents(
+                            &dirstate,
+                        )?;
+                    self.dirstate_parents.set(parents);
+                    Ok((identity, None, 0))
+                }
+            }
         }
     }
 
     fn new_dirstate_map(&self) -> Result<OwningDirstateMap, DirstateError> {
         if self.use_dirstate_v2() {
             // The v2 dirstate is split into a docket and a data file.
             // Since we don't always take the `wlock` to read it
             // (like in `hg status`), it is susceptible to races.
             // A simple retry method should be enough since full rewrites
             // only happen when too much garbage data is present and
             // this race is unlikely.
             let mut tries = 0;
 
             while tries < V2_MAX_READ_ATTEMPTS {
                 tries += 1;
                 match self.read_docket_and_data_file() {
                     Ok(m) => {
                         return Ok(m);
                     }
                     Err(e) => match e {
                         DirstateError::Common(HgError::RaceDetected(
                             context,
                         )) => {
                             log::info!(
                                 "dirstate read race detected {} (retry {}/{})",
                                 context,
                                 tries,
                                 V2_MAX_READ_ATTEMPTS,
                             );
                             continue;
                         }
-                        _ => return Err(e),
+                        _ => {
+                            log::info!(
+                                "Reading dirstate v2 failed, \
+                                falling back to v1"
+                            );
+                            return self.new_dirstate_map_v1();
+                        }
                     },
                 }
             }
             let error = HgError::abort(
                 format!("dirstate read race happened {tries} times in a row"),
                 255,
                 None,
             );
             Err(DirstateError::Common(error))
         } else {
-            debug_wait_for_file_or_print(
-                self.config(),
-                "dirstate.pre-read-file",
-            );
-            let identity = self.dirstate_identity()?;
-            let dirstate_file_contents = self.dirstate_file_contents()?;
-            if dirstate_file_contents.is_empty() {
-                self.dirstate_parents.set(DirstateParents::NULL);
-                Ok(OwningDirstateMap::new_empty(Vec::new()))
-            } else {
-                let (map, parents) = OwningDirstateMap::new_v1(
-                    dirstate_file_contents,
-                    identity,
-                )?;
-                self.dirstate_parents.set(parents);
-                Ok(map)
-            }
+            self.new_dirstate_map_v1()
+        }
+    }
+
+    fn new_dirstate_map_v1(&self) -> Result<OwningDirstateMap, DirstateError> {
+        debug_wait_for_file_or_print(self.config(), "dirstate.pre-read-file");
+        let identity = self.dirstate_identity()?;
+        let dirstate_file_contents = self.dirstate_file_contents()?;
+        if dirstate_file_contents.is_empty() {
+            self.dirstate_parents.set(DirstateParents::NULL);
+            Ok(OwningDirstateMap::new_empty(Vec::new()))
+        } else {
+            let (map, parents) =
+                OwningDirstateMap::new_v1(dirstate_file_contents, identity)?;
+            self.dirstate_parents.set(parents);
+            Ok(map)
         }
     }
 
     fn read_docket_and_data_file(
         &self,
     ) -> Result<OwningDirstateMap, DirstateError> {
         debug_wait_for_file_or_print(self.config(), "dirstate.pre-read-file");
         let dirstate_file_contents = self.dirstate_file_contents()?;
         let identity = self.dirstate_identity()?;
         if dirstate_file_contents.is_empty() {
             self.dirstate_parents.set(DirstateParents::NULL);
             return Ok(OwningDirstateMap::new_empty(Vec::new()));
         }
         let docket = crate::dirstate_tree::on_disk::read_docket(
             &dirstate_file_contents,
         )?;
         debug_wait_for_file_or_print(
             self.config(),
             "dirstate.post-docket-read-file",
         );
         self.dirstate_parents.set(docket.parents());
         let uuid = docket.uuid.to_owned();
         let data_size = docket.data_size();
 
         let context = "between reading dirstate docket and data file";
         let race_error = HgError::RaceDetected(context.into());
         let metadata = docket.tree_metadata();
 
         let mut map = if crate::vfs::is_on_nfs_mount(docket.data_filename()) {
             // Don't mmap on NFS to prevent `SIGBUS` error on deletion
             let contents = self.hg_vfs().read(docket.data_filename());
             let contents = match contents {
                 Ok(c) => c,
                 Err(HgError::IoError { error, context }) => {
                     match error.raw_os_error().expect("real os error") {
                         // 2 = ENOENT, No such file or directory
                         // 116 = ESTALE, Stale NFS file handle
                         //
                         // TODO match on `error.kind()` when
                         // `ErrorKind::StaleNetworkFileHandle` is stable.
                         2 | 116 => {
                             // Race where the data file was deleted right after
                             // we read the docket, try again
                             return Err(race_error.into());
                         }
                         _ => {
                             return Err(
                                 HgError::IoError { error, context }.into()
                             )
                         }
                     }
                 }
                 Err(e) => return Err(e.into()),
             };
             OwningDirstateMap::new_v2(
                 contents, data_size, metadata, uuid, identity,
             )
         } else {
             match self
                 .hg_vfs()
                 .mmap_open(docket.data_filename())
                 .io_not_found_as_none()
             {
                 Ok(Some(data_mmap)) => OwningDirstateMap::new_v2(
                     data_mmap, data_size, metadata, uuid, identity,
                 ),
                 Ok(None) => {
                     // Race where the data file was deleted right after we
                     // read the docket, try again
                     return Err(race_error.into());
                 }
                 Err(e) => return Err(e.into()),
             }
         }?;
 
         let write_mode_config = self
             .config()
             .get_str(b"devel", b"dirstate.v2.data_update_mode")
             .unwrap_or(Some("auto"))
             .unwrap_or("auto"); // don't bother for devel options
         let write_mode = match write_mode_config {
             "auto" => DirstateMapWriteMode::Auto,
             "force-new" => DirstateMapWriteMode::ForceNewDataFile,
             "force-append" => DirstateMapWriteMode::ForceAppend,
             _ => DirstateMapWriteMode::Auto,
         };
 
         map.with_dmap_mut(|m| m.set_write_mode(write_mode));
 
         Ok(map)
     }
 
     pub fn dirstate_map(
         &self,
     ) -> Result<Ref<OwningDirstateMap>, DirstateError> {
         self.dirstate_map.get_or_init(|| self.new_dirstate_map())
     }
 
     pub fn dirstate_map_mut(
         &self,
     ) -> Result<RefMut<OwningDirstateMap>, DirstateError> {
         self.dirstate_map
             .get_mut_or_init(|| self.new_dirstate_map())
     }
 
     fn new_changelog(&self) -> Result<Changelog, HgError> {
         Changelog::open(&self.store_vfs(), self.has_nodemap())
     }
 
     pub fn changelog(&self) -> Result<Ref<Changelog>, HgError> {
         self.changelog.get_or_init(|| self.new_changelog())
     }
 
     pub fn changelog_mut(&self) -> Result<RefMut<Changelog>, HgError> {
         self.changelog.get_mut_or_init(|| self.new_changelog())
     }
 
     fn new_manifestlog(&self) -> Result<Manifestlog, HgError> {
         Manifestlog::open(&self.store_vfs(), self.has_nodemap())
     }
 
     pub fn manifestlog(&self) -> Result<Ref<Manifestlog>, HgError> {
         self.manifestlog.get_or_init(|| self.new_manifestlog())
     }
 
     pub fn manifestlog_mut(&self) -> Result<RefMut<Manifestlog>, HgError> {
         self.manifestlog.get_mut_or_init(|| self.new_manifestlog())
     }
 
     /// Returns the manifest of the *changeset* with the given node ID
     pub fn manifest_for_node(
         &self,
         node: impl Into<NodePrefix>,
     ) -> Result<Manifest, RevlogError> {
         self.manifestlog()?.data_for_node(
             self.changelog()?
                 .data_for_node(node.into())?
                 .manifest_node()?
                 .into(),
         )
     }
 
     /// Returns the manifest of the *changeset* with the given revision number
     pub fn manifest_for_rev(
         &self,
         revision: Revision,
     ) -> Result<Manifest, RevlogError> {
         self.manifestlog()?.data_for_node(
             self.changelog()?
                 .data_for_rev(revision)?
                 .manifest_node()?
                 .into(),
         )
     }
 
     pub fn has_subrepos(&self) -> Result<bool, DirstateError> {
         if let Some(entry) = self.dirstate_map()?.get(HgPath::new(".hgsub"))? {
             Ok(entry.tracked())
         } else {
             Ok(false)
         }
     }
 
     pub fn filelog(&self, path: &HgPath) -> Result<Filelog, HgError> {
         Filelog::open(self, path)
     }
 
     /// Write to disk any updates that were made through `dirstate_map_mut`.
     ///
     /// The "wlock" must be held while calling this.
     /// See for example `try_with_wlock_no_wait`.
     ///
     /// TODO: have a `WritableRepo` type only accessible while holding the
     /// lock?
     pub fn write_dirstate(&self) -> Result<(), DirstateError> {
         let map = self.dirstate_map()?;
         // TODO: Maintain a `DirstateMap::dirty` flag, and return early here if
         // it’s unset
         let parents = self.dirstate_parents()?;
         let (packed_dirstate, old_uuid_to_remove) = if self.use_dirstate_v2() {
             let (identity, uuid, data_size) =
                 self.get_dirstate_data_file_integrity()?;
             let identity_changed = identity != map.old_identity();
             let uuid_changed = uuid.as_deref() != map.old_uuid();
             let data_length_changed = data_size != map.old_data_size();
 
             if identity_changed || uuid_changed || data_length_changed {
                 // If any of identity, uuid or length have changed since
                 // last disk read, don't write.
                 // This is fine because either we're in a command that doesn't
                 // write anything too important (like `hg status`), or we're in
                 // `hg add` and we're supposed to have taken the lock before
                 // reading anyway.
                 //
                 // TODO complain loudly if we've changed anything important
                 // without taking the lock.
                 // (see `hg help config.format.use-dirstate-tracked-hint`)
                 log::debug!(
                     "dirstate has changed since last read, not updating."
                 );
                 return Ok(());
             }
 
             let uuid_opt = map.old_uuid();
             let write_mode = if uuid_opt.is_some() {
                 DirstateMapWriteMode::Auto
             } else {
                 DirstateMapWriteMode::ForceNewDataFile
             };
             let (data, tree_metadata, append, old_data_size) =
                 map.pack_v2(write_mode)?;
 
             // Reuse the uuid, or generate a new one, keeping the old for
             // deletion.
             let (uuid, old_uuid) = match uuid_opt {
                 Some(uuid) => {
                     let as_str = std::str::from_utf8(uuid)
                         .map_err(|_| {
                             HgError::corrupted(
                                 "non-UTF-8 dirstate data file ID",
                             )
                         })?
                         .to_owned();
                     if append {
                         (as_str, None)
                     } else {
                         (DirstateDocket::new_uid(), Some(as_str))
                     }
                 }
                 None => (DirstateDocket::new_uid(), None),
             };
 
             let data_filename = format!("dirstate.{}", uuid);
             let data_filename = self.hg_vfs().join(data_filename);
             let mut options = std::fs::OpenOptions::new();
             options.write(true);
 
             // Why are we not using the O_APPEND flag when appending?
             //
             // - O_APPEND makes it trickier to deal with garbage at the end of
             //   the file, left by a previous uncommitted transaction. By
             //   starting the write at [old_data_size] we make sure we erase
             //   all such garbage.
             //
             // - O_APPEND requires to special-case 0-byte writes, whereas we
             //   don't need that.
             //
             // - Some OSes have bugs in implementation O_APPEND:
             //   revlog.py talks about a Solaris bug, but we also saw some ZFS
             //   bug: https://github.com/openzfs/zfs/pull/3124,
             //   https://github.com/openzfs/zfs/issues/13370
             //
             if !append {
                 log::trace!("creating a new dirstate data file");
                 options.create_new(true);
             } else {
                 log::trace!("appending to the dirstate data file");
             }
 
             let data_size = (|| {
                 // TODO: loop and try another random ID if !append and this
                 // returns `ErrorKind::AlreadyExists`? Collision chance of two
                 // random IDs is one in 2**32
                 let mut file = options.open(&data_filename)?;
                 if append {
                     file.seek(SeekFrom::Start(old_data_size as u64))?;
                 }
                 file.write_all(&data)?;
                 file.flush()?;
                 file.seek(SeekFrom::Current(0))
             })()
             .when_writing_file(&data_filename)?;
 
             let packed_dirstate = DirstateDocket::serialize(
                 parents,
                 tree_metadata,
                 data_size,
                 uuid.as_bytes(),
             )
             .map_err(|_: std::num::TryFromIntError| {
                 HgError::corrupted("overflow in dirstate docket serialization")
             })?;
 
             (packed_dirstate, old_uuid)
         } else {
             let identity = self.dirstate_identity()?;
             if identity != map.old_identity() {
                 // If identity changed since last disk read, don't write.
                 // This is fine because either we're in a command that doesn't
                 // write anything too important (like `hg status`), or we're in
                 // `hg add` and we're supposed to have taken the lock before
                 // reading anyway.
                 //
                 // TODO complain loudly if we've changed anything important
                 // without taking the lock.
                 // (see `hg help config.format.use-dirstate-tracked-hint`)
                 log::debug!(
                     "dirstate has changed since last read, not updating."
                 );
                 return Ok(());
             }
             (map.pack_v1(parents)?, None)
         };
 
         let vfs = self.hg_vfs();
         vfs.atomic_write("dirstate", &packed_dirstate)?;
         if let Some(uuid) = old_uuid_to_remove {
             // Remove the old data file after the new docket pointing to the
             // new data file was written.
             vfs.remove_file(format!("dirstate.{}", uuid))?;
         }
         Ok(())
     }
 }
 
 /// Lazily-initialized component of `Repo` with interior mutability
 ///
 /// This differs from `OnceCell` in that the value can still be "deinitialized"
 /// later by setting its inner `Option` to `None`. It also takes the
 /// initialization function as an argument when the value is requested, not
 /// when the instance is created.
 struct LazyCell<T> {
     value: RefCell<Option<T>>,
 }
 
 impl<T> LazyCell<T> {
     fn new() -> Self {
         Self {
             value: RefCell::new(None),
         }
     }
 
     fn set(&self, value: T) {
         *self.value.borrow_mut() = Some(value)
     }
 
     fn get_or_init<E>(
         &self,
         init: impl Fn() -> Result<T, E>,
     ) -> Result<Ref<T>, E> {
         let mut borrowed = self.value.borrow();
         if borrowed.is_none() {
             drop(borrowed);
             // Only use `borrow_mut` if it is really needed to avoid panic in
             // case there is another outstanding borrow but mutation is not
             // needed.
             *self.value.borrow_mut() = Some(init()?);
             borrowed = self.value.borrow()
         }
         Ok(Ref::map(borrowed, |option| option.as_ref().unwrap()))
     }
 
     fn get_mut_or_init<E>(
         &self,
         init: impl Fn() -> Result<T, E>,
     ) -> Result<RefMut<T>, E> {
         let mut borrowed = self.value.borrow_mut();
         if borrowed.is_none() {
             *borrowed = Some(init()?);
         }
         Ok(RefMut::map(borrowed, |option| option.as_mut().unwrap()))
     }
 }

tests/test-dirstate-version-fallback.t

@@ -1,43 +1,49 b''
   $ cat >> $HGRCPATH << EOF
   > [storage]
   > dirstate-v2.slow-path=allow
   > EOF
 
 Set up a v1 repo
 
   $ hg init repo
   $ cd repo
   $ echo a > a
   $ hg add a
   $ hg commit -m a
   $ hg debugrequires | grep dirstate
   [1]
   $ ls -1 .hg/dirstate*
   .hg/dirstate
 
 Copy v1 dirstate
   $ cp .hg/dirstate $TESTTMP/dirstate-v1-backup
 
 Upgrade it to v2
 
-  $ hg debugupgraderepo -q --config format.use-dirstate-v2=1 --run | grep added
+  $ hg debugupgraderepo -q --config format.use-dirstate-v2=1 --run | egrep 'added:|removed:'
      added: dirstate-v2
   $ hg debugrequires | grep dirstate
   dirstate-v2
   $ ls -1 .hg/dirstate*
   .hg/dirstate
   .hg/dirstate.* (glob)
 
 Manually reset to dirstate v1 to simulate an incomplete dirstate-v2 upgrade
 
   $ rm .hg/dirstate*
   $ cp $TESTTMP/dirstate-v1-backup .hg/dirstate
 
 There should be no errors, but a v2 dirstate should be written back to disk
   $ hg st
-  abort: dirstate-v2 parse error: when reading docket, Expected at least * bytes, got * (glob) (known-bad-output !)
-  [255]
   $ ls -1 .hg/dirstate*
   .hg/dirstate
-  .hg/dirstate.* (glob) (missing-correct-output !)
+  .hg/dirstate.* (glob)
+
+Corrupt the dirstate to see how the errors show up to the user
+  $ echo "I ate your data" > .hg/dirstate
 
+  $ hg st
+  abort: working directory state appears damaged! (no-rhg !)
+  (falling back to dirstate-v1 from v2 also failed) (no-rhg !)
+  abort: Too little data for dirstate. (rhg !)
+  [255]