dirstate-v2: adds two flag to track the presence of some unrecorded files

Right now, we don't record ignored or unknown files in the dirstate. However
the structure would allow it. So we introduce two flags that can be used to
clarify whether all unknown/ignored children are recorded or not. This will
allow for more information to be stored in the future if this end up being
relevant.

Differential Revision: https://phab.mercurial-scm.org/D11682

mercurial/cext/util.h

@@ -40,6 +40,8 static const int dirstate_flag_has_direc
 static const int dirstate_flag_mode_exec_perm = 1 << 6;
 static const int dirstate_flag_mode_is_symlink = 1 << 7;
 static const int dirstate_flag_expected_state_is_modified = 1 << 8;
+static const int dirstate_flag_all_unknown_recorded = 1 << 9;
+static const int dirstate_flag_all_ignored_recorded = 1 << 10;
 
 extern PyTypeObject dirstateItemType;
 #define dirstate_tuple_check(op) (Py_TYPE(op) == &dirstateItemType)

mercurial/dirstateutils/v2.py

@@ -81,6 +81,9 def parse_nodes(map, copy_map, data, sta
     """parse <len> nodes from <data> starting at offset <start>
 
     This is used by parse_dirstate to recursively fill `map` and `copy_map`.
+
+    All directory specific information is ignored and do not need any
+    processing (HAS_DIRECTORY_MTIME, ALL_UNKNOWN_RECORDED, ALL_IGNORED_RECORDED)
     """
     for i in range(len):
         node_start = start + NODE_SIZE * i

mercurial/helptext/internals/dirstate-v2.txt

@@ -384,6 +384,8 Node components are:
     MODE_EXEC_PERM = 1 << 6
     MODE_IS_SYMLINK = 1 << 7
     EXPECTED_STATE_IS_MODIFIED = 1 << 8
+    ALL_UNKNOWN_RECORDED = 1 << 9
+    ALL_IGNORED_RECORDED = 1 << 10
 
   The meaning of each bit is described below.
 
@@ -530,3 +532,29 by enabling it to skip `readdir` in more
     does not need to do the same again.
     It is valid to never set this bit,
     and consider expected metadata ambiguous if it is set.
+
+`ALL_UNKNOWN_RECORDED`
+    If set, all "unknown" children existing on disk (at the time of the last
+    status) have been recorded and the `mtime` associated with
+    `HAS_DIRECTORY_MTIME` can be used for optimization even when "unknown" file
+    are listed.
+
+    Note that the amount recorded "unknown" children can still be zero if None
+    where present.
+
+    Also note that having this flag unset does not imply that no "unknown"
+    children have been recorded. Some might be present, but there is no garantee
+    that is will be all of them.
+
+`ALL_IGNORED_RECORDED`
+    If set, all "ignored" children existing on disk (at the time of the last
+    status) have been recorded and the `mtime` associated with
+    `HAS_DIRECTORY_MTIME` can be used for optimization even when "ignored" file
+    are listed.
+
+    Note that the amount recorded "ignored" children can still be zero if None
+    where present.
+
+    Also note that having this flag unset does not imply that no "ignored"
+    children have been recorded. Some might be present, but there is no garantee
+    that is will be all of them.

mercurial/pure/parsers.py

@@ -54,6 +54,8 DIRSTATE_V2_HAS_FILE_MTIME = 1 << 4
 DIRSTATE_V2_MODE_EXEC_PERM = 1 << 6
 DIRSTATE_V2_MODE_IS_SYMLINK = 1 << 7
 DIRSTATE_V2_EXPECTED_STATE_IS_MODIFIED = 1 << 8
+DIRSTATE_V2_ALL_UNKNOWN_RECORDED = 1 << 9
+DIRSTATE_V2_ALL_IGNORED_RECORDED = 1 << 10
 
 
 @attr.s(slots=True, init=False)
@@ -340,6 +342,9 class DirstateItem(object):
                 flags |= DIRSTATE_V2_MODE_IS_SYMLINK
         if self._mtime is not None:
             flags |= DIRSTATE_V2_HAS_FILE_MTIME
+        # Note: we do not need to do anything regarding
+        # DIRSTATE_V2_ALL_UNKNOWN_RECORDED and DIRSTATE_V2_ALL_IGNORED_RECORDED
+        # since we never set _DIRSTATE_V2_HAS_DIRCTORY_MTIME
         return (flags, self._size or 0, self._mtime or 0)
 
     def v1_state(self):

rust/hg-core/src/dirstate_tree/on_disk.rs

@@ -111,6 +111,8 bitflags! {
         const MODE_EXEC_PERM = 1 << 6;
         const MODE_IS_SYMLINK = 1 << 7;
         const EXPECTED_STATE_IS_MODIFIED = 1 << 8;
+        const ALL_UNKNOWN_RECORDED = 1 << 9;
+        const ALL_IGNORED_RECORDED = 1 << 10;
     }
 }
 
@@ -322,7 +324,11 impl Node {
     pub(super) fn cached_directory_mtime(
         &self,
     ) -> Result<Option<TruncatedTimestamp>, DirstateV2ParseError> {
-        if self.flags().contains(Flags::HAS_DIRECTORY_MTIME) {
+        // For now we do not have code to handle ALL_UNKNOWN_RECORDED, so we
+        // ignore the mtime if the flag is set.
+        if self.flags().contains(Flags::HAS_DIRECTORY_MTIME)
+            && self.flags().contains(Flags::ALL_UNKNOWN_RECORDED)
+        {
             if self.flags().contains(Flags::HAS_FILE_MTIME) {
                 Err(DirstateV2ParseError)
             } else {
@@ -589,7 +595,18 impl Writer<'_, '_> {
                             Node::from_dirstate_entry(entry)
                         }
                         dirstate_map::NodeData::CachedDirectory { mtime } => (
-                            Flags::HAS_DIRECTORY_MTIME,
+                            // we currently never set a mtime if unknown file
+                            // are present.
+                            // So if we have a mtime for a directory, we know
+                            // they are no unknown
+                            // files and we
+                            // blindly set ALL_UNKNOWN_RECORDED.
+                            //
+                            // We never set ALL_IGNORED_RECORDED since we
+                            // don't track that case
+                            // currently.
+                            Flags::HAS_DIRECTORY_MTIME
+                                | Flags::ALL_UNKNOWN_RECORDED,
                             0.into(),
                             (*mtime).into(),
                         ),