rust-nodemap: falling back to C impl as mitigation

This is a mitigation for https://bz.mercurial-scm.org/show_bug.cgi?id=6554

We see sometimes almost all data except the most recent revisions
removed from the persistent nodemap, but we don't know how to
reproduce yet.

This has sadly repercussions beyond just needing to reconstruct the
persistent nodemap: for instance, this automatically filters out
all bookmarks pointing to revisions that the nodemap cannot resolve.
If such filtering happens in a transaction, the update of the
bookmarks file that happens at the end of transaction loses all
bookmarks that have been affected. There may be similar consequences
for other data.

So this is a data loss, something that we have to prevent as soon as
possible.

As a mitigation measure, we will now fallback to the C implementation
in case nodemap lookups failed. This will add some latency, e.g., in
discovery, yet less than disabling the persistent nodemap entirely.

We considered implementing the fallback directly on the Python
side, but `revlog.get_rev()` is not systematically used, there are
also several direct calls to the index method (`self.index.rev()` for
a `revlog` instance). It is therefore more direct to implement the
mitigation in the rust-cpython wrapper.

Differential Revision: https://phab.mercurial-scm.org/D11238

rust/hg-cpython/src/revlog.rs

@@ -1,512 +1,529 b''
 // revlog.rs
 //
 // Copyright 2019-2020 Georges Racinet <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 use crate::{
     cindex,
     utils::{node_from_py_bytes, node_from_py_object},
 };
 use cpython::{
     buffer::{Element, PyBuffer},
     exc::{IndexError, ValueError},
     ObjectProtocol, PyBytes, PyClone, PyDict, PyErr, PyInt, PyModule,
     PyObject, PyResult, PyString, PyTuple, Python, PythonObject, ToPyObject,
 };
 use hg::{
     nodemap::{Block, NodeMapError, NodeTree},
     revlog::{nodemap::NodeMap, NodePrefix, RevlogIndex},
     Revision,
 };
 use std::cell::RefCell;
 
 /// Return a Struct implementing the Graph trait
 pub(crate) fn pyindex_to_graph(
     py: Python,
     index: PyObject,
 ) -> PyResult<cindex::Index> {
     match index.extract::<MixedIndex>(py) {
         Ok(midx) => Ok(midx.clone_cindex(py)),
         Err(_) => cindex::Index::new(py, index),
     }
 }
 
 py_class!(pub class MixedIndex |py| {
     data cindex: RefCell<cindex::Index>;
     data nt: RefCell<Option<NodeTree>>;
     data docket: RefCell<Option<PyObject>>;
     // Holds a reference to the mmap'ed persistent nodemap data
     data mmap: RefCell<Option<PyBuffer>>;
 
     def __new__(_cls, cindex: PyObject) -> PyResult<MixedIndex> {
         Self::new(py, cindex)
     }
 
     /// Compatibility layer used for Python consumers needing access to the C index
     ///
     /// Only use case so far is `scmutil.shortesthexnodeidprefix`,
     /// that may need to build a custom `nodetree`, based on a specified revset.
     /// With a Rust implementation of the nodemap, we will be able to get rid of
     /// this, by exposing our own standalone nodemap class,
     /// ready to accept `MixedIndex`.
     def get_cindex(&self) -> PyResult<PyObject> {
         Ok(self.cindex(py).borrow().inner().clone_ref(py))
     }
 
     // Index API involving nodemap, as defined in mercurial/pure/parsers.py
 
     /// Return Revision if found, raises a bare `error.RevlogError`
     /// in case of ambiguity, same as C version does
-    def get_rev(&self, node: PyBytes) -> PyResult<Option<Revision>> {
+    def get_rev(&self, pynode: PyBytes) -> PyResult<Option<Revision>> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
-        let node = node_from_py_bytes(py, &node)?;
-        nt.find_bin(idx, node.into()).map_err(|e| nodemap_error(py, e))
+        let node = node_from_py_bytes(py, &pynode)?;
+        match nt.find_bin(idx, node.into())
+        {
+            Ok(None) =>
+                // fallback to C implementation, remove once
+                // https://bz.mercurial-scm.org/show_bug.cgi?id=6554
+                // is fixed (a simple backout should do)
+                self.call_cindex(py, "get_rev", &PyTuple::new(py, &[pynode.into_object()]), None)?
+                .extract(py),
+            Ok(Some(rev)) => Ok(Some(rev)),
+            Err(e) => Err(nodemap_error(py, e)),
+        }
     }
 
     /// same as `get_rev()` but raises a bare `error.RevlogError` if node
     /// is not found.
     ///
     /// No need to repeat `node` in the exception, `mercurial/revlog.py`
     /// will catch and rewrap with it
     def rev(&self, node: PyBytes) -> PyResult<Revision> {
         self.get_rev(py, node)?.ok_or_else(|| revlog_error(py))
     }
 
     /// return True if the node exist in the index
     def has_node(&self, node: PyBytes) -> PyResult<bool> {
         self.get_rev(py, node).map(|opt| opt.is_some())
     }
 
     /// find length of shortest hex nodeid of a binary ID
     def shortest(&self, node: PyBytes) -> PyResult<usize> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
         match nt.unique_prefix_len_node(idx, &node_from_py_bytes(py, &node)?)
         {
             Ok(Some(l)) => Ok(l),
             Ok(None) => Err(revlog_error(py)),
             Err(e) => Err(nodemap_error(py, e)),
         }
     }
 
-    def partialmatch(&self, node: PyObject) -> PyResult<Option<PyBytes>> {
+    def partialmatch(&self, pynode: PyObject) -> PyResult<Option<PyBytes>> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
 
         let node_as_string = if cfg!(feature = "python3-sys") {
-            node.cast_as::<PyString>(py)?.to_string(py)?.to_string()
+            pynode.cast_as::<PyString>(py)?.to_string(py)?.to_string()
         }
         else {
-            let node = node.extract::<PyBytes>(py)?;
+            let node = pynode.extract::<PyBytes>(py)?;
             String::from_utf8_lossy(node.data(py)).to_string()
         };
 
         let prefix = NodePrefix::from_hex(&node_as_string).map_err(|_| PyErr::new::<ValueError, _>(py, "Invalid node or prefix"))?;
 
-        nt.find_bin(idx, prefix)
-            // TODO make an inner API returning the node directly
-            .map(|opt| opt.map(
-                |rev| PyBytes::new(py, idx.node(rev).unwrap().as_bytes())))
-            .map_err(|e| nodemap_error(py, e))
-
+        match nt.find_bin(idx, prefix) {
+            Ok(None) =>
+                // fallback to C implementation, remove once
+                // https://bz.mercurial-scm.org/show_bug.cgi?id=6554
+                // is fixed (a simple backout should do)
+                self.call_cindex(
+                    py, "partialmatch",
+                    &PyTuple::new(py, &[pynode]), None
+                )?.extract(py),
+            Ok(Some(rev)) =>
+                Ok(Some(PyBytes::new(py, idx.node(rev).unwrap().as_bytes()))),
+            Err(e) => Err(nodemap_error(py, e)),
+        }
     }
 
     /// append an index entry
     def append(&self, tup: PyTuple) -> PyResult<PyObject> {
         if tup.len(py) < 8 {
             // this is better than the panic promised by tup.get_item()
             return Err(
                 PyErr::new::<IndexError, _>(py, "tuple index out of range"))
         }
         let node_bytes = tup.get_item(py, 7).extract(py)?;
         let node = node_from_py_object(py, &node_bytes)?;
 
         let mut idx = self.cindex(py).borrow_mut();
         let rev = idx.len() as Revision;
 
         idx.append(py, tup)?;
         self.get_nodetree(py)?.borrow_mut().as_mut().unwrap()
             .insert(&*idx, &node, rev)
             .map_err(|e| nodemap_error(py, e))?;
         Ok(py.None())
     }
 
     def __delitem__(&self, key: PyObject) -> PyResult<()> {
         // __delitem__ is both for `del idx[r]` and `del idx[r1:r2]`
         self.cindex(py).borrow().inner().del_item(py, key)?;
         let mut opt = self.get_nodetree(py)?.borrow_mut();
         let mut nt = opt.as_mut().unwrap();
         nt.invalidate_all();
         self.fill_nodemap(py, &mut nt)?;
         Ok(())
     }
 
     //
     // Reforwarded C index API
     //
 
     // index_methods (tp_methods). Same ordering as in revlog.c
 
     /// return the gca set of the given revs
     def ancestors(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "ancestors", args, kw)
     }
 
     /// return the heads of the common ancestors of the given revs
     def commonancestorsheads(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "commonancestorsheads", args, kw)
     }
 
     /// Clear the index caches and inner py_class data.
     /// It is Python's responsibility to call `update_nodemap_data` again.
     def clearcaches(&self, *args, **kw) -> PyResult<PyObject> {
         self.nt(py).borrow_mut().take();
         self.docket(py).borrow_mut().take();
         self.mmap(py).borrow_mut().take();
         self.call_cindex(py, "clearcaches", args, kw)
     }
 
     /// return the raw binary string representing a revision
     def entry_binary(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "entry_binary", args, kw)
     }
 
     /// return a binary packed version of the header
     def pack_header(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "pack_header", args, kw)
     }
 
     /// get an index entry
     def get(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "get", args, kw)
     }
 
     /// compute phases
     def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "computephasesmapsets", args, kw)
     }
 
     /// reachableroots
     def reachableroots2(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "reachableroots2", args, kw)
     }
 
     /// get head revisions
     def headrevs(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "headrevs", args, kw)
     }
 
     /// get filtered head revisions
     def headrevsfiltered(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "headrevsfiltered", args, kw)
     }
 
     /// True if the object is a snapshot
     def issnapshot(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "issnapshot", args, kw)
     }
 
     /// Gather snapshot data in a cache dict
     def findsnapshots(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "findsnapshots", args, kw)
     }
 
     /// determine revisions with deltas to reconstruct fulltext
     def deltachain(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "deltachain", args, kw)
     }
 
     /// slice planned chunk read to reach a density threshold
     def slicechunktodensity(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "slicechunktodensity", args, kw)
     }
 
     /// stats for the index
     def stats(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "stats", args, kw)
     }
 
     // index_sequence_methods and index_mapping_methods.
     //
     // Since we call back through the high level Python API,
     // there's no point making a distinction between index_get
     // and index_getitem.
 
     def __len__(&self) -> PyResult<usize> {
         self.cindex(py).borrow().inner().len(py)
     }
 
     def __getitem__(&self, key: PyObject) -> PyResult<PyObject> {
         // this conversion seems needless, but that's actually because
         // `index_getitem` does not handle conversion from PyLong,
         // which expressions such as [e for e in index] internally use.
         // Note that we don't seem to have a direct way to call
         // PySequence_GetItem (does the job), which would possibly be better
         // for performance
         let key = match key.extract::<Revision>(py) {
             Ok(rev) => rev.to_py_object(py).into_object(),
             Err(_) => key,
         };
         self.cindex(py).borrow().inner().get_item(py, key)
     }
 
     def __setitem__(&self, key: PyObject, value: PyObject) -> PyResult<()> {
         self.cindex(py).borrow().inner().set_item(py, key, value)
     }
 
     def __contains__(&self, item: PyObject) -> PyResult<bool> {
         // ObjectProtocol does not seem to provide contains(), so
         // this is an equivalent implementation of the index_contains()
         // defined in revlog.c
         let cindex = self.cindex(py).borrow();
         match item.extract::<Revision>(py) {
             Ok(rev) => {
                 Ok(rev >= -1 && rev < cindex.inner().len(py)? as Revision)
             }
             Err(_) => {
                 cindex.inner().call_method(
                     py,
                     "has_node",
                     PyTuple::new(py, &[item]),
                     None)?
                 .extract(py)
             }
         }
     }
 
     def nodemap_data_all(&self) -> PyResult<PyBytes> {
         self.inner_nodemap_data_all(py)
     }
 
     def nodemap_data_incremental(&self) -> PyResult<PyObject> {
         self.inner_nodemap_data_incremental(py)
     }
     def update_nodemap_data(
         &self,
         docket: PyObject,
         nm_data: PyObject
     ) -> PyResult<PyObject> {
         self.inner_update_nodemap_data(py, docket, nm_data)
     }
 
     @property
     def entry_size(&self) -> PyResult<PyInt> {
         self.cindex(py).borrow().inner().getattr(py, "entry_size")?.extract::<PyInt>(py)
     }
 
     @property
     def rust_ext_compat(&self) -> PyResult<PyInt> {
         self.cindex(py).borrow().inner().getattr(py, "rust_ext_compat")?.extract::<PyInt>(py)
     }
 
 });
 
 impl MixedIndex {
     fn new(py: Python, cindex: PyObject) -> PyResult<MixedIndex> {
         Self::create_instance(
             py,
             RefCell::new(cindex::Index::new(py, cindex)?),
             RefCell::new(None),
             RefCell::new(None),
             RefCell::new(None),
         )
     }
 
     /// This is scaffolding at this point, but it could also become
     /// a way to start a persistent nodemap or perform a
     /// vacuum / repack operation
     fn fill_nodemap(
         &self,
         py: Python,
         nt: &mut NodeTree,
     ) -> PyResult<PyObject> {
         let index = self.cindex(py).borrow();
         for r in 0..index.len() {
             let rev = r as Revision;
             // in this case node() won't ever return None
             nt.insert(&*index, index.node(rev).unwrap(), rev)
                 .map_err(|e| nodemap_error(py, e))?
         }
         Ok(py.None())
     }
 
     fn get_nodetree<'a>(
         &'a self,
         py: Python<'a>,
     ) -> PyResult<&'a RefCell<Option<NodeTree>>> {
         if self.nt(py).borrow().is_none() {
             let readonly = Box::new(Vec::new());
             let mut nt = NodeTree::load_bytes(readonly, 0);
             self.fill_nodemap(py, &mut nt)?;
             self.nt(py).borrow_mut().replace(nt);
         }
         Ok(self.nt(py))
     }
 
     /// forward a method call to the underlying C index
     fn call_cindex(
         &self,
         py: Python,
         name: &str,
         args: &PyTuple,
         kwargs: Option<&PyDict>,
     ) -> PyResult<PyObject> {
         self.cindex(py)
             .borrow()
             .inner()
             .call_method(py, name, args, kwargs)
     }
 
     pub fn clone_cindex(&self, py: Python) -> cindex::Index {
         self.cindex(py).borrow().clone_ref(py)
     }
 
     /// Returns the full nodemap bytes to be written as-is to disk
     fn inner_nodemap_data_all(&self, py: Python) -> PyResult<PyBytes> {
         let nodemap = self.get_nodetree(py)?.borrow_mut().take().unwrap();
         let (readonly, bytes) = nodemap.into_readonly_and_added_bytes();
 
         // If there's anything readonly, we need to build the data again from
         // scratch
         let bytes = if readonly.len() > 0 {
             let mut nt = NodeTree::load_bytes(Box::new(vec![]), 0);
             self.fill_nodemap(py, &mut nt)?;
 
             let (readonly, bytes) = nt.into_readonly_and_added_bytes();
             assert_eq!(readonly.len(), 0);
 
             bytes
         } else {
             bytes
         };
 
         let bytes = PyBytes::new(py, &bytes);
         Ok(bytes)
     }
 
     /// Returns the last saved docket along with the size of any changed data
     /// (in number of blocks), and said data as bytes.
     fn inner_nodemap_data_incremental(
         &self,
         py: Python,
     ) -> PyResult<PyObject> {
         let docket = self.docket(py).borrow();
         let docket = match docket.as_ref() {
             Some(d) => d,
             None => return Ok(py.None()),
         };
 
         let node_tree = self.get_nodetree(py)?.borrow_mut().take().unwrap();
         let masked_blocks = node_tree.masked_readonly_blocks();
         let (_, data) = node_tree.into_readonly_and_added_bytes();
         let changed = masked_blocks * std::mem::size_of::<Block>();
 
         Ok((docket, changed, PyBytes::new(py, &data))
             .to_py_object(py)
             .into_object())
     }
 
     /// Update the nodemap from the new (mmaped) data.
     /// The docket is kept as a reference for later incremental calls.
     fn inner_update_nodemap_data(
         &self,
         py: Python,
         docket: PyObject,
         nm_data: PyObject,
     ) -> PyResult<PyObject> {
         let buf = PyBuffer::get(py, &nm_data)?;
         let len = buf.item_count();
 
         // Build a slice from the mmap'ed buffer data
         let cbuf = buf.buf_ptr();
         let bytes = if std::mem::size_of::<u8>() == buf.item_size()
             && buf.is_c_contiguous()
             && u8::is_compatible_format(buf.format())
         {
             unsafe { std::slice::from_raw_parts(cbuf as *const u8, len) }
         } else {
             return Err(PyErr::new::<ValueError, _>(
                 py,
                 "Nodemap data buffer has an invalid memory representation"
                     .to_string(),
             ));
         };
 
         // Keep a reference to the mmap'ed buffer, otherwise we get a dangling
         // pointer.
         self.mmap(py).borrow_mut().replace(buf);
 
         let mut nt = NodeTree::load_bytes(Box::new(bytes), len);
 
         let data_tip =
             docket.getattr(py, "tip_rev")?.extract::<Revision>(py)?;
         self.docket(py).borrow_mut().replace(docket.clone_ref(py));
         let idx = self.cindex(py).borrow();
         let current_tip = idx.len();
 
         for r in (data_tip + 1)..current_tip as Revision {
             let rev = r as Revision;
             // in this case node() won't ever return None
             nt.insert(&*idx, idx.node(rev).unwrap(), rev)
                 .map_err(|e| nodemap_error(py, e))?
         }
 
         *self.nt(py).borrow_mut() = Some(nt);
 
         Ok(py.None())
     }
 }
 
 fn revlog_error(py: Python) -> PyErr {
     match py
         .import("mercurial.error")
         .and_then(|m| m.get(py, "RevlogError"))
     {
         Err(e) => e,
         Ok(cls) => PyErr::from_instance(
             py,
             cls.call(py, (py.None(),), None).ok().into_py_object(py),
         ),
     }
 }
 
 fn rev_not_in_index(py: Python, rev: Revision) -> PyErr {
     PyErr::new::<ValueError, _>(
         py,
         format!(
             "Inconsistency: Revision {} found in nodemap \
              is not in revlog index",
             rev
         ),
     )
 }
 
 /// Standard treatment of NodeMapError
 fn nodemap_error(py: Python, err: NodeMapError) -> PyErr {
     match err {
         NodeMapError::MultipleResults => revlog_error(py),
         NodeMapError::RevisionNotInIndex(r) => rev_not_in_index(py, r),
     }
 }
 
 /// Create the module, with __package__ given from parent
 pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
     let dotted_name = &format!("{}.revlog", package);
     let m = PyModule::new(py, dotted_name)?;
     m.add(py, "__package__", package)?;
     m.add(py, "__doc__", "RevLog - Rust implementations")?;
 
     m.add_class::<MixedIndex>(py)?;
 
     let sys = PyModule::import(py, "sys")?;
     let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
     sys_modules.set_item(py, dotted_name, &m)?;
 
     Ok(m)
 }

tests/test-persistent-nodemap.t

@@ -1,1067 +1,1107 b''
 ===================================
 Test the persistent on-disk nodemap
 ===================================
 
 
 #if no-rust
 
   $ cat << EOF >> $HGRCPATH
   > [format]
   > use-persistent-nodemap=yes
   > [devel]
   > persistent-nodemap=yes
   > EOF
 
 #endif
 
   $ hg init test-repo --config storage.revlog.persistent-nodemap.slow-path=allow
   $ cd test-repo
 
 Check handling of the default slow-path value
 
 #if no-pure no-rust
 
   $ hg id
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
 Unlock further check (we are here to test the feature)
 
   $ cat << EOF >> $HGRCPATH
   > [storage]
   > # to avoid spamming the test
   > revlog.persistent-nodemap.slow-path=allow
   > EOF
 
 #endif
 
 #if rust
 
 Regression test for a previous bug in Rust/C FFI for the `Revlog_CAPI` capsule:
 in places where `mercurial/cext/revlog.c` function signatures use `Py_ssize_t`
 (64 bits on Linux x86_64), corresponding declarations in `rust/hg-cpython/src/cindex.rs`
 incorrectly used `libc::c_int` (32 bits).
 As a result, -1 passed from Rust for the null revision became 4294967295 in C.
 
   $ hg log -r 00000000
   changeset:   -1:000000000000
   tag:         tip
   user:        
   date:        Thu Jan 01 00:00:00 1970 +0000
   
 
 #endif
 
 
   $ hg debugformat
   format-variant     repo
   fncache:            yes
   dirstate-v2:         no
   dotencode:          yes
   generaldelta:       yes
   share-safe:          no
   sparserevlog:       yes
   persistent-nodemap: yes
   copies-sdc:          no
   revlog-v2:           no
   changelog-v2:        no
   plain-cl-delta:     yes
   compression:        zlib (no-zstd !)
   compression:        zstd (zstd !)
   compression-level:  default
   $ hg debugbuilddag .+5000 --new-file
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5000
   tip-node: 6b02b8c7b96654c25e86ba69eda198d7e6ad8b3c
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=62
 
 Simple lookup works
 
   $ ANYNODE=`hg log --template '{node|short}\n' --rev tip`
   $ hg log -r "$ANYNODE" --template '{rev}\n'
   5000
 
 
 #if rust
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd (glob)
 
   $ f --sha256 .hg/store/00manifest-*.nd
   .hg/store/00manifest-????????.nd: sha256=97117b1c064ea2f86664a124589e47db0e254e8d34739b5c5cc5bf31c9da2b51 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=121088, sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=121088, sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd
   0000: 00 00 00 91 00 00 00 20 00 00 00 bb 00 00 00 e7 |....... ........|
   0010: 00 00 00 66 00 00 00 a1 00 00 01 13 00 00 01 22 |...f..........."|
   0020: 00 00 00 23 00 00 00 fc 00 00 00 ba 00 00 00 5e |...#...........^|
   0030: 00 00 00 df 00 00 01 4e 00 00 01 65 00 00 00 ab |.......N...e....|
   0040: 00 00 00 a9 00 00 00 95 00 00 00 73 00 00 00 38 |...........s...8|
   0050: 00 00 00 cc 00 00 00 92 00 00 00 90 00 00 00 69 |...............i|
   0060: 00 00 00 ec 00 00 00 8d 00 00 01 4f 00 00 00 12 |...........O....|
   0070: 00 00 02 0c 00 00 00 77 00 00 00 9c 00 00 00 8f |.......w........|
   0080: 00 00 00 d5 00 00 00 6b 00 00 00 48 00 00 00 b3 |.......k...H....|
   0090: 00 00 00 e5 00 00 00 b5 00 00 00 8e 00 00 00 ad |................|
   00a0: 00 00 00 7b 00 00 00 7c 00 00 00 0b 00 00 00 2b |...{...|.......+|
   00b0: 00 00 00 c6 00 00 00 1e 00 00 01 08 00 00 00 11 |................|
   00c0: 00 00 01 30 00 00 00 26 00 00 01 9c 00 00 00 35 |...0...&.......5|
   00d0: 00 00 00 b8 00 00 01 31 00 00 00 2c 00 00 00 55 |.......1...,...U|
   00e0: 00 00 00 8a 00 00 00 9a 00 00 00 0c 00 00 01 1e |................|
   00f0: 00 00 00 a4 00 00 00 83 00 00 00 c9 00 00 00 8c |................|
 
 
 #else
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=121088, sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=121088, sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0020: ff ff ff ff ff ff f5 06 ff ff ff ff ff ff f3 e7 |................|
   0030: ff ff ef ca ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0040: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0050: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ed 08 |................|
   0060: ff ff ed 66 ff ff ff ff ff ff ff ff ff ff ff ff |...f............|
   0070: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0090: ff ff ff ff ff ff ff ff ff ff ff ff ff ff f6 ed |................|
   00a0: ff ff ff ff ff ff fe 61 ff ff ff ff ff ff ff ff |.......a........|
   00b0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00c0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff f1 02 |................|
   00f0: ff ff ff ff ff ff ed 1b ff ff ff ff ff ff ff ff |................|
 
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5001
   revision in nodemap: 5001
 
 add a new commit
 
   $ hg up
   5001 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo foo > foo
   $ hg add foo
 
 
 Check slow-path config value handling
 -------------------------------------
 
 #if no-pure no-rust
 
   $ hg id --config "storage.revlog.persistent-nodemap.slow-path=invalid-value"
   unknown value for config "storage.revlog.persistent-nodemap.slow-path": "invalid-value"
   falling back to default value: abort
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
   $ hg log -r . --config "storage.revlog.persistent-nodemap.slow-path=warn"
   warning: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   changeset:   5000:6b02b8c7b966
   tag:         tip
   user:        debugbuilddag
   date:        Thu Jan 01 01:23:20 1970 +0000
   summary:     r5000
   
   $ hg ci -m 'foo' --config "storage.revlog.persistent-nodemap.slow-path=abort"
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
 #else
 
   $ hg id --config "storage.revlog.persistent-nodemap.slow-path=invalid-value"
   unknown value for config "storage.revlog.persistent-nodemap.slow-path": "invalid-value"
   falling back to default value: abort
   6b02b8c7b966+ tip
 
 #endif
 
   $ hg ci -m 'foo'
 
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5001
   tip-node: 16395c3cf7e231394735e6b1717823ada303fb0c
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5001
   tip-node: 16395c3cf7e231394735e6b1717823ada303fb0c
   data-length: 121344
   data-unused: 256
   data-unused: 0.211%
 #endif
 
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=62
 
 (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
 
 #if pure
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121344, sha256=cce54c5da5bde3ad72a4938673ed4064c86231b9c64376b082b163fdb20f8f66 (glob)
 #endif
 
 #if rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121344, sha256=952b042fcf614ceb37b542b1b723e04f18f83efe99bee4e0f5ccd232ef470e58 (glob)
 #endif
 
 #if no-pure no-rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121088, sha256=df7c06a035b96cb28c7287d349d603baef43240be7736fe34eea419a49702e17 (glob)
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
 
 Test code path without mmap
 ---------------------------
 
   $ echo bar > bar
   $ hg add bar
   $ hg ci -m 'bar' --config storage.revlog.persistent-nodemap.mmap=no
 
   $ hg debugnodemap --check --config storage.revlog.persistent-nodemap.mmap=yes
   revision in index:   5003
   revision in nodemap: 5003
   $ hg debugnodemap --check --config storage.revlog.persistent-nodemap.mmap=no
   revision in index:   5003
   revision in nodemap: 5003
 
 
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121600
   data-unused: 512
   data-unused: 0.421%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121600, sha256=def52503d049ccb823974af313a98a935319ba61f40f3aa06a8be4d35c215054 (glob)
 #endif
 #if rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121600
   data-unused: 512
   data-unused: 0.421%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121600, sha256=dacf5b5f1d4585fee7527d0e67cad5b1ba0930e6a0928f650f779aefb04ce3fb (glob)
 #endif
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121088, sha256=59fcede3e3cc587755916ceed29e3c33748cd1aa7d2f91828ac83e7979d935e8 (glob)
 #endif
 
 Test force warming the cache
 
   $ rm .hg/store/00changelog.n
   $ hg debugnodemap --metadata
   $ hg debugupdatecache
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #endif
 
 Check out of sync nodemap
 =========================
 
 First copy old data on the side.
 
   $ mkdir ../tmp-copies
   $ cp .hg/store/00changelog-????????.nd .hg/store/00changelog.n ../tmp-copies
 
 Nodemap lagging behind
 ----------------------
 
 make a new commit
 
   $ echo bar2 > bar
   $ hg ci -m 'bar2'
   $ NODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 If the nodemap is lagging behind, it can catch up fine
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5003
   tip-node: c9329770f979ade2d16912267c38ba5f82fd37b3
   data-length: 121344 (pure !)
   data-length: 121344 (rust !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-rust no-pure !)
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 changelog altered
 -----------------
 
 If the nodemap is not gated behind a requirements, an unaware client can alter
 the repository so the revlog used to generate the nodemap is not longer
 compatible with the persistent nodemap. We need to detect that.
 
   $ hg up "$NODE~5"
   0 files updated, 0 files merged, 4 files removed, 0 files unresolved
   $ echo bar > babar
   $ hg add babar
   $ hg ci -m 'babar'
   created new head
   $ OTHERNODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5004
 
   $ hg --config extensions.strip= strip --rev "$NODE~1" --no-backup
 
 the nodemap should detect the changelog have been tampered with and recover.
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: b355ef8adce0949b8bdf6afc72ca853740d65944
   data-length: 121536 (pure !)
   data-length: 121088 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.000% (rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5002
 
 missing data file
 -----------------
 
   $ UUID=`hg debugnodemap --metadata| grep 'uid:' | \
   > sed 's/uid: //'`
   $ FILE=.hg/store/00changelog-"${UUID}".nd
   $ mv $FILE ../tmp-data-file
   $ cp .hg/store/00changelog.n ../tmp-docket
 
 mercurial don't crash
 
   $ hg log -r .
   changeset:   5002:b355ef8adce0
   tag:         tip
   parent:      4998:d918ad6d18d3
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     babar
   
   $ hg debugnodemap --metadata
 
   $ hg debugupdatecache
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5002
   tip-node: b355ef8adce0949b8bdf6afc72ca853740d65944
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
+
+Sub-case: fallback for corrupted data file
+------------------------------------------
+
+Sabotaging the data file so that nodemap resolutions fail, triggering fallback to
+(non-persistent) C implementation.
+
+
+  $ UUID=`hg debugnodemap --metadata| grep 'uid:' | \
+  > sed 's/uid: //'`
+  $ FILE=.hg/store/00changelog-"${UUID}".nd
+  $ python -c "fobj = open('$FILE', 'r+b'); fobj.write(b'\xff' * 121088); fobj.close()"
+
+The nodemap data file is still considered in sync with the docket. This
+would fail without the fallback to the (non-persistent) C implementation:
+
+  $ hg log -r b355ef8adce0949b8bdf6afc72ca853740d65944 -T '{rev}\n' --traceback
+  5002
+
+The nodemap data file hasn't been fixed, more tests can be inserted:
+
+  $ hg debugnodemap --dump-disk | f --bytes=256 --hexdump --size
+  size=121088
+  0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0030: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0040: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0050: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0060: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0070: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  0090: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00a0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00b0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00c0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+  00f0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
+
   $ mv ../tmp-data-file $FILE
   $ mv ../tmp-docket .hg/store/00changelog.n
 
 Check transaction related property
 ==================================
 
 An up to date nodemap should be available to shell hooks,
 
   $ echo dsljfl > a
   $ hg add a
   $ hg ci -m a
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5003
   tip-node: a52c5079765b5865d97b993b303a18740113bbb2
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ echo babar2 > babar
   $ hg ci -m 'babar2' --config "hooks.pretxnclose.nodemap-test=hg debugnodemap --metadata"
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
 Another process does not see the pending nodemap content during run.
 
   $ echo qpoasp > a
   $ hg ci -m a2 \
   > --config "hooks.pretxnclose=sh \"$RUNTESTDIR/testlib/wait-on-file\" 20 sync-repo-read sync-txn-pending" \
   > --config "hooks.txnclose=touch sync-txn-close" > output.txt 2>&1 &
 
 (read the repository while the commit transaction is pending)
 
   $ sh "$RUNTESTDIR/testlib/wait-on-file" 20 sync-txn-pending && \
   > hg debugnodemap --metadata && \
   > sh "$RUNTESTDIR/testlib/wait-on-file" 20 sync-txn-close sync-repo-read
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121536 (pure !)
   data-length: 121536 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.369% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cat output.txt
 
 Check that a failing transaction will properly revert the data
 
   $ echo plakfe > a
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: size=121536, sha256=bb414468d225cf52d69132e1237afba34d4346ee2eb81b505027e6197b107f03 (glob) (pure !)
   .hg/store/00changelog-????????.nd: size=121536, sha256=909ac727bc4d1c0fda5f7bff3c620c98bd4a2967c143405a1503439e33b377da (glob) (rust !)
   .hg/store/00changelog-????????.nd: size=121088, sha256=342d36d30d86dde67d3cb6c002606c4a75bcad665595d941493845066d9c8ee0 (glob) (no-pure no-rust !)
   $ hg ci -m a3 --config "extensions.abort=$RUNTESTDIR/testlib/crash_transaction_late.py"
   transaction abort!
   rollback completed
   abort: This is a late abort
   [255]
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121536 (pure !)
   data-length: 121536 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.369% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: size=121536, sha256=bb414468d225cf52d69132e1237afba34d4346ee2eb81b505027e6197b107f03 (glob) (pure !)
   .hg/store/00changelog-????????.nd: size=121536, sha256=909ac727bc4d1c0fda5f7bff3c620c98bd4a2967c143405a1503439e33b377da (glob) (rust !)
   .hg/store/00changelog-????????.nd: size=121088, sha256=342d36d30d86dde67d3cb6c002606c4a75bcad665595d941493845066d9c8ee0 (glob) (no-pure no-rust !)
 
 Check that removing content does not confuse the nodemap
 --------------------------------------------------------
 
 removing data with rollback
 
   $ echo aso > a
   $ hg ci -m a4
   $ hg rollback
   repository tip rolled back to revision 5005 (undo commit)
   working directory now based on revision 5005
   $ hg id -r .
   90d5d3ba2fc4 tip
 
 roming data with strip
 
   $ echo aso > a
   $ hg ci -m a4
   $ hg --config extensions.strip= strip -r . --no-backup
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg id -r . --traceback
   90d5d3ba2fc4 tip
 
 Test upgrade / downgrade
 ========================
 
 downgrading
 
   $ cat << EOF >> .hg/hgrc
   > [format]
   > use-persistent-nodemap=no
   > EOF
   $ hg debugformat -v
   format-variant     repo config default
   fncache:            yes    yes     yes
   dirstate-v2:         no     no      no
   dotencode:          yes    yes     yes
   generaldelta:       yes    yes     yes
   share-safe:          no     no      no
   sparserevlog:       yes    yes     yes
   persistent-nodemap: yes     no      no
   copies-sdc:          no     no      no
   revlog-v2:           no     no      no
   changelog-v2:        no     no      no
   plain-cl-delta:     yes    yes     yes
   compression:        zlib   zlib    zlib (no-zstd !)
   compression:        zstd   zstd    zstd (zstd !)
   compression-level:  default default default
   $ hg debugupgraderepo --run --no-backup
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd dirstate-v2 !)
      removed: persistent-nodemap
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   beginning upgrade...
   repository locked and read-only
   creating temporary repository to stage upgraded data: $TESTTMP/test-repo/.hg/upgrade.* (glob)
   (it is safe to interrupt this process any time before data migration completes)
   downgrading repository to not use persistent nodemap feature
   removing temporary repository $TESTTMP/test-repo/.hg/upgrade.* (glob)
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00manifest-*.nd (glob)
   undo.backup.00changelog.n
   undo.backup.00manifest.n
   $ hg debugnodemap --metadata
 
 
 upgrading
 
   $ cat << EOF >> .hg/hgrc
   > [format]
   > use-persistent-nodemap=yes
   > EOF
   $ hg debugformat -v
   format-variant     repo config default
   fncache:            yes    yes     yes
   dirstate-v2:         no     no      no
   dotencode:          yes    yes     yes
   generaldelta:       yes    yes     yes
   share-safe:          no     no      no
   sparserevlog:       yes    yes     yes
   persistent-nodemap:  no    yes      no
   copies-sdc:          no     no      no
   revlog-v2:           no     no      no
   changelog-v2:        no     no      no
   plain-cl-delta:     yes    yes     yes
   compression:        zlib   zlib    zlib (no-zstd !)
   compression:        zstd   zstd    zstd (zstd !)
   compression-level:  default default default
   $ hg debugupgraderepo --run --no-backup
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd dirstate-v2 !)
      added: persistent-nodemap
   
   persistent-nodemap
      Speedup revision lookup by node id.
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   beginning upgrade...
   repository locked and read-only
   creating temporary repository to stage upgraded data: $TESTTMP/test-repo/.hg/upgrade.* (glob)
   (it is safe to interrupt this process any time before data migration completes)
   upgrading repository to use persistent nodemap feature
   removing temporary repository $TESTTMP/test-repo/.hg/upgrade.* (glob)
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   undo.backup.00changelog.n
   undo.backup.00manifest.n
 
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Running unrelated upgrade
 
   $ hg debugupgraderepo --run --no-backup --quiet --optimize re-delta-all
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, persistent-nodemap, revlogv1, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, sparserevlog, store (zstd dirstate-v2 !)
   
   optimisations: re-delta-all
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
 
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Persistent nodemap and local/streaming clone
 ============================================
 
   $ cd ..
 
 standard clone
 --------------
 
 The persistent nodemap should exist after a streaming clone
 
   $ hg clone --pull --quiet -U test-repo standard-clone
   $ ls -1 standard-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R standard-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 
 local clone
 ------------
 
 The persistent nodemap should exist after a streaming clone
 
   $ hg clone -U test-repo local-clone
   $ ls -1 local-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R local-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Test various corruption case
 ============================
 
 Missing datafile
 ----------------
 
 Test behavior with a missing datafile
 
   $ hg clone --quiet --pull test-repo corruption-test-repo
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog-*.nd (glob)
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
   $ rm corruption-test-repo/.hg/store/00changelog*.nd
   $ hg log -R corruption-test-repo -r .
   changeset:   5005:90d5d3ba2fc4
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     a2
   
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
 
 Truncated data file
 -------------------
 
 Test behavior with a too short datafile
 
 rebuild the missing data
   $ hg -R corruption-test-repo debugupdatecache
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog-*.nd (glob)
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
 
 truncate the file
 
   $ datafilepath=`ls corruption-test-repo/.hg/store/00changelog*.nd`
   $ f -s $datafilepath
   corruption-test-repo/.hg/store/00changelog-*.nd: size=121088 (glob)
   $ dd if=$datafilepath bs=1000 count=10 of=$datafilepath-tmp status=noxfer
   10+0 records in
   10+0 records out
   $ mv $datafilepath-tmp $datafilepath
   $ f -s $datafilepath
   corruption-test-repo/.hg/store/00changelog-*.nd: size=10000 (glob)
 
 Check that Mercurial reaction to this event
 
   $ hg -R corruption-test-repo log -r . --traceback
   changeset:   5005:90d5d3ba2fc4
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     a2
   
 
 
 stream clone
 ============
 
 The persistent nodemap should exist after a streaming clone
 
 Simple case
 -----------
 
 No race condition
 
   $ hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone --debug | egrep '00(changelog|manifest)'
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00manifest.d (491 KB) (zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
   $ ls -1 stream-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R stream-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 new data appened
 -----------------
 
 Other commit happening on the server during the stream clone
 
 setup the step-by-step stream cloning
 
   $ HG_TEST_STREAM_WALKED_FILE_1="$TESTTMP/sync_file_walked_1"
   $ export HG_TEST_STREAM_WALKED_FILE_1
   $ HG_TEST_STREAM_WALKED_FILE_2="$TESTTMP/sync_file_walked_2"
   $ export HG_TEST_STREAM_WALKED_FILE_2
   $ HG_TEST_STREAM_WALKED_FILE_3="$TESTTMP/sync_file_walked_3"
   $ export HG_TEST_STREAM_WALKED_FILE_3
   $ cat << EOF >> test-repo/.hg/hgrc
   > [extensions]
   > steps=$RUNTESTDIR/testlib/ext-stream-clone-steps.py
   > EOF
 
 Check and record file state beforehand
 
   $ f --size test-repo/.hg/store/00changelog*
   test-repo/.hg/store/00changelog-*.nd: size=121088 (glob)
   test-repo/.hg/store/00changelog.d: size=376891 (zstd !)
   test-repo/.hg/store/00changelog.d: size=368890 (no-zstd !)
   test-repo/.hg/store/00changelog.i: size=320384
   test-repo/.hg/store/00changelog.n: size=62
   $ hg -R test-repo debugnodemap --metadata | tee server-metadata.txt
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Prepare a commit
 
   $ echo foo >> test-repo/foo
   $ hg -R test-repo/ add test-repo/foo
 
 Do a mix of clone and commit at the same time so that the file listed on disk differ at actual transfer time.
 
   $ (hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone-race-1 --debug 2>> clone-output | egrep '00(changelog|manifest)' >> clone-output; touch $HG_TEST_STREAM_WALKED_FILE_3) &
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_1
   $ hg -R test-repo/ commit -m foo
   $ touch $HG_TEST_STREAM_WALKED_FILE_2
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_3
   $ cat clone-output
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00manifest.d (491 KB) (zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
 
 Check the result state
 
   $ f --size stream-clone-race-1/.hg/store/00changelog*
   stream-clone-race-1/.hg/store/00changelog-*.nd: size=121088 (glob)
   stream-clone-race-1/.hg/store/00changelog.d: size=368890 (no-zstd !)
   stream-clone-race-1/.hg/store/00changelog.d: size=376891 (zstd !)
   stream-clone-race-1/.hg/store/00changelog.i: size=320384
   stream-clone-race-1/.hg/store/00changelog.n: size=62
 
   $ hg -R stream-clone-race-1 debugnodemap --metadata | tee client-metadata.txt
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 We get a usable nodemap, so no rewrite would be needed and the metadata should be identical
 (ie: the following diff should be empty)
 
 This isn't the case for the `no-rust` `no-pure` implementation as it use a very minimal nodemap implementation that unconditionnaly rewrite the nodemap "all the time".
 
 #if no-rust no-pure
   $ diff -u server-metadata.txt client-metadata.txt
   --- server-metadata.txt	* (glob)
   +++ client-metadata.txt	* (glob)
   @@ -1,4 +1,4 @@
   -uid: * (glob)
   +uid: * (glob)
    tip-rev: 5005
    tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
    data-length: 121088
   [1]
 #else
   $ diff -u server-metadata.txt client-metadata.txt
 #endif
 
 
 Clean up after the test.
 
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_1"
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_2"
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_3"
 
 full regeneration
 -----------------
 
 A full nodemap is generated
 
 (ideally this test would append enough data to make sure the nodemap data file
 get changed, however to make thing simpler we will force the regeneration for
 this test.
 
 Check the initial state
 
   $ f --size test-repo/.hg/store/00changelog*
   test-repo/.hg/store/00changelog-*.nd: size=121344 (glob) (rust !)
   test-repo/.hg/store/00changelog-*.nd: size=121344 (glob) (pure !)
   test-repo/.hg/store/00changelog-*.nd: size=121152 (glob) (no-rust no-pure !)
   test-repo/.hg/store/00changelog.d: size=376950 (zstd !)
   test-repo/.hg/store/00changelog.d: size=368949 (no-zstd !)
   test-repo/.hg/store/00changelog.i: size=320448
   test-repo/.hg/store/00changelog.n: size=62
   $ hg -R test-repo debugnodemap --metadata | tee server-metadata-2.txt
   uid: * (glob)
   tip-rev: 5006
   tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
   data-length: 121344 (rust !)
   data-length: 121344 (pure !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 192 (rust !)
   data-unused: 192 (pure !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.000% (no-rust no-pure !)
 
 Performe the mix of clone and full refresh of the nodemap, so that the files
 (and filenames) are different between listing time and actual transfer time.
 
   $ (hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone-race-2 --debug 2>> clone-output-2 | egrep '00(changelog|manifest)' >> clone-output-2; touch $HG_TEST_STREAM_WALKED_FILE_3) &
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_1
   $ rm test-repo/.hg/store/00changelog.n
   $ rm test-repo/.hg/store/00changelog-*.nd
   $ hg -R test-repo/ debugupdatecache
   $ touch $HG_TEST_STREAM_WALKED_FILE_2
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_3
 
 (note: the stream clone code wronly pick the `undo.` files)
 
   $ cat clone-output-2
   adding [s] undo.backup.00manifest.n (62 bytes) (known-bad-output !)
   adding [s] undo.backup.00changelog.n (62 bytes) (known-bad-output !)
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (492 KB) (zstd !)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
 
 Check the result.
 
   $ f --size stream-clone-race-2/.hg/store/00changelog*
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121344 (glob) (rust !)
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121344 (glob) (pure !)
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121152 (glob) (no-rust no-pure !)
   stream-clone-race-2/.hg/store/00changelog.d: size=376950 (zstd !)
   stream-clone-race-2/.hg/store/00changelog.d: size=368949 (no-zstd !)
   stream-clone-race-2/.hg/store/00changelog.i: size=320448
   stream-clone-race-2/.hg/store/00changelog.n: size=62
 
   $ hg -R stream-clone-race-2 debugnodemap --metadata | tee client-metadata-2.txt
   uid: * (glob)
   tip-rev: 5006
   tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
   data-length: 121344 (rust !)
   data-unused: 192 (rust !)
   data-unused: 0.158% (rust !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.000% (no-rust no-pure !)
   data-length: 121344 (pure !)
   data-unused: 192 (pure !)
   data-unused: 0.158% (pure !)
 
 We get a usable nodemap, so no rewrite would be needed and the metadata should be identical
 (ie: the following diff should be empty)
 
 This isn't the case for the `no-rust` `no-pure` implementation as it use a very minimal nodemap implementation that unconditionnaly rewrite the nodemap "all the time".
 
 #if no-rust no-pure
   $ diff -u server-metadata-2.txt client-metadata-2.txt
   --- server-metadata-2.txt	* (glob)
   +++ client-metadata-2.txt	* (glob)
   @@ -1,4 +1,4 @@
   -uid: * (glob)
   +uid: * (glob)
    tip-rev: 5006
    tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
    data-length: 121152
   [1]
 #else
   $ diff -u server-metadata-2.txt client-metadata-2.txt
 #endif
 
 Clean up after the test
 
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_1
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_2
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_3