persistent-nodemap: introduce a test to highlight possible race

Weakness in the current file caching of the changelog means that a writer can
end up using an outdated docket. This might result in "committed"
persistent-nodemap data from a previous writer to be overwritten by a later
writer. This break the strong "append only" assumption of the persistent nodemap
and can result in confused reader.

The race windows are quite narrow. See the test documentation for details.

The issues is fixed in the next changeset.

Differential Revision: https://phab.mercurial-scm.org/D11481

tests/testlib/persistent-nodemap-race-ext.py

@@ -0,0 +1,294 b''
+"""Create the race condition for issue6554
+
+The persistent nodemap issues had an issue where a second writer could
+overwrite the data that a previous write just wrote. The would break the append
+only garantee of the persistent nodemap and could confuse reader. This
+extensions create all the necessary synchronisation point to the race condition
+to happen.
+
+It involves 3 process <LEFT> (a writer) <RIGHT> (a writer) and <READER>
+
+[1] <LEFT> take the lock and start a transaction
+[2] <LEFT> updated `00changelog.i` with the new data
+[3] <RIGHT> reads:
+    - the new changelog index `00changelog.i`
+    - the old `00changelog.n`
+[4] <LEFT> update the persistent nodemap:
+    - writing new data from the last valid offset
+    - updating the docket (00changelog.n)
+[5] <LEFT> release the lock
+[6] <RIGHT> grab the lock and run `repo.invalidate`
+[7] <READER> reads:
+    - the changelog index after <LEFT> write
+    - the nodemap docket after <LEFT> write
+[8] <RIGHT> reload the changelog since `00changelog.n` changed
+    /!\ This is the faulty part in issue 6554, the outdated docket is kept
+[9] <RIGHT> write:
+    - the changelog index (00changelog.i)
+    - the nodemap data (00changelog*.nd)
+      /!\ if the outdated docket is used, the write starts from the same ofset
+      /!\ as in [4], overwriting data that <LEFT> wrote in step [4].
+    - the nodemap docket (00changelog.n)
+[10] <READER> reads the nodemap data from `00changelog*.nd`
+     /!\ if step [9] was wrong, the data matching the docket that <READER>
+     /!\ loaded have been overwritten and the expected root-nodes is no longer
+     /!\ valid.
+"""
+
+from __future__ import print_function
+
+import os
+
+from mercurial.revlogutils.constants import KIND_CHANGELOG
+
+from mercurial import (
+    changelog,
+    encoding,
+    extensions,
+    localrepo,
+    node,
+    pycompat,
+    registrar,
+    testing,
+    util,
+)
+
+from mercurial.revlogutils import (
+    nodemap as nodemaputil,
+)
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem(b'devel', b'nodemap-race.role', default=None)
+
+cmdtable = {}
+command = registrar.command(cmdtable)
+
+LEFT = b'left'
+RIGHT = b'right'
+READER = b'reader'
+
+SYNC_DIR = os.path.join(encoding.environ[b'TESTTMP'], b'sync-files')
+
+# mark the end of step [1]
+FILE_LEFT_LOCKED = os.path.join(SYNC_DIR, b'left-locked')
+# mark that step [3] is ready to run.
+FILE_RIGHT_READY_TO_LOCK = os.path.join(SYNC_DIR, b'right-ready-to-lock')
+
+# mark the end of step [2]
+FILE_LEFT_CL_DATA_WRITE = os.path.join(SYNC_DIR, b'left-data')
+# mark the end of step [4]
+FILE_LEFT_CL_NODEMAP_WRITE = os.path.join(SYNC_DIR, b'left-nodemap')
+# mark the end of step [3]
+FILE_RIGHT_CL_NODEMAP_READ = os.path.join(SYNC_DIR, b'right-nodemap')
+# mark that step [9] is read to run
+FILE_RIGHT_CL_NODEMAP_PRE_WRITE = os.path.join(
+    SYNC_DIR, b'right-pre-nodemap-write'
+)
+# mark that step [9] has run.
+FILE_RIGHT_CL_NODEMAP_POST_WRITE = os.path.join(
+    SYNC_DIR, b'right-post-nodemap-write'
+)
+# mark that step [7] is ready to run
+FILE_READER_READY = os.path.join(SYNC_DIR, b'reader-ready')
+# mark that step [7] has run
+FILE_READER_READ_DOCKET = os.path.join(SYNC_DIR, b'reader-read-docket')
+
+
+def _print(*args, **kwargs):
+    print(*args, **kwargs)
+
+
+def _role(repo):
+    """find the role associated with the process"""
+    return repo.ui.config(b'devel', b'nodemap-race.role')
+
+
+def wrap_changelog_finalize(orig, cl, tr):
+    """wrap the update of `00changelog.i` during transaction finalization
+
+    This is useful for synchronisation before or after the file is updated on disk.
+    """
+    role = getattr(tr, '_race_role', None)
+    if role == RIGHT:
+        print('right ready to write, waiting for reader')
+        testing.wait_file(FILE_READER_READY)
+        testing.write_file(FILE_RIGHT_CL_NODEMAP_PRE_WRITE)
+        testing.wait_file(FILE_READER_READ_DOCKET)
+        print('right proceeding with writing its changelog index and nodemap')
+    ret = orig(cl, tr)
+    print("finalized changelog write")
+    if role == LEFT:
+        testing.write_file(FILE_LEFT_CL_DATA_WRITE)
+    return ret
+
+
+def wrap_persist_nodemap(orig, tr, revlog, *args, **kwargs):
+    """wrap the update of `00changelog.n` and `*.nd` during tr finalization
+
+    This is useful for synchronisation before or after the files are updated on
+    disk.
+    """
+    is_cl = revlog.target[0] == KIND_CHANGELOG
+    role = getattr(tr, '_race_role', None)
+    if is_cl:
+        if role == LEFT:
+            testing.wait_file(FILE_RIGHT_CL_NODEMAP_READ)
+    if is_cl:
+        print("persisting changelog nodemap")
+        print("  new data start at", revlog._nodemap_docket.data_length)
+    ret = orig(tr, revlog, *args, **kwargs)
+    if is_cl:
+        print("persisted changelog nodemap")
+        print_nodemap_details(revlog)
+        if role == LEFT:
+            testing.write_file(FILE_LEFT_CL_NODEMAP_WRITE)
+        elif role == RIGHT:
+            testing.write_file(FILE_RIGHT_CL_NODEMAP_POST_WRITE)
+    return ret
+
+
+def print_nodemap_details(cl):
+    """print relevant information about the nodemap docket currently in memory"""
+    dkt = cl._nodemap_docket
+    print('docket-details:')
+    if dkt is None:
+        print('  <no-docket>')
+        return
+    print('  uid:        ', pycompat.sysstr(dkt.uid))
+    print('  actual-tip: ', cl.tiprev())
+    print('  tip-rev:    ', dkt.tip_rev)
+    print('  data-length:', dkt.data_length)
+
+
+def wrap_persisted_data(orig, revlog):
+    """print some information about the nodemap information we just read
+
+    Used by the <READER> process only.
+    """
+    ret = orig(revlog)
+    if ret is not None:
+        docket, data = ret
+        file_path = nodemaputil._rawdata_filepath(revlog, docket)
+        file_path = revlog.opener.join(file_path)
+        file_size = os.path.getsize(file_path)
+        print('record-data-length:', docket.data_length)
+        print('actual-data-length:', len(data))
+        print('file-actual-length:', file_size)
+    return ret
+
+
+def sync_read(orig):
+    """used by <READER> to force the race window
+
+    This make sure we read the docker from <LEFT> while reading the datafile
+    after <RIGHT> write.
+    """
+    orig()
+    testing.write_file(FILE_READER_READ_DOCKET)
+    print('reader: nodemap docket read')
+    testing.wait_file(FILE_RIGHT_CL_NODEMAP_POST_WRITE)
+
+
+def uisetup(ui):
+    class RacedRepo(localrepo.localrepository):
+        def lock(self, wait=True):
+            # make sure <RIGHT> as the "Wrong" information in memory before
+            # grabbing the lock
+            newlock = self._currentlock(self._lockref) is None
+            if newlock and _role(self) == LEFT:
+                cl = self.unfiltered().changelog
+                print_nodemap_details(cl)
+            elif newlock and _role(self) == RIGHT:
+                testing.write_file(FILE_RIGHT_READY_TO_LOCK)
+                print('nodemap-race: right side start of the locking sequence')
+                testing.wait_file(FILE_LEFT_LOCKED)
+                testing.wait_file(FILE_LEFT_CL_DATA_WRITE)
+                self.invalidate(clearfilecache=True)
+                print('nodemap-race: right side reading changelog')
+                cl = self.unfiltered().changelog
+                tiprev = cl.tiprev()
+                tip = cl.node(tiprev)
+                tiprev2 = cl.rev(tip)
+                if tiprev != tiprev2:
+                    raise RuntimeError(
+                        'bad tip -round-trip %d %d' % (tiprev, tiprev2)
+                    )
+                testing.write_file(FILE_RIGHT_CL_NODEMAP_READ)
+                print('nodemap-race: right side reading of changelog is done')
+                print_nodemap_details(cl)
+                testing.wait_file(FILE_LEFT_CL_NODEMAP_WRITE)
+                print('nodemap-race: right side ready to wait for the lock')
+            ret = super(RacedRepo, self).lock(wait=wait)
+            if newlock and _role(self) == LEFT:
+                print('nodemap-race: left side locked and ready to commit')
+                testing.write_file(FILE_LEFT_LOCKED)
+                testing.wait_file(FILE_RIGHT_READY_TO_LOCK)
+                cl = self.unfiltered().changelog
+                print_nodemap_details(cl)
+            elif newlock and _role(self) == RIGHT:
+                print('nodemap-race: right side locked and ready to commit')
+                cl = self.unfiltered().changelog
+                print_nodemap_details(cl)
+            return ret
+
+        def transaction(self, *args, **kwargs):
+            # duck punch the role on the transaction to help other pieces of code
+            tr = super(RacedRepo, self).transaction(*args, **kwargs)
+            tr._race_role = _role(self)
+            return tr
+
+    localrepo.localrepository = RacedRepo
+
+    extensions.wrapfunction(
+        nodemaputil, 'persist_nodemap', wrap_persist_nodemap
+    )
+    extensions.wrapfunction(
+        changelog.changelog, '_finalize', wrap_changelog_finalize
+    )
+
+
+def reposetup(ui, repo):
+    if _role(repo) == READER:
+        extensions.wrapfunction(
+            nodemaputil, 'persisted_data', wrap_persisted_data
+        )
+        extensions.wrapfunction(nodemaputil, 'test_race_hook_1', sync_read)
+
+        class ReaderRepo(repo.__class__):
+            @util.propertycache
+            def changelog(self):
+                print('reader ready to read the changelog, waiting for right')
+                testing.write_file(FILE_READER_READY)
+                testing.wait_file(FILE_RIGHT_CL_NODEMAP_PRE_WRITE)
+                return super(ReaderRepo, self).changelog
+
+        repo.__class__ = ReaderRepo
+
+
+@command(b'check-nodemap-race')
+def cmd_check_nodemap_race(ui, repo):
+    """Run proper <READER> access in the race Windows and check nodemap content"""
+    repo = repo.unfiltered()
+    print('reader: reading changelog')
+    cl = repo.changelog
+    print('reader: changelog read')
+    print_nodemap_details(cl)
+    tip_rev = cl.tiprev()
+    tip_node = cl.node(tip_rev)
+    print('tip-rev: ', tip_rev)
+    print('tip-node:', node.short(tip_node).decode('ascii'))
+    print('node-rev:', cl.rev(tip_node))
+    for r in cl.revs():
+        n = cl.node(r)
+        try:
+            r2 = cl.rev(n)
+        except ValueError as exc:
+            print('error while checking revision:', r)
+            print(' ', exc)
+            return 1
+        else:
+            if r2 != r:
+                print('revision %d is missing from the nodemap' % r)
+                return 1

mercurial/revlogutils/nodemap.py

@@ -1,647 +1,657 b''
 # nodemap.py - nodemap related code and utilities
 #
 # Copyright 2019 Pierre-Yves David <[e-mail]>
 # Copyright 2019 George Racinet <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import re
 import struct
 
 from ..node import hex
 
 from .. import (
     error,
     util,
 )
 from . import docket as docket_mod
 
 
 class NodeMap(dict):
     def __missing__(self, x):
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def test_race_hook_1():
+    """hook point for test
+
+    This let tests to have things happens between the docket reading and the
+    data reading"""
+    pass
+
+
 def persisted_data(revlog):
     """read the nodemap for a revlog from disk"""
     if revlog._nodemap_file is None:
         return None
     pdata = revlog.opener.tryread(revlog._nodemap_file)
     if not pdata:
         return None
     offset = 0
     (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
     if version != ONDISK_VERSION:
         return None
     offset += S_VERSION.size
     headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
     uid_size, tip_rev, data_length, data_unused, tip_node_size = headers
     offset += S_HEADER.size
     docket = NodeMapDocket(pdata[offset : offset + uid_size])
     offset += uid_size
     docket.tip_rev = tip_rev
     docket.tip_node = pdata[offset : offset + tip_node_size]
     docket.data_length = data_length
     docket.data_unused = data_unused
 
     filename = _rawdata_filepath(revlog, docket)
     use_mmap = revlog.opener.options.get(b"persistent-nodemap.mmap")
+
+    test_race_hook_1()
     try:
         with revlog.opener(filename) as fd:
             if use_mmap:
                 try:
                     data = util.buffer(util.mmapread(fd, data_length))
                 except ValueError:
                     # raised when the read file is too small
                     data = b''
             else:
                 data = fd.read(data_length)
     except (IOError, OSError) as e:
         if e.errno == errno.ENOENT:
             return None
         else:
             raise
     if len(data) < data_length:
         return None
     return docket, data
 
 
 def setup_persistent_nodemap(tr, revlog):
     """Install whatever is needed transaction side to persist a nodemap on disk
 
     (only actually persist the nodemap if this is relevant for this revlog)
     """
     if revlog._inline:
         return  # inlined revlog are too small for this to be relevant
     if revlog._nodemap_file is None:
         return  # we do not use persistent_nodemap on this revlog
 
     # we need to happen after the changelog finalization, in that use "cl-"
     callback_id = b"nm-revlog-persistent-nodemap-%s" % revlog._nodemap_file
     if tr.hasfinalize(callback_id):
         return  # no need to register again
     tr.addpending(
         callback_id, lambda tr: persist_nodemap(tr, revlog, pending=True)
     )
     tr.addfinalize(callback_id, lambda tr: persist_nodemap(tr, revlog))
 
 
 class _NoTransaction(object):
     """transaction like object to update the nodemap outside a transaction"""
 
     def __init__(self):
         self._postclose = {}
 
     def addpostclose(self, callback_id, callback_func):
         self._postclose[callback_id] = callback_func
 
     def registertmp(self, *args, **kwargs):
         pass
 
     def addbackup(self, *args, **kwargs):
         pass
 
     def add(self, *args, **kwargs):
         pass
 
     def addabort(self, *args, **kwargs):
         pass
 
     def _report(self, *args):
         pass
 
 
 def update_persistent_nodemap(revlog):
     """update the persistent nodemap right now
 
     To be used for updating the nodemap on disk outside of a normal transaction
     setup (eg, `debugupdatecache`).
     """
     if revlog._inline:
         return  # inlined revlog are too small for this to be relevant
     if revlog._nodemap_file is None:
         return  # we do not use persistent_nodemap on this revlog
 
     notr = _NoTransaction()
     persist_nodemap(notr, revlog)
     for k in sorted(notr._postclose):
         notr._postclose[k](None)
 
 
 def delete_nodemap(tr, repo, revlog):
     """Delete nodemap data on disk for a given revlog"""
     if revlog._nodemap_file is None:
         msg = "calling persist nodemap on a revlog without the feature enabled"
         raise error.ProgrammingError(msg)
     repo.svfs.unlink(revlog._nodemap_file)
 
 
 def persist_nodemap(tr, revlog, pending=False, force=False):
     """Write nodemap data on disk for a given revlog"""
     if getattr(revlog, 'filteredrevs', ()):
         raise error.ProgrammingError(
             "cannot persist nodemap of a filtered changelog"
         )
     if revlog._nodemap_file is None:
         if force:
             revlog._nodemap_file = get_nodemap_file(revlog)
         else:
             msg = "calling persist nodemap on a revlog without the feature enabled"
             raise error.ProgrammingError(msg)
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
     feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
     use_mmap = revlog.opener.options.get(b"persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
     if can_incremental and ondisk_docket is not None:
         target_docket = revlog._nodemap_docket.copy()
         (
             src_docket,
             data_changed_count,
             data,
         ) = revlog.index.nodemap_data_incremental()
         new_length = target_docket.data_length + len(data)
         new_unused = target_docket.data_unused + data_changed_count
         if src_docket != target_docket:
             data = None
         elif new_length <= (new_unused * 10):  # under 10% of unused data
             data = None
         else:
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
             tr.add(datafile, target_docket.data_length)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
                 if feed_data:
                     if use_mmap:
                         fd.seek(0)
                         new_data = fd.read(new_length)
                     else:
                         fd.flush()
                         new_data = util.buffer(util.mmapread(fd, new_length))
             target_docket.data_length = new_length
             target_docket.data_unused = new_unused
 
     if data is None:
         # otherwise fallback to a full new export
         target_docket = NodeMapDocket()
         datafile = _rawdata_filepath(revlog, target_docket)
         if util.safehasattr(revlog.index, "nodemap_data_all"):
             data = revlog.index.nodemap_data_all()
         else:
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
 
         tryunlink = revlog.opener.tryunlink
 
         def abortck(tr):
             tryunlink(datafile)
 
         callback_id = b"delete-%s" % datafile
 
         # some flavor of the transaction abort does not cleanup new file, it
         # simply empty them.
         tr.addabort(callback_id, abortck)
         with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
             if feed_data:
                 if use_mmap:
                     new_data = data
                 else:
                     fd.flush()
                     new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     target_docket.tip_node = revlog.node(target_docket.tip_rev)
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
     # store vfs
     file_path = revlog._nodemap_file
     if pending:
         file_path += b'.a'
         tr.registertmp(file_path)
     else:
         tr.addbackup(file_path)
 
     with revlog.opener(file_path, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
     if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # search for old index file in all cases, some older process might have
     # left one behind.
     olds = _other_rawdata_filepath(revlog, target_docket)
     if olds:
         realvfs = getattr(revlog, '_realopener', revlog.opener)
 
         def cleanup(tr):
             for oldfile in olds:
                 realvfs.tryunlink(oldfile)
 
         callback_id = b"revlog-cleanup-nodemap-%s" % revlog._nodemap_file
         tr.addpostclose(callback_id, cleanup)
 
 
 ### Nodemap docket file
 #
 # The nodemap data are stored on disk using 2 files:
 #
 # * a raw data files containing a persistent nodemap
 #   (see `Nodemap Trie` section)
 #
 # * a small "docket" file containing medatadata
 #
 # While the nodemap data can be multiple tens of megabytes, the "docket" is
 # small, it is easy to update it automatically or to duplicated its content
 # during a transaction.
 #
 # Multiple raw data can exist at the same time (The currently valid one and a
 # new one beind used by an in progress transaction). To accomodate this, the
 # filename hosting the raw data has a variable parts. The exact filename is
 # specified inside the "docket" file.
 #
 # The docket file contains information to find, qualify and validate the raw
 # data. Its content is currently very light, but it will expand as the on disk
 # nodemap gains the necessary features to be used in production.
 
 ONDISK_VERSION = 1
 S_VERSION = struct.Struct(">B")
 S_HEADER = struct.Struct(">BQQQQ")
 
 
 class NodeMapDocket(object):
     """metadata associated with persistent nodemap data
 
     The persistent data may come from disk or be on their way to disk.
     """
 
     def __init__(self, uid=None):
         if uid is None:
             uid = docket_mod.make_uid()
         # a unique identifier for the data file:
         #   - When new data are appended, it is preserved.
         #   - When a new data file is created, a new identifier is generated.
         self.uid = uid
         # the tipmost revision stored in the data file. This revision and all
         # revision before it are expected to be encoded in the data file.
         self.tip_rev = None
         # the node of that tipmost revision, if it mismatch the current index
         # data the docket is not valid for the current index and should be
         # discarded.
         #
         # note: this method is not perfect as some destructive operation could
         # preserve the same tip_rev + tip_node while altering lower revision.
         # However this multiple other caches have the same vulnerability (eg:
         # brancmap cache).
         self.tip_node = None
         # the size (in bytes) of the persisted data to encode the nodemap valid
         # for `tip_rev`.
         #   - data file shorter than this are corrupted,
         #   - any extra data should be ignored.
         self.data_length = None
         # the amount (in bytes) of "dead" data, still in the data file but no
         # longer used for the nodemap.
         self.data_unused = 0
 
     def copy(self):
         new = NodeMapDocket(uid=self.uid)
         new.tip_rev = self.tip_rev
         new.tip_node = self.tip_node
         new.data_length = self.data_length
         new.data_unused = self.data_unused
         return new
 
     def __cmp__(self, other):
         if self.uid < other.uid:
             return -1
         if self.uid > other.uid:
             return 1
         elif self.data_length < other.data_length:
             return -1
         elif self.data_length > other.data_length:
             return 1
         return 0
 
     def __eq__(self, other):
         return self.uid == other.uid and self.data_length == other.data_length
 
     def serialize(self):
         """return serialized bytes for a docket using the passed uid"""
         data = []
         data.append(S_VERSION.pack(ONDISK_VERSION))
         headers = (
             len(self.uid),
             self.tip_rev,
             self.data_length,
             self.data_unused,
             len(self.tip_node),
         )
         data.append(S_HEADER.pack(*headers))
         data.append(self.uid)
         data.append(self.tip_node)
         return b''.join(data)
 
 
 def _rawdata_filepath(revlog, docket):
     """The (vfs relative) nodemap's rawdata file for a given uid"""
     prefix = revlog.radix
     return b"%s-%s.nd" % (prefix, docket.uid)
 
 
 def _other_rawdata_filepath(revlog, docket):
     prefix = revlog.radix
     pattern = re.compile(br"(^|/)%s-[0-9a-f]+\.nd$" % prefix)
     new_file_path = _rawdata_filepath(revlog, docket)
     new_file_name = revlog.opener.basename(new_file_path)
     dirpath = revlog.opener.dirname(new_file_path)
     others = []
     for f in revlog.opener.listdir(dirpath):
         if pattern.match(f) and f != new_file_name:
             others.append(f)
     return others
 
 
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
 # trie. This reference implementation is write only. The python version of this
 # is not expected to be actually used, since it wont provide performance
 # improvement over existing non-persistent C implementation.
 #
 # The nodemap is persisted as Trie using 4bits-address/16-entries block. each
 # revision can be adressed using its node shortest prefix.
 #
 # The trie is stored as a sequence of block. Each block contains 16 entries
 # (signed 64bit integer, big endian). Each entry can be one of the following:
 #
 #  * value >=  0 -> index of sub-block
 #  * value == -1 -> no value
 #  * value <  -1 -> encoded revision: rev = -(value+2)
 #
 # See REV_OFFSET and _transform_rev below.
 #
 # The implementation focus on simplicity, not on performance. A Rust
 # implementation should provide a efficient version of the same binary
 # persistence. This reference python implementation is never meant to be
 # extensively use in production.
 
 
 def persistent_data(index):
     """return the persistent binary form for a nodemap for a given index"""
     trie = _build_trie(index)
     return _persist_trie(trie)
 
 
 def update_persistent_data(index, root, max_idx, last_rev):
     """return the incremental update for persistent nodemap from a given index"""
     changed_block, trie = _update_trie(index, root, last_rev)
     return (
         changed_block * S_BLOCK.size,
         _persist_trie(trie, existing_idx=max_idx),
     )
 
 
 S_BLOCK = struct.Struct(">" + ("l" * 16))
 
 NO_ENTRY = -1
 # rev 0 need to be -2 because 0 is used by block, -1 is a special value.
 REV_OFFSET = 2
 
 
 def _transform_rev(rev):
     """Return the number used to represent the rev in the tree.
 
     (or retrieve a rev number from such representation)
 
     Note that this is an involution, a function equal to its inverse (i.e.
     which gives the identity when applied to itself).
     """
     return -(rev + REV_OFFSET)
 
 
 def _to_int(hex_digit):
     """turn an hexadecimal digit into a proper integer"""
     return int(hex_digit, 16)
 
 
 class Block(dict):
     """represent a block of the Trie
 
     contains up to 16 entry indexed from 0 to 15"""
 
     def __init__(self):
         super(Block, self).__init__()
         # If this block exist on disk, here is its ID
         self.ondisk_id = None
 
     def __iter__(self):
         return iter(self.get(i) for i in range(16))
 
 
 def _build_trie(index):
     """build a nodemap trie
 
     The nodemap stores revision number for each unique prefix.
 
     Each block is a dictionary with keys in `[0, 15]`. Values are either
     another block or a revision number.
     """
     root = Block()
     for rev in range(len(index)):
         current_hex = hex(index[rev][7])
         _insert_into_block(index, 0, root, rev, current_hex)
     return root
 
 
 def _update_trie(index, root, last_rev):
     """consume"""
     changed = 0
     for rev in range(last_rev + 1, len(index)):
         current_hex = hex(index[rev][7])
         changed += _insert_into_block(index, 0, root, rev, current_hex)
     return changed, root
 
 
 def _insert_into_block(index, level, block, current_rev, current_hex):
     """insert a new revision in a block
 
     index: the index we are adding revision for
     level: the depth of the current block in the trie
     block: the block currently being considered
     current_rev: the revision number we are adding
     current_hex: the hexadecimal representation of the of that revision
     """
     changed = 1
     if block.ondisk_id is not None:
         block.ondisk_id = None
     hex_digit = _to_int(current_hex[level : level + 1])
     entry = block.get(hex_digit)
     if entry is None:
         # no entry, simply store the revision number
         block[hex_digit] = current_rev
     elif isinstance(entry, dict):
         # need to recurse to an underlying block
         changed += _insert_into_block(
             index, level + 1, entry, current_rev, current_hex
         )
     else:
         # collision with a previously unique prefix, inserting new
         # vertices to fit both entry.
         other_hex = hex(index[entry][7])
         other_rev = entry
         new = Block()
         block[hex_digit] = new
         _insert_into_block(index, level + 1, new, other_rev, other_hex)
         _insert_into_block(index, level + 1, new, current_rev, current_hex)
     return changed
 
 
 def _persist_trie(root, existing_idx=None):
     """turn a nodemap trie into persistent binary data
 
     See `_build_trie` for nodemap trie structure"""
     block_map = {}
     if existing_idx is not None:
         base_idx = existing_idx + 1
     else:
         base_idx = 0
     chunks = []
     for tn in _walk_trie(root):
         if tn.ondisk_id is not None:
             block_map[id(tn)] = tn.ondisk_id
         else:
             block_map[id(tn)] = len(chunks) + base_idx
             chunks.append(_persist_block(tn, block_map))
     return b''.join(chunks)
 
 
 def _walk_trie(block):
     """yield all the block in a trie
 
     Children blocks are always yield before their parent block.
     """
     for (__, item) in sorted(block.items()):
         if isinstance(item, dict):
             for sub_block in _walk_trie(item):
                 yield sub_block
     yield block
 
 
 def _persist_block(block_node, block_map):
     """produce persistent binary data for a single block
 
     Children block are assumed to be already persisted and present in
     block_map.
     """
     data = tuple(_to_value(v, block_map) for v in block_node)
     return S_BLOCK.pack(*data)
 
 
 def _to_value(item, block_map):
     """persist any value as an integer"""
     if item is None:
         return NO_ENTRY
     elif isinstance(item, dict):
         return block_map[id(item)]
     else:
         return _transform_rev(item)
 
 
 def parse_data(data):
     """parse parse nodemap data into a nodemap Trie"""
     if (len(data) % S_BLOCK.size) != 0:
         msg = b"nodemap data size is not a multiple of block size (%d): %d"
         raise error.Abort(msg % (S_BLOCK.size, len(data)))
     if not data:
         return Block(), None
     block_map = {}
     new_blocks = []
     for i in range(0, len(data), S_BLOCK.size):
         block = Block()
         block.ondisk_id = len(block_map)
         block_map[block.ondisk_id] = block
         block_data = data[i : i + S_BLOCK.size]
         values = S_BLOCK.unpack(block_data)
         new_blocks.append((block, values))
     for b, values in new_blocks:
         for idx, v in enumerate(values):
             if v == NO_ENTRY:
                 continue
             elif v >= 0:
                 b[idx] = block_map[v]
             else:
                 b[idx] = _transform_rev(v)
     return block, i // S_BLOCK.size
 
 
 # debug utility
 
 
 def check_data(ui, index, data):
     """verify that the provided nodemap data are valid for the given idex"""
     ret = 0
     ui.status((b"revision in index:   %d\n") % len(index))
     root, __ = parse_data(data)
     all_revs = set(_all_revisions(root))
     ui.status((b"revision in nodemap: %d\n") % len(all_revs))
     for r in range(len(index)):
         if r not in all_revs:
             msg = b"  revision missing from nodemap: %d\n" % r
             ui.write_err(msg)
             ret = 1
         else:
             all_revs.remove(r)
         nm_rev = _find_node(root, hex(index[r][7]))
         if nm_rev is None:
             msg = b"  revision node does not match any entries: %d\n" % r
             ui.write_err(msg)
             ret = 1
         elif nm_rev != r:
             msg = (
                 b"  revision node does not match the expected revision: "
                 b"%d != %d\n" % (r, nm_rev)
             )
             ui.write_err(msg)
             ret = 1
 
     if all_revs:
         for r in sorted(all_revs):
             msg = b"  extra revision in  nodemap: %d\n" % r
             ui.write_err(msg)
         ret = 1
     return ret
 
 
 def _all_revisions(root):
     """return all revisions stored in a Trie"""
     for block in _walk_trie(root):
         for v in block:
             if v is None or isinstance(v, Block):
                 continue
             yield v
 
 
 def _find_node(block, node):
     """find the revision associated with a given node"""
     entry = block.get(_to_int(node[0:1]))
     if isinstance(entry, dict):
         return _find_node(entry, node[1:])
     return entry
 
 
 def get_nodemap_file(revlog):
     if revlog._trypending:
         pending_path = revlog.radix + b".n.a"
         if revlog.opener.exists(pending_path):
             return pending_path
     return revlog.radix + b".n"

tests/test-persistent-nodemap.t

@@ -1,1099 +1,1297 b''
 ===================================
 Test the persistent on-disk nodemap
 ===================================
 
 
   $ cat << EOF >> $HGRCPATH
   > [format]
   > use-share-safe=yes
   > [extensions]
   > share=
   > EOF
 
 #if no-rust
 
   $ cat << EOF >> $HGRCPATH
   > [format]
   > use-persistent-nodemap=yes
   > [devel]
   > persistent-nodemap=yes
   > EOF
 
 #endif
 
   $ hg init test-repo --config storage.revlog.persistent-nodemap.slow-path=allow
   $ cd test-repo
 
 Check handling of the default slow-path value
 
 #if no-pure no-rust
 
   $ hg id
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
 Unlock further check (we are here to test the feature)
 
   $ cat << EOF >> $HGRCPATH
   > [storage]
   > # to avoid spamming the test
   > revlog.persistent-nodemap.slow-path=allow
   > EOF
 
 #endif
 
 #if rust
 
 Regression test for a previous bug in Rust/C FFI for the `Revlog_CAPI` capsule:
 in places where `mercurial/cext/revlog.c` function signatures use `Py_ssize_t`
 (64 bits on Linux x86_64), corresponding declarations in `rust/hg-cpython/src/cindex.rs`
 incorrectly used `libc::c_int` (32 bits).
 As a result, -1 passed from Rust for the null revision became 4294967295 in C.
 
   $ hg log -r 00000000
   changeset:   -1:000000000000
   tag:         tip
   user:        
   date:        Thu Jan 01 00:00:00 1970 +0000
   
 
 #endif
 
 
   $ hg debugformat
   format-variant     repo
   fncache:            yes
   dirstate-v2:         no
   dotencode:          yes
   generaldelta:       yes
   share-safe:         yes
   sparserevlog:       yes
   persistent-nodemap: yes
   copies-sdc:          no
   revlog-v2:           no
   changelog-v2:        no
   plain-cl-delta:     yes
   compression:        zlib (no-zstd !)
   compression:        zstd (zstd !)
   compression-level:  default
   $ hg debugbuilddag .+5000 --new-file
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5000
   tip-node: 6b02b8c7b96654c25e86ba69eda198d7e6ad8b3c
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=62
 
 Simple lookup works
 
   $ ANYNODE=`hg log --template '{node|short}\n' --rev tip`
   $ hg log -r "$ANYNODE" --template '{rev}\n'
   5000
 
 
 #if rust
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd (glob)
 
   $ f --sha256 .hg/store/00manifest-*.nd
   .hg/store/00manifest-????????.nd: sha256=97117b1c064ea2f86664a124589e47db0e254e8d34739b5c5cc5bf31c9da2b51 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=121088, sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=121088, sha256=2e029d3200bd1a986b32784fc2ef1a3bd60dc331f025718bcf5ff44d93f026fd
   0000: 00 00 00 91 00 00 00 20 00 00 00 bb 00 00 00 e7 |....... ........|
   0010: 00 00 00 66 00 00 00 a1 00 00 01 13 00 00 01 22 |...f..........."|
   0020: 00 00 00 23 00 00 00 fc 00 00 00 ba 00 00 00 5e |...#...........^|
   0030: 00 00 00 df 00 00 01 4e 00 00 01 65 00 00 00 ab |.......N...e....|
   0040: 00 00 00 a9 00 00 00 95 00 00 00 73 00 00 00 38 |...........s...8|
   0050: 00 00 00 cc 00 00 00 92 00 00 00 90 00 00 00 69 |...............i|
   0060: 00 00 00 ec 00 00 00 8d 00 00 01 4f 00 00 00 12 |...........O....|
   0070: 00 00 02 0c 00 00 00 77 00 00 00 9c 00 00 00 8f |.......w........|
   0080: 00 00 00 d5 00 00 00 6b 00 00 00 48 00 00 00 b3 |.......k...H....|
   0090: 00 00 00 e5 00 00 00 b5 00 00 00 8e 00 00 00 ad |................|
   00a0: 00 00 00 7b 00 00 00 7c 00 00 00 0b 00 00 00 2b |...{...|.......+|
   00b0: 00 00 00 c6 00 00 00 1e 00 00 01 08 00 00 00 11 |................|
   00c0: 00 00 01 30 00 00 00 26 00 00 01 9c 00 00 00 35 |...0...&.......5|
   00d0: 00 00 00 b8 00 00 01 31 00 00 00 2c 00 00 00 55 |.......1...,...U|
   00e0: 00 00 00 8a 00 00 00 9a 00 00 00 0c 00 00 01 1e |................|
   00f0: 00 00 00 a4 00 00 00 83 00 00 00 c9 00 00 00 8c |................|
 
 
 #else
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=121088, sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=121088, sha256=f544f5462ff46097432caf6d764091f6d8c46d6121be315ead8576d548c9dd79
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0020: ff ff ff ff ff ff f5 06 ff ff ff ff ff ff f3 e7 |................|
   0030: ff ff ef ca ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0040: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0050: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ed 08 |................|
   0060: ff ff ed 66 ff ff ff ff ff ff ff ff ff ff ff ff |...f............|
   0070: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0090: ff ff ff ff ff ff ff ff ff ff ff ff ff ff f6 ed |................|
   00a0: ff ff ff ff ff ff fe 61 ff ff ff ff ff ff ff ff |.......a........|
   00b0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00c0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff f1 02 |................|
   00f0: ff ff ff ff ff ff ed 1b ff ff ff ff ff ff ff ff |................|
 
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5001
   revision in nodemap: 5001
 
 add a new commit
 
   $ hg up
   5001 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo foo > foo
   $ hg add foo
 
 
 Check slow-path config value handling
 -------------------------------------
 
 #if no-pure no-rust
 
   $ hg id --config "storage.revlog.persistent-nodemap.slow-path=invalid-value"
   unknown value for config "storage.revlog.persistent-nodemap.slow-path": "invalid-value"
   falling back to default value: abort
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
   $ hg log -r . --config "storage.revlog.persistent-nodemap.slow-path=warn"
   warning: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   changeset:   5000:6b02b8c7b966
   tag:         tip
   user:        debugbuilddag
   date:        Thu Jan 01 01:23:20 1970 +0000
   summary:     r5000
   
   $ hg ci -m 'foo' --config "storage.revlog.persistent-nodemap.slow-path=abort"
   abort: accessing `persistent-nodemap` repository without associated fast implementation.
   (check `hg help config.format.use-persistent-nodemap` for details)
   [255]
 
 #else
 
   $ hg id --config "storage.revlog.persistent-nodemap.slow-path=invalid-value"
   unknown value for config "storage.revlog.persistent-nodemap.slow-path": "invalid-value"
   falling back to default value: abort
   6b02b8c7b966+ tip
 
 #endif
 
   $ hg ci -m 'foo'
 
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5001
   tip-node: 16395c3cf7e231394735e6b1717823ada303fb0c
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5001
   tip-node: 16395c3cf7e231394735e6b1717823ada303fb0c
   data-length: 121344
   data-unused: 256
   data-unused: 0.211%
 #endif
 
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=62
 
 (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
 
 #if pure
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121344, sha256=cce54c5da5bde3ad72a4938673ed4064c86231b9c64376b082b163fdb20f8f66 (glob)
 #endif
 
 #if rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121344, sha256=952b042fcf614ceb37b542b1b723e04f18f83efe99bee4e0f5ccd232ef470e58 (glob)
 #endif
 
 #if no-pure no-rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121088, sha256=df7c06a035b96cb28c7287d349d603baef43240be7736fe34eea419a49702e17 (glob)
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
 
 Test code path without mmap
 ---------------------------
 
   $ echo bar > bar
   $ hg add bar
   $ hg ci -m 'bar' --config storage.revlog.persistent-nodemap.mmap=no
 
   $ hg debugnodemap --check --config storage.revlog.persistent-nodemap.mmap=yes
   revision in index:   5003
   revision in nodemap: 5003
   $ hg debugnodemap --check --config storage.revlog.persistent-nodemap.mmap=no
   revision in index:   5003
   revision in nodemap: 5003
 
 
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121600
   data-unused: 512
   data-unused: 0.421%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121600, sha256=def52503d049ccb823974af313a98a935319ba61f40f3aa06a8be4d35c215054 (glob)
 #endif
 #if rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121600
   data-unused: 512
   data-unused: 0.421%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121600, sha256=dacf5b5f1d4585fee7527d0e67cad5b1ba0930e6a0928f650f779aefb04ce3fb (glob)
 #endif
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????.nd: size=121088, sha256=59fcede3e3cc587755916ceed29e3c33748cd1aa7d2f91828ac83e7979d935e8 (glob)
 #endif
 
 Test force warming the cache
 
   $ rm .hg/store/00changelog.n
   $ hg debugnodemap --metadata
   $ hg debugupdatecache
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 #endif
 
 Check out of sync nodemap
 =========================
 
 First copy old data on the side.
 
   $ mkdir ../tmp-copies
   $ cp .hg/store/00changelog-????????.nd .hg/store/00changelog.n ../tmp-copies
 
 Nodemap lagging behind
 ----------------------
 
 make a new commit
 
   $ echo bar2 > bar
   $ hg ci -m 'bar2'
   $ NODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 If the nodemap is lagging behind, it can catch up fine
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5003
   tip-node: c9329770f979ade2d16912267c38ba5f82fd37b3
   data-length: 121344 (pure !)
   data-length: 121344 (rust !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-rust no-pure !)
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 changelog altered
 -----------------
 
 If the nodemap is not gated behind a requirements, an unaware client can alter
 the repository so the revlog used to generate the nodemap is not longer
 compatible with the persistent nodemap. We need to detect that.
 
   $ hg up "$NODE~5"
   0 files updated, 0 files merged, 4 files removed, 0 files unresolved
   $ echo bar > babar
   $ hg add babar
   $ hg ci -m 'babar'
   created new head
   $ OTHERNODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5004
 
   $ hg --config extensions.strip= strip --rev "$NODE~1" --no-backup
 
 the nodemap should detect the changelog have been tampered with and recover.
 
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: b355ef8adce0949b8bdf6afc72ca853740d65944
   data-length: 121536 (pure !)
   data-length: 121088 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.000% (rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5002
   tip-node: 880b18d239dfa9f632413a2071bfdbcc4806a4fd
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5002
 
 missing data file
 -----------------
 
   $ UUID=`hg debugnodemap --metadata| grep 'uid:' | \
   > sed 's/uid: //'`
   $ FILE=.hg/store/00changelog-"${UUID}".nd
   $ mv $FILE ../tmp-data-file
   $ cp .hg/store/00changelog.n ../tmp-docket
 
 mercurial don't crash
 
   $ hg log -r .
   changeset:   5002:b355ef8adce0
   tag:         tip
   parent:      4998:d918ad6d18d3
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     babar
   
   $ hg debugnodemap --metadata
 
   $ hg debugupdatecache
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5002
   tip-node: b355ef8adce0949b8bdf6afc72ca853740d65944
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Sub-case: fallback for corrupted data file
 ------------------------------------------
 
 Sabotaging the data file so that nodemap resolutions fail, triggering fallback to
 (non-persistent) C implementation.
 
 
   $ UUID=`hg debugnodemap --metadata| grep 'uid:' | \
   > sed 's/uid: //'`
   $ FILE=.hg/store/00changelog-"${UUID}".nd
   $ python -c "fobj = open('$FILE', 'r+b'); fobj.write(b'\xff' * 121088); fobj.close()"
 
 The nodemap data file is still considered in sync with the docket. This
 would fail without the fallback to the (non-persistent) C implementation:
 
   $ hg log -r b355ef8adce0949b8bdf6afc72ca853740d65944 -T '{rev}\n' --traceback
   5002
 
 The nodemap data file hasn't been fixed, more tests can be inserted:
 
   $ hg debugnodemap --dump-disk | f --bytes=256 --hexdump --size
   size=121088
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0030: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0040: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0050: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0060: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0070: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0090: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00a0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00b0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00c0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00f0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
 
   $ mv ../tmp-data-file $FILE
   $ mv ../tmp-docket .hg/store/00changelog.n
 
 Check transaction related property
 ==================================
 
 An up to date nodemap should be available to shell hooks,
 
   $ echo dsljfl > a
   $ hg add a
   $ hg ci -m a
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5003
   tip-node: a52c5079765b5865d97b993b303a18740113bbb2
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
   $ echo babar2 > babar
   $ hg ci -m 'babar2' --config "hooks.pretxnclose.nodemap-test=hg debugnodemap --metadata"
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
 Another process does not see the pending nodemap content during run.
 
   $ echo qpoasp > a
   $ hg ci -m a2 \
   > --config "hooks.pretxnclose=sh \"$RUNTESTDIR/testlib/wait-on-file\" 20 sync-repo-read sync-txn-pending" \
   > --config "hooks.txnclose=touch sync-txn-close" > output.txt 2>&1 &
 
 (read the repository while the commit transaction is pending)
 
   $ sh "$RUNTESTDIR/testlib/wait-on-file" 20 sync-txn-pending && \
   > hg debugnodemap --metadata && \
   > sh "$RUNTESTDIR/testlib/wait-on-file" 20 sync-txn-close sync-repo-read
   uid: ???????? (glob)
   tip-rev: 5004
   tip-node: 2f5fb1c06a16834c5679d672e90da7c5f3b1a984
   data-length: 121280 (pure !)
   data-length: 121280 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121536 (pure !)
   data-length: 121536 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.369% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cat output.txt
 
 Check that a failing transaction will properly revert the data
 
   $ echo plakfe > a
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: size=121536, sha256=bb414468d225cf52d69132e1237afba34d4346ee2eb81b505027e6197b107f03 (glob) (pure !)
   .hg/store/00changelog-????????.nd: size=121536, sha256=909ac727bc4d1c0fda5f7bff3c620c98bd4a2967c143405a1503439e33b377da (glob) (rust !)
   .hg/store/00changelog-????????.nd: size=121088, sha256=342d36d30d86dde67d3cb6c002606c4a75bcad665595d941493845066d9c8ee0 (glob) (no-pure no-rust !)
   $ hg ci -m a3 --config "extensions.abort=$RUNTESTDIR/testlib/crash_transaction_late.py"
   transaction abort!
   rollback completed
   abort: This is a late abort
   [255]
   $ hg debugnodemap --metadata
   uid: ???????? (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121536 (pure !)
   data-length: 121536 (rust !)
   data-length: 121088 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.369% (pure !)
   data-unused: 0.369% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????.nd: size=121536, sha256=bb414468d225cf52d69132e1237afba34d4346ee2eb81b505027e6197b107f03 (glob) (pure !)
   .hg/store/00changelog-????????.nd: size=121536, sha256=909ac727bc4d1c0fda5f7bff3c620c98bd4a2967c143405a1503439e33b377da (glob) (rust !)
   .hg/store/00changelog-????????.nd: size=121088, sha256=342d36d30d86dde67d3cb6c002606c4a75bcad665595d941493845066d9c8ee0 (glob) (no-pure no-rust !)
 
 Check that removing content does not confuse the nodemap
 --------------------------------------------------------
 
 removing data with rollback
 
   $ echo aso > a
   $ hg ci -m a4
   $ hg rollback
   repository tip rolled back to revision 5005 (undo commit)
   working directory now based on revision 5005
   $ hg id -r .
   90d5d3ba2fc4 tip
 
 removing data with strip
 
   $ echo aso > a
   $ hg ci -m a4
   $ hg --config extensions.strip= strip -r . --no-backup
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg id -r . --traceback
   90d5d3ba2fc4 tip
 
+(be a good citizen and regenerate the nodemap)
+  $ hg debugupdatecaches
+  $ hg debugnodemap --metadata
+  uid: * (glob)
+  tip-rev: 5005
+  tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
+  data-length: 121088
+  data-unused: 0
+  data-unused: 0.000%
+
+Check race condition when multiple process write new data to the repository
+---------------------------------------------------------------------------
+
+In this test, we check that two writers touching the repositories will not
+overwrite each other data. This test is prompted by the existent of issue6554.
+Where a writer ended up using and outdated docket to update the repository. See
+the dedicated extension for details on the race windows and read/write schedule
+necessary to end up in this situation: testlib/persistent-nodemap-race-ext.py
+
+The issue was initially observed on a server with a high push trafic, but it
+can be reproduced using a share and two commiting process which seems simpler.
+
+The test is Rust only as the other implementation does not use the same
+read/write patterns.
+
+  $ cd ..
+
+#if rust
+
+  $ cp -R test-repo race-repo
+  $ hg share race-repo ./other-wc --config format.use-share-safe=yes
+  updating working directory
+  5001 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg debugformat -R ./race-repo | egrep 'share-safe|persistent-nodemap'
+  share-safe:         yes
+  persistent-nodemap: yes
+  $ hg debugformat -R ./other-wc/ | egrep 'share-safe|persistent-nodemap'
+  share-safe:         yes
+  persistent-nodemap: yes
+  $ hg -R ./other-wc update 'min(head())'
+  3 files updated, 0 files merged, 2 files removed, 0 files unresolved
+  $ hg -R ./race-repo debugnodemap --metadata
+  uid: 43c37dde
+  tip-rev: 5005
+  tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
+  data-length: 121088
+  data-unused: 0
+  data-unused: 0.000%
+  $ hg -R ./race-repo log -G -r 'head()'
+  @  changeset:   5005:90d5d3ba2fc4
+  |  tag:         tip
+  ~  user:        test
+     date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     a2
+  
+  o  changeset:   5001:16395c3cf7e2
+  |  user:        test
+  ~  date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     foo
+  
+  $ hg -R ./other-wc log -G -r 'head()'
+  o  changeset:   5005:90d5d3ba2fc4
+  |  tag:         tip
+  ~  user:        test
+     date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     a2
+  
+  @  changeset:   5001:16395c3cf7e2
+  |  user:        test
+  ~  date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     foo
+  
+  $ echo left-side-race > race-repo/left-side-race
+  $ hg -R ./race-repo/ add race-repo/left-side-race
+
+  $ echo right-side-race > ./other-wc/right-side-race
+  $ hg -R ./other-wc/ add ./other-wc/right-side-race
+
+  $ mkdir sync-files
+  $ mkdir outputs
+  $ (
+  >    hg -R ./race-repo/ commit -m left-side-commit \
+  >    --config "extensions.race=${RUNTESTDIR}/testlib/persistent-nodemap-race-ext.py" \
+  >    --config 'devel.nodemap-race.role=left';
+  >    touch sync-files/left-done
+  > ) > outputs/left.txt 2>&1 &
+  $ (
+  >    hg -R ./other-wc/ commit -m right-side-commit \
+  >    --config "extensions.race=${RUNTESTDIR}/testlib/persistent-nodemap-race-ext.py" \
+  >    --config 'devel.nodemap-race.role=right';
+  >    touch sync-files/right-done
+  > ) > outputs/right.txt 2>&1 &
+  $ (
+  >    hg -R ./race-repo/ check-nodemap-race \
+  >    --config "extensions.race=${RUNTESTDIR}/testlib/persistent-nodemap-race-ext.py" \
+  >    --config 'devel.nodemap-race.role=reader';
+  >    touch sync-files/reader-done
+  > ) > outputs/reader.txt 2>&1 &
+  $ sh "$RUNTESTDIR"/testlib/wait-on-file 10 sync-files/left-done
+  $ cat outputs/left.txt
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5005
+    tip-rev:     5005
+    data-length: 121088
+  nodemap-race: left side locked and ready to commit
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5005
+    tip-rev:     5005
+    data-length: 121088
+  finalized changelog write
+  persisting changelog nodemap
+    new data start at 121088
+  persisted changelog nodemap
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5006
+    tip-rev:     5006
+    data-length: 121280
+  $ sh "$RUNTESTDIR"/testlib/wait-on-file 10 sync-files/right-done
+  $ cat outputs/right.txt
+  nodemap-race: right side start of the locking sequence
+  nodemap-race: right side reading changelog
+  nodemap-race: right side reading of changelog is done
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5006
+    tip-rev:     5005
+    data-length: 121088
+  nodemap-race: right side ready to wait for the lock
+  nodemap-race: right side locked and ready to commit
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5006
+    tip-rev:     5005
+    data-length: 121088
+  right ready to write, waiting for reader
+  right proceeding with writing its changelog index and nodemap
+  finalized changelog write
+  persisting changelog nodemap
+    new data start at 121088
+  persisted changelog nodemap
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5007
+    tip-rev:     5007
+    data-length: 121472
+  $ sh "$RUNTESTDIR"/testlib/wait-on-file 10 sync-files/reader-done
+  $ cat outputs/reader.txt
+  reader: reading changelog
+  reader ready to read the changelog, waiting for right
+  reader: nodemap docket read
+  record-data-length: 121280
+  actual-data-length: 121280
+  file-actual-length: 121472
+  reader: changelog read
+  docket-details:
+    uid:         43c37dde
+    actual-tip:  5006
+    tip-rev:     5006
+    data-length: 121280
+  tip-rev:  5006
+  tip-node: 492901161367
+  node-rev: 5006
+  error while checking revision: 18 (known-bad-output !)
+    Inconsistency: Revision 5007 found in nodemap is not in revlog indexi (known-bad-output !)
+
+  $ hg -R ./race-repo log -G -r 'head()'
+  o  changeset:   5007:ac4a2abde241
+  |  tag:         tip
+  ~  parent:      5001:16395c3cf7e2
+     user:        test
+     date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     right-side-commit
+  
+  @  changeset:   5006:492901161367
+  |  user:        test
+  ~  date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     left-side-commit
+  
+  $ hg -R ./other-wc log -G -r 'head()'
+  @  changeset:   5007:ac4a2abde241
+  |  tag:         tip
+  ~  parent:      5001:16395c3cf7e2
+     user:        test
+     date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     right-side-commit
+  
+  o  changeset:   5006:492901161367
+  |  user:        test
+  ~  date:        Thu Jan 01 00:00:00 1970 +0000
+     summary:     left-side-commit
+  
+#endif
+
 Test upgrade / downgrade
 ========================
 
+  $ cd ./test-repo/
+
 downgrading
 
   $ cat << EOF >> .hg/hgrc
   > [format]
   > use-persistent-nodemap=no
   > EOF
   $ hg debugformat -v
   format-variant     repo config default
   fncache:            yes    yes     yes
   dirstate-v2:         no     no      no
   dotencode:          yes    yes     yes
   generaldelta:       yes    yes     yes
   share-safe:         yes    yes      no
   sparserevlog:       yes    yes     yes
   persistent-nodemap: yes     no      no
   copies-sdc:          no     no      no
   revlog-v2:           no     no      no
   changelog-v2:        no     no      no
   plain-cl-delta:     yes    yes     yes
   compression:        zlib   zlib    zlib (no-zstd !)
   compression:        zstd   zstd    zstd (zstd !)
   compression-level:  default default default
   $ hg debugupgraderepo --run --no-backup --quiet
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd dirstate-v2 !)
      removed: persistent-nodemap
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00manifest-*.nd (glob)
   undo.backup.00changelog.n
   undo.backup.00manifest.n
   $ hg debugnodemap --metadata
 
 
 upgrading
 
   $ cat << EOF >> .hg/hgrc
   > [format]
   > use-persistent-nodemap=yes
   > EOF
   $ hg debugformat -v
   format-variant     repo config default
   fncache:            yes    yes     yes
   dirstate-v2:         no     no      no
   dotencode:          yes    yes     yes
   generaldelta:       yes    yes     yes
   share-safe:         yes    yes      no
   sparserevlog:       yes    yes     yes
   persistent-nodemap:  no    yes      no
   copies-sdc:          no     no      no
   revlog-v2:           no     no      no
   changelog-v2:        no     no      no
   plain-cl-delta:     yes    yes     yes
   compression:        zlib   zlib    zlib (no-zstd !)
   compression:        zstd   zstd    zstd (zstd !)
   compression-level:  default default default
   $ hg debugupgraderepo --run --no-backup --quiet
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd dirstate-v2 !)
      added: persistent-nodemap
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   undo.backup.00changelog.n
   undo.backup.00manifest.n
 
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Running unrelated upgrade
 
   $ hg debugupgraderepo --run --no-backup --quiet --optimize re-delta-all
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, persistent-nodemap, revlogv1, share-safe, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, exp-dirstate-v2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd dirstate-v2 !)
   
   optimisations: re-delta-all
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ ls -1 .hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
 
   $ hg debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Persistent nodemap and local/streaming clone
 ============================================
 
   $ cd ..
 
 standard clone
 --------------
 
 The persistent nodemap should exist after a streaming clone
 
   $ hg clone --pull --quiet -U test-repo standard-clone
   $ ls -1 standard-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R standard-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 
 local clone
 ------------
 
 The persistent nodemap should exist after a streaming clone
 
   $ hg clone -U test-repo local-clone
   $ ls -1 local-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R local-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Test various corruption case
 ============================
 
 Missing datafile
 ----------------
 
 Test behavior with a missing datafile
 
   $ hg clone --quiet --pull test-repo corruption-test-repo
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog-*.nd (glob)
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
   $ rm corruption-test-repo/.hg/store/00changelog*.nd
   $ hg log -R corruption-test-repo -r .
   changeset:   5005:90d5d3ba2fc4
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     a2
   
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
 
 Truncated data file
 -------------------
 
 Test behavior with a too short datafile
 
 rebuild the missing data
   $ hg -R corruption-test-repo debugupdatecache
   $ ls -1 corruption-test-repo/.hg/store/00changelog*
   corruption-test-repo/.hg/store/00changelog-*.nd (glob)
   corruption-test-repo/.hg/store/00changelog.d
   corruption-test-repo/.hg/store/00changelog.i
   corruption-test-repo/.hg/store/00changelog.n
 
 truncate the file
 
   $ datafilepath=`ls corruption-test-repo/.hg/store/00changelog*.nd`
   $ f -s $datafilepath
   corruption-test-repo/.hg/store/00changelog-*.nd: size=121088 (glob)
   $ dd if=$datafilepath bs=1000 count=10 of=$datafilepath-tmp status=noxfer
   10+0 records in
   10+0 records out
   $ mv $datafilepath-tmp $datafilepath
   $ f -s $datafilepath
   corruption-test-repo/.hg/store/00changelog-*.nd: size=10000 (glob)
 
 Check that Mercurial reaction to this event
 
   $ hg -R corruption-test-repo log -r . --traceback
   changeset:   5005:90d5d3ba2fc4
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     a2
   
 
 
 stream clone
 ============
 
 The persistent nodemap should exist after a streaming clone
 
 Simple case
 -----------
 
 No race condition
 
   $ hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone --debug | egrep '00(changelog|manifest)'
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00manifest.d (491 KB) (zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
   $ ls -1 stream-clone/.hg/store/ | egrep '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
   00manifest-*.nd (glob)
   00manifest.n
   $ hg -R stream-clone debugnodemap --metadata
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 new data appened
 -----------------
 
 Other commit happening on the server during the stream clone
 
 setup the step-by-step stream cloning
 
   $ HG_TEST_STREAM_WALKED_FILE_1="$TESTTMP/sync_file_walked_1"
   $ export HG_TEST_STREAM_WALKED_FILE_1
   $ HG_TEST_STREAM_WALKED_FILE_2="$TESTTMP/sync_file_walked_2"
   $ export HG_TEST_STREAM_WALKED_FILE_2
   $ HG_TEST_STREAM_WALKED_FILE_3="$TESTTMP/sync_file_walked_3"
   $ export HG_TEST_STREAM_WALKED_FILE_3
   $ cat << EOF >> test-repo/.hg/hgrc
   > [extensions]
   > steps=$RUNTESTDIR/testlib/ext-stream-clone-steps.py
   > EOF
 
 Check and record file state beforehand
 
   $ f --size test-repo/.hg/store/00changelog*
   test-repo/.hg/store/00changelog-*.nd: size=121088 (glob)
   test-repo/.hg/store/00changelog.d: size=376891 (zstd !)
   test-repo/.hg/store/00changelog.d: size=368890 (no-zstd !)
   test-repo/.hg/store/00changelog.i: size=320384
   test-repo/.hg/store/00changelog.n: size=62
   $ hg -R test-repo debugnodemap --metadata | tee server-metadata.txt
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 Prepare a commit
 
   $ echo foo >> test-repo/foo
   $ hg -R test-repo/ add test-repo/foo
 
 Do a mix of clone and commit at the same time so that the file listed on disk differ at actual transfer time.
 
   $ (hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone-race-1 --debug 2>> clone-output | egrep '00(changelog|manifest)' >> clone-output; touch $HG_TEST_STREAM_WALKED_FILE_3) &
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_1
   $ hg -R test-repo/ commit -m foo
   $ touch $HG_TEST_STREAM_WALKED_FILE_2
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_3
   $ cat clone-output
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00manifest.d (491 KB) (zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
 
 Check the result state
 
   $ f --size stream-clone-race-1/.hg/store/00changelog*
   stream-clone-race-1/.hg/store/00changelog-*.nd: size=121088 (glob)
   stream-clone-race-1/.hg/store/00changelog.d: size=368890 (no-zstd !)
   stream-clone-race-1/.hg/store/00changelog.d: size=376891 (zstd !)
   stream-clone-race-1/.hg/store/00changelog.i: size=320384
   stream-clone-race-1/.hg/store/00changelog.n: size=62
 
   $ hg -R stream-clone-race-1 debugnodemap --metadata | tee client-metadata.txt
   uid: * (glob)
   tip-rev: 5005
   tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
   data-length: 121088
   data-unused: 0
   data-unused: 0.000%
 
 We get a usable nodemap, so no rewrite would be needed and the metadata should be identical
 (ie: the following diff should be empty)
 
 This isn't the case for the `no-rust` `no-pure` implementation as it use a very minimal nodemap implementation that unconditionnaly rewrite the nodemap "all the time".
 
 #if no-rust no-pure
   $ diff -u server-metadata.txt client-metadata.txt
   --- server-metadata.txt	* (glob)
   +++ client-metadata.txt	* (glob)
   @@ -1,4 +1,4 @@
   -uid: * (glob)
   +uid: * (glob)
    tip-rev: 5005
    tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
    data-length: 121088
   [1]
 #else
   $ diff -u server-metadata.txt client-metadata.txt
 #endif
 
 
 Clean up after the test.
 
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_1"
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_2"
   $ rm -f "$HG_TEST_STREAM_WALKED_FILE_3"
 
 full regeneration
 -----------------
 
 A full nodemap is generated
 
 (ideally this test would append enough data to make sure the nodemap data file
 get changed, however to make thing simpler we will force the regeneration for
 this test.
 
 Check the initial state
 
   $ f --size test-repo/.hg/store/00changelog*
   test-repo/.hg/store/00changelog-*.nd: size=121344 (glob) (rust !)
   test-repo/.hg/store/00changelog-*.nd: size=121344 (glob) (pure !)
   test-repo/.hg/store/00changelog-*.nd: size=121152 (glob) (no-rust no-pure !)
   test-repo/.hg/store/00changelog.d: size=376950 (zstd !)
   test-repo/.hg/store/00changelog.d: size=368949 (no-zstd !)
   test-repo/.hg/store/00changelog.i: size=320448
   test-repo/.hg/store/00changelog.n: size=62
   $ hg -R test-repo debugnodemap --metadata | tee server-metadata-2.txt
   uid: * (glob)
   tip-rev: 5006
   tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
   data-length: 121344 (rust !)
   data-length: 121344 (pure !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 192 (rust !)
   data-unused: 192 (pure !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.158% (rust !)
   data-unused: 0.158% (pure !)
   data-unused: 0.000% (no-rust no-pure !)
 
 Performe the mix of clone and full refresh of the nodemap, so that the files
 (and filenames) are different between listing time and actual transfer time.
 
   $ (hg clone -U --stream --config ui.ssh="\"$PYTHON\" \"$TESTDIR/dummyssh\"" ssh://user@dummy/test-repo stream-clone-race-2 --debug 2>> clone-output-2 | egrep '00(changelog|manifest)' >> clone-output-2; touch $HG_TEST_STREAM_WALKED_FILE_3) &
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_1
   $ rm test-repo/.hg/store/00changelog.n
   $ rm test-repo/.hg/store/00changelog-*.nd
   $ hg -R test-repo/ debugupdatecache
   $ touch $HG_TEST_STREAM_WALKED_FILE_2
   $ $RUNTESTDIR/testlib/wait-on-file 10 $HG_TEST_STREAM_WALKED_FILE_3
 
 (note: the stream clone code wronly pick the `undo.` files)
 
   $ cat clone-output-2
   adding [s] undo.backup.00manifest.n (62 bytes) (known-bad-output !)
   adding [s] undo.backup.00changelog.n (62 bytes) (known-bad-output !)
   adding [s] 00manifest.n (62 bytes)
   adding [s] 00manifest-*.nd (118 KB) (glob)
   adding [s] 00changelog.n (62 bytes)
   adding [s] 00changelog-*.nd (118 KB) (glob)
   adding [s] 00manifest.d (492 KB) (zstd !)
   adding [s] 00manifest.d (452 KB) (no-zstd !)
   adding [s] 00changelog.d (360 KB) (no-zstd !)
   adding [s] 00changelog.d (368 KB) (zstd !)
   adding [s] 00manifest.i (313 KB)
   adding [s] 00changelog.i (313 KB)
 
 Check the result.
 
   $ f --size stream-clone-race-2/.hg/store/00changelog*
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121344 (glob) (rust !)
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121344 (glob) (pure !)
   stream-clone-race-2/.hg/store/00changelog-*.nd: size=121152 (glob) (no-rust no-pure !)
   stream-clone-race-2/.hg/store/00changelog.d: size=376950 (zstd !)
   stream-clone-race-2/.hg/store/00changelog.d: size=368949 (no-zstd !)
   stream-clone-race-2/.hg/store/00changelog.i: size=320448
   stream-clone-race-2/.hg/store/00changelog.n: size=62
 
   $ hg -R stream-clone-race-2 debugnodemap --metadata | tee client-metadata-2.txt
   uid: * (glob)
   tip-rev: 5006
   tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
   data-length: 121344 (rust !)
   data-unused: 192 (rust !)
   data-unused: 0.158% (rust !)
   data-length: 121152 (no-rust no-pure !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.000% (no-rust no-pure !)
   data-length: 121344 (pure !)
   data-unused: 192 (pure !)
   data-unused: 0.158% (pure !)
 
 We get a usable nodemap, so no rewrite would be needed and the metadata should be identical
 (ie: the following diff should be empty)
 
 This isn't the case for the `no-rust` `no-pure` implementation as it use a very minimal nodemap implementation that unconditionnaly rewrite the nodemap "all the time".
 
 #if no-rust no-pure
   $ diff -u server-metadata-2.txt client-metadata-2.txt
   --- server-metadata-2.txt	* (glob)
   +++ client-metadata-2.txt	* (glob)
   @@ -1,4 +1,4 @@
   -uid: * (glob)
   +uid: * (glob)
    tip-rev: 5006
    tip-node: ed2ec1eef9aa2a0ec5057c51483bc148d03e810b
    data-length: 121152
   [1]
 #else
   $ diff -u server-metadata-2.txt client-metadata-2.txt
 #endif
 
 Clean up after the test
 
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_1
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_2
   $ rm -f $HG_TEST_STREAM_WALKED_FILE_3