nodemap: automatically "vacuum" the persistent nodemap when too sparse

We arbitrarily pick "10%" as the threshold.

Differential Revision: https://phab.mercurial-scm.org/D8193

mercurial/configitems.py

@@ -1,1580 +1,1579
 # configitems.py - centralized declaration of configuration option
 #
 #  Copyright 2017 Pierre-Yves David <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import functools
 import re
 
 from . import (
     encoding,
     error,
 )
 
 
 def loadconfigtable(ui, extname, configtable):
     """update config item known to the ui with the extension ones"""
     for section, items in sorted(configtable.items()):
         knownitems = ui._knownconfig.setdefault(section, itemregister())
         knownkeys = set(knownitems)
         newkeys = set(items)
         for key in sorted(knownkeys & newkeys):
             msg = b"extension '%s' overwrite config item '%s.%s'"
             msg %= (extname, section, key)
             ui.develwarn(msg, config=b'warn-config')
 
         knownitems.update(items)
 
 
 class configitem(object):
     """represent a known config item
 
     :section: the official config section where to find this item,
        :name: the official name within the section,
     :default: default value for this item,
     :alias: optional list of tuples as alternatives,
     :generic: this is a generic definition, match name using regular expression.
     """
 
     def __init__(
         self,
         section,
         name,
         default=None,
         alias=(),
         generic=False,
         priority=0,
         experimental=False,
     ):
         self.section = section
         self.name = name
         self.default = default
         self.alias = list(alias)
         self.generic = generic
         self.priority = priority
         self.experimental = experimental
         self._re = None
         if generic:
             self._re = re.compile(self.name)
 
 
 class itemregister(dict):
     """A specialized dictionary that can handle wild-card selection"""
 
     def __init__(self):
         super(itemregister, self).__init__()
         self._generics = set()
 
     def update(self, other):
         super(itemregister, self).update(other)
         self._generics.update(other._generics)
 
     def __setitem__(self, key, item):
         super(itemregister, self).__setitem__(key, item)
         if item.generic:
             self._generics.add(item)
 
     def get(self, key):
         baseitem = super(itemregister, self).get(key)
         if baseitem is not None and not baseitem.generic:
             return baseitem
 
         # search for a matching generic item
         generics = sorted(self._generics, key=(lambda x: (x.priority, x.name)))
         for item in generics:
             # we use 'match' instead of 'search' to make the matching simpler
             # for people unfamiliar with regular expression. Having the match
             # rooted to the start of the string will produce less surprising
             # result for user writing simple regex for sub-attribute.
             #
             # For example using "color\..*" match produces an unsurprising
             # result, while using search could suddenly match apparently
             # unrelated configuration that happens to contains "color."
             # anywhere. This is a tradeoff where we favor requiring ".*" on
             # some match to avoid the need to prefix most pattern with "^".
             # The "^" seems more error prone.
             if item._re.match(key):
                 return item
 
         return None
 
 
 coreitems = {}
 
 
 def _register(configtable, *args, **kwargs):
     item = configitem(*args, **kwargs)
     section = configtable.setdefault(item.section, itemregister())
     if item.name in section:
         msg = b"duplicated config item registration for '%s.%s'"
         raise error.ProgrammingError(msg % (item.section, item.name))
     section[item.name] = item
 
 
 # special value for case where the default is derived from other values
 dynamicdefault = object()
 
 # Registering actual config items
 
 
 def getitemregister(configtable):
     f = functools.partial(_register, configtable)
     # export pseudo enum as configitem.*
     f.dynamicdefault = dynamicdefault
     return f
 
 
 coreconfigitem = getitemregister(coreitems)
 
 
 def _registerdiffopts(section, configprefix=b''):
     coreconfigitem(
         section, configprefix + b'nodates', default=False,
     )
     coreconfigitem(
         section, configprefix + b'showfunc', default=False,
     )
     coreconfigitem(
         section, configprefix + b'unified', default=None,
     )
     coreconfigitem(
         section, configprefix + b'git', default=False,
     )
     coreconfigitem(
         section, configprefix + b'ignorews', default=False,
     )
     coreconfigitem(
         section, configprefix + b'ignorewsamount', default=False,
     )
     coreconfigitem(
         section, configprefix + b'ignoreblanklines', default=False,
     )
     coreconfigitem(
         section, configprefix + b'ignorewseol', default=False,
     )
     coreconfigitem(
         section, configprefix + b'nobinary', default=False,
     )
     coreconfigitem(
         section, configprefix + b'noprefix', default=False,
     )
     coreconfigitem(
         section, configprefix + b'word-diff', default=False,
     )
 
 
 coreconfigitem(
     b'alias', b'.*', default=dynamicdefault, generic=True,
 )
 coreconfigitem(
     b'auth', b'cookiefile', default=None,
 )
 _registerdiffopts(section=b'annotate')
 # bookmarks.pushing: internal hack for discovery
 coreconfigitem(
     b'bookmarks', b'pushing', default=list,
 )
 # bundle.mainreporoot: internal hack for bundlerepo
 coreconfigitem(
     b'bundle', b'mainreporoot', default=b'',
 )
 coreconfigitem(
     b'censor', b'policy', default=b'abort', experimental=True,
 )
 coreconfigitem(
     b'chgserver', b'idletimeout', default=3600,
 )
 coreconfigitem(
     b'chgserver', b'skiphash', default=False,
 )
 coreconfigitem(
     b'cmdserver', b'log', default=None,
 )
 coreconfigitem(
     b'cmdserver', b'max-log-files', default=7,
 )
 coreconfigitem(
     b'cmdserver', b'max-log-size', default=b'1 MB',
 )
 coreconfigitem(
     b'cmdserver', b'max-repo-cache', default=0, experimental=True,
 )
 coreconfigitem(
     b'cmdserver', b'message-encodings', default=list, experimental=True,
 )
 coreconfigitem(
     b'cmdserver',
     b'track-log',
     default=lambda: [b'chgserver', b'cmdserver', b'repocache'],
 )
 coreconfigitem(
     b'color', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'color', b'mode', default=b'auto',
 )
 coreconfigitem(
     b'color', b'pagermode', default=dynamicdefault,
 )
 _registerdiffopts(section=b'commands', configprefix=b'commit.interactive.')
 coreconfigitem(
     b'commands', b'commit.post-status', default=False,
 )
 coreconfigitem(
     b'commands', b'grep.all-files', default=False, experimental=True,
 )
 coreconfigitem(
     b'commands', b'merge.require-rev', default=False,
 )
 coreconfigitem(
     b'commands', b'push.require-revs', default=False,
 )
 coreconfigitem(
     b'commands', b'resolve.confirm', default=False,
 )
 coreconfigitem(
     b'commands', b'resolve.explicit-re-merge', default=False,
 )
 coreconfigitem(
     b'commands', b'resolve.mark-check', default=b'none',
 )
 _registerdiffopts(section=b'commands', configprefix=b'revert.interactive.')
 coreconfigitem(
     b'commands', b'show.aliasprefix', default=list,
 )
 coreconfigitem(
     b'commands', b'status.relative', default=False,
 )
 coreconfigitem(
     b'commands', b'status.skipstates', default=[], experimental=True,
 )
 coreconfigitem(
     b'commands', b'status.terse', default=b'',
 )
 coreconfigitem(
     b'commands', b'status.verbose', default=False,
 )
 coreconfigitem(
     b'commands', b'update.check', default=None,
 )
 coreconfigitem(
     b'commands', b'update.requiredest', default=False,
 )
 coreconfigitem(
     b'committemplate', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'convert', b'bzr.saverev', default=True,
 )
 coreconfigitem(
     b'convert', b'cvsps.cache', default=True,
 )
 coreconfigitem(
     b'convert', b'cvsps.fuzz', default=60,
 )
 coreconfigitem(
     b'convert', b'cvsps.logencoding', default=None,
 )
 coreconfigitem(
     b'convert', b'cvsps.mergefrom', default=None,
 )
 coreconfigitem(
     b'convert', b'cvsps.mergeto', default=None,
 )
 coreconfigitem(
     b'convert', b'git.committeractions', default=lambda: [b'messagedifferent'],
 )
 coreconfigitem(
     b'convert', b'git.extrakeys', default=list,
 )
 coreconfigitem(
     b'convert', b'git.findcopiesharder', default=False,
 )
 coreconfigitem(
     b'convert', b'git.remoteprefix', default=b'remote',
 )
 coreconfigitem(
     b'convert', b'git.renamelimit', default=400,
 )
 coreconfigitem(
     b'convert', b'git.saverev', default=True,
 )
 coreconfigitem(
     b'convert', b'git.similarity', default=50,
 )
 coreconfigitem(
     b'convert', b'git.skipsubmodules', default=False,
 )
 coreconfigitem(
     b'convert', b'hg.clonebranches', default=False,
 )
 coreconfigitem(
     b'convert', b'hg.ignoreerrors', default=False,
 )
 coreconfigitem(
     b'convert', b'hg.preserve-hash', default=False,
 )
 coreconfigitem(
     b'convert', b'hg.revs', default=None,
 )
 coreconfigitem(
     b'convert', b'hg.saverev', default=False,
 )
 coreconfigitem(
     b'convert', b'hg.sourcename', default=None,
 )
 coreconfigitem(
     b'convert', b'hg.startrev', default=None,
 )
 coreconfigitem(
     b'convert', b'hg.tagsbranch', default=b'default',
 )
 coreconfigitem(
     b'convert', b'hg.usebranchnames', default=True,
 )
 coreconfigitem(
     b'convert', b'ignoreancestorcheck', default=False, experimental=True,
 )
 coreconfigitem(
     b'convert', b'localtimezone', default=False,
 )
 coreconfigitem(
     b'convert', b'p4.encoding', default=dynamicdefault,
 )
 coreconfigitem(
     b'convert', b'p4.startrev', default=0,
 )
 coreconfigitem(
     b'convert', b'skiptags', default=False,
 )
 coreconfigitem(
     b'convert', b'svn.debugsvnlog', default=True,
 )
 coreconfigitem(
     b'convert', b'svn.trunk', default=None,
 )
 coreconfigitem(
     b'convert', b'svn.tags', default=None,
 )
 coreconfigitem(
     b'convert', b'svn.branches', default=None,
 )
 coreconfigitem(
     b'convert', b'svn.startrev', default=0,
 )
 coreconfigitem(
     b'debug', b'dirstate.delaywrite', default=0,
 )
 coreconfigitem(
     b'defaults', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'devel', b'all-warnings', default=False,
 )
 coreconfigitem(
     b'devel', b'bundle2.debug', default=False,
 )
 coreconfigitem(
     b'devel', b'bundle.delta', default=b'',
 )
 coreconfigitem(
     b'devel', b'cache-vfs', default=None,
 )
 coreconfigitem(
     b'devel', b'check-locks', default=False,
 )
 coreconfigitem(
     b'devel', b'check-relroot', default=False,
 )
 coreconfigitem(
     b'devel', b'default-date', default=None,
 )
 coreconfigitem(
     b'devel', b'deprec-warn', default=False,
 )
 coreconfigitem(
     b'devel', b'disableloaddefaultcerts', default=False,
 )
 coreconfigitem(
     b'devel', b'warn-empty-changegroup', default=False,
 )
 coreconfigitem(
     b'devel', b'legacy.exchange', default=list,
 )
 # TODO before getting `persistent-nodemap` out of experimental
 #
-# * regenerate a new nodemap when the unused/total ration is to high
 # * decide for a "status" of the persistent nodemap and associated location
 #   - part of the store next the revlog itself (new requirements)
 #   - part of the cache directory
 #   - part of an `index` directory
 #     (https://www.mercurial-scm.org/wiki/ComputedIndexPlan)
 # * do we want to use this for more than just changelog? if so we need:
 #   - simpler "pending" logic for them
 #   - double check the memory story (we dont want to keep all revlog in memory)
 #   - think about the naming scheme if we are in "cache"
 # * increment the version format to "1" and freeze it.
 coreconfigitem(
     b'devel', b'persistent-nodemap', default=False,
 )
 coreconfigitem(
     b'devel', b'servercafile', default=b'',
 )
 coreconfigitem(
     b'devel', b'serverexactprotocol', default=b'',
 )
 coreconfigitem(
     b'devel', b'serverrequirecert', default=False,
 )
 coreconfigitem(
     b'devel', b'strip-obsmarkers', default=True,
 )
 coreconfigitem(
     b'devel', b'warn-config', default=None,
 )
 coreconfigitem(
     b'devel', b'warn-config-default', default=None,
 )
 coreconfigitem(
     b'devel', b'user.obsmarker', default=None,
 )
 coreconfigitem(
     b'devel', b'warn-config-unknown', default=None,
 )
 coreconfigitem(
     b'devel', b'debug.copies', default=False,
 )
 coreconfigitem(
     b'devel', b'debug.extensions', default=False,
 )
 coreconfigitem(
     b'devel', b'debug.repo-filters', default=False,
 )
 coreconfigitem(
     b'devel', b'debug.peer-request', default=False,
 )
 coreconfigitem(
     b'devel', b'discovery.randomize', default=True,
 )
 _registerdiffopts(section=b'diff')
 coreconfigitem(
     b'email', b'bcc', default=None,
 )
 coreconfigitem(
     b'email', b'cc', default=None,
 )
 coreconfigitem(
     b'email', b'charsets', default=list,
 )
 coreconfigitem(
     b'email', b'from', default=None,
 )
 coreconfigitem(
     b'email', b'method', default=b'smtp',
 )
 coreconfigitem(
     b'email', b'reply-to', default=None,
 )
 coreconfigitem(
     b'email', b'to', default=None,
 )
 coreconfigitem(
     b'experimental', b'archivemetatemplate', default=dynamicdefault,
 )
 coreconfigitem(
     b'experimental', b'auto-publish', default=b'publish',
 )
 coreconfigitem(
     b'experimental', b'bundle-phases', default=False,
 )
 coreconfigitem(
     b'experimental', b'bundle2-advertise', default=True,
 )
 coreconfigitem(
     b'experimental', b'bundle2-output-capture', default=False,
 )
 coreconfigitem(
     b'experimental', b'bundle2.pushback', default=False,
 )
 coreconfigitem(
     b'experimental', b'bundle2lazylocking', default=False,
 )
 coreconfigitem(
     b'experimental', b'bundlecomplevel', default=None,
 )
 coreconfigitem(
     b'experimental', b'bundlecomplevel.bzip2', default=None,
 )
 coreconfigitem(
     b'experimental', b'bundlecomplevel.gzip', default=None,
 )
 coreconfigitem(
     b'experimental', b'bundlecomplevel.none', default=None,
 )
 coreconfigitem(
     b'experimental', b'bundlecomplevel.zstd', default=None,
 )
 coreconfigitem(
     b'experimental', b'changegroup3', default=False,
 )
 coreconfigitem(
     b'experimental', b'cleanup-as-archived', default=False,
 )
 coreconfigitem(
     b'experimental', b'clientcompressionengines', default=list,
 )
 coreconfigitem(
     b'experimental', b'copytrace', default=b'on',
 )
 coreconfigitem(
     b'experimental', b'copytrace.movecandidateslimit', default=100,
 )
 coreconfigitem(
     b'experimental', b'copytrace.sourcecommitlimit', default=100,
 )
 coreconfigitem(
     b'experimental', b'copies.read-from', default=b"filelog-only",
 )
 coreconfigitem(
     b'experimental', b'copies.write-to', default=b'filelog-only',
 )
 coreconfigitem(
     b'experimental', b'crecordtest', default=None,
 )
 coreconfigitem(
     b'experimental', b'directaccess', default=False,
 )
 coreconfigitem(
     b'experimental', b'directaccess.revnums', default=False,
 )
 coreconfigitem(
     b'experimental', b'editortmpinhg', default=False,
 )
 coreconfigitem(
     b'experimental', b'evolution', default=list,
 )
 coreconfigitem(
     b'experimental',
     b'evolution.allowdivergence',
     default=False,
     alias=[(b'experimental', b'allowdivergence')],
 )
 coreconfigitem(
     b'experimental', b'evolution.allowunstable', default=None,
 )
 coreconfigitem(
     b'experimental', b'evolution.createmarkers', default=None,
 )
 coreconfigitem(
     b'experimental',
     b'evolution.effect-flags',
     default=True,
     alias=[(b'experimental', b'effect-flags')],
 )
 coreconfigitem(
     b'experimental', b'evolution.exchange', default=None,
 )
 coreconfigitem(
     b'experimental', b'evolution.bundle-obsmarker', default=False,
 )
 coreconfigitem(
     b'experimental', b'log.topo', default=False,
 )
 coreconfigitem(
     b'experimental', b'evolution.report-instabilities', default=True,
 )
 coreconfigitem(
     b'experimental', b'evolution.track-operation', default=True,
 )
 # repo-level config to exclude a revset visibility
 #
 # The target use case is to use `share` to expose different subset of the same
 # repository, especially server side. See also `server.view`.
 coreconfigitem(
     b'experimental', b'extra-filter-revs', default=None,
 )
 coreconfigitem(
     b'experimental', b'maxdeltachainspan', default=-1,
 )
 coreconfigitem(
     b'experimental', b'mergetempdirprefix', default=None,
 )
 coreconfigitem(
     b'experimental', b'mmapindexthreshold', default=None,
 )
 coreconfigitem(
     b'experimental', b'narrow', default=False,
 )
 coreconfigitem(
     b'experimental', b'nonnormalparanoidcheck', default=False,
 )
 coreconfigitem(
     b'experimental', b'exportableenviron', default=list,
 )
 coreconfigitem(
     b'experimental', b'extendedheader.index', default=None,
 )
 coreconfigitem(
     b'experimental', b'extendedheader.similarity', default=False,
 )
 coreconfigitem(
     b'experimental', b'graphshorten', default=False,
 )
 coreconfigitem(
     b'experimental', b'graphstyle.parent', default=dynamicdefault,
 )
 coreconfigitem(
     b'experimental', b'graphstyle.missing', default=dynamicdefault,
 )
 coreconfigitem(
     b'experimental', b'graphstyle.grandparent', default=dynamicdefault,
 )
 coreconfigitem(
     b'experimental', b'hook-track-tags', default=False,
 )
 coreconfigitem(
     b'experimental', b'httppeer.advertise-v2', default=False,
 )
 coreconfigitem(
     b'experimental', b'httppeer.v2-encoder-order', default=None,
 )
 coreconfigitem(
     b'experimental', b'httppostargs', default=False,
 )
 coreconfigitem(
     b'experimental', b'mergedriver', default=None,
 )
 coreconfigitem(b'experimental', b'nointerrupt', default=False)
 coreconfigitem(b'experimental', b'nointerrupt-interactiveonly', default=True)
 
 coreconfigitem(
     b'experimental', b'obsmarkers-exchange-debug', default=False,
 )
 coreconfigitem(
     b'experimental', b'remotenames', default=False,
 )
 coreconfigitem(
     b'experimental', b'removeemptydirs', default=True,
 )
 coreconfigitem(
     b'experimental', b'revert.interactive.select-to-keep', default=False,
 )
 coreconfigitem(
     b'experimental', b'revisions.prefixhexnode', default=False,
 )
 coreconfigitem(
     b'experimental', b'revlogv2', default=None,
 )
 coreconfigitem(
     b'experimental', b'revisions.disambiguatewithin', default=None,
 )
 coreconfigitem(
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
     b'experimental', b'exp-persistent-nodemap.mmap', default=True,
 )
 coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
     b'experimental',
     b'server.manifestdata.recommended-batch-size',
     default=100000,
 )
 coreconfigitem(
     b'experimental', b'server.stream-narrow-clones', default=False,
 )
 coreconfigitem(
     b'experimental', b'single-head-per-branch', default=False,
 )
 coreconfigitem(
     b'experimental',
     b'single-head-per-branch:account-closed-heads',
     default=False,
 )
 coreconfigitem(
     b'experimental', b'sshserver.support-v2', default=False,
 )
 coreconfigitem(
     b'experimental', b'sparse-read', default=False,
 )
 coreconfigitem(
     b'experimental', b'sparse-read.density-threshold', default=0.50,
 )
 coreconfigitem(
     b'experimental', b'sparse-read.min-gap-size', default=b'65K',
 )
 coreconfigitem(
     b'experimental', b'treemanifest', default=False,
 )
 coreconfigitem(
     b'experimental', b'update.atomic-file', default=False,
 )
 coreconfigitem(
     b'experimental', b'sshpeer.advertise-v2', default=False,
 )
 coreconfigitem(
     b'experimental', b'web.apiserver', default=False,
 )
 coreconfigitem(
     b'experimental', b'web.api.http-v2', default=False,
 )
 coreconfigitem(
     b'experimental', b'web.api.debugreflect', default=False,
 )
 coreconfigitem(
     b'experimental', b'worker.wdir-get-thread-safe', default=False,
 )
 coreconfigitem(
     b'experimental', b'worker.repository-upgrade', default=False,
 )
 coreconfigitem(
     b'experimental', b'xdiff', default=False,
 )
 coreconfigitem(
     b'extensions', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'extdata', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'format', b'bookmarks-in-store', default=False,
 )
 coreconfigitem(
     b'format', b'chunkcachesize', default=None, experimental=True,
 )
 coreconfigitem(
     b'format', b'dotencode', default=True,
 )
 coreconfigitem(
     b'format', b'generaldelta', default=False, experimental=True,
 )
 coreconfigitem(
     b'format', b'manifestcachesize', default=None, experimental=True,
 )
 coreconfigitem(
     b'format', b'maxchainlen', default=dynamicdefault, experimental=True,
 )
 coreconfigitem(
     b'format', b'obsstore-version', default=None,
 )
 coreconfigitem(
     b'format', b'sparse-revlog', default=True,
 )
 coreconfigitem(
     b'format',
     b'revlog-compression',
     default=lambda: [b'zlib'],
     alias=[(b'experimental', b'format.compression')],
 )
 coreconfigitem(
     b'format', b'usefncache', default=True,
 )
 coreconfigitem(
     b'format', b'usegeneraldelta', default=True,
 )
 coreconfigitem(
     b'format', b'usestore', default=True,
 )
 coreconfigitem(
     b'format',
     b'exp-use-copies-side-data-changeset',
     default=False,
     experimental=True,
 )
 coreconfigitem(
     b'format', b'exp-use-side-data', default=False, experimental=True,
 )
 coreconfigitem(
     b'format', b'internal-phase', default=False, experimental=True,
 )
 coreconfigitem(
     b'fsmonitor', b'warn_when_unused', default=True,
 )
 coreconfigitem(
     b'fsmonitor', b'warn_update_file_count', default=50000,
 )
 coreconfigitem(
     b'help', br'hidden-command\..*', default=False, generic=True,
 )
 coreconfigitem(
     b'help', br'hidden-topic\..*', default=False, generic=True,
 )
 coreconfigitem(
     b'hooks', b'.*', default=dynamicdefault, generic=True,
 )
 coreconfigitem(
     b'hgweb-paths', b'.*', default=list, generic=True,
 )
 coreconfigitem(
     b'hostfingerprints', b'.*', default=list, generic=True,
 )
 coreconfigitem(
     b'hostsecurity', b'ciphers', default=None,
 )
 coreconfigitem(
     b'hostsecurity', b'disabletls10warning', default=False,
 )
 coreconfigitem(
     b'hostsecurity', b'minimumprotocol', default=dynamicdefault,
 )
 coreconfigitem(
     b'hostsecurity',
     b'.*:minimumprotocol$',
     default=dynamicdefault,
     generic=True,
 )
 coreconfigitem(
     b'hostsecurity', b'.*:ciphers$', default=dynamicdefault, generic=True,
 )
 coreconfigitem(
     b'hostsecurity', b'.*:fingerprints$', default=list, generic=True,
 )
 coreconfigitem(
     b'hostsecurity', b'.*:verifycertsfile$', default=None, generic=True,
 )
 
 coreconfigitem(
     b'http_proxy', b'always', default=False,
 )
 coreconfigitem(
     b'http_proxy', b'host', default=None,
 )
 coreconfigitem(
     b'http_proxy', b'no', default=list,
 )
 coreconfigitem(
     b'http_proxy', b'passwd', default=None,
 )
 coreconfigitem(
     b'http_proxy', b'user', default=None,
 )
 
 coreconfigitem(
     b'http', b'timeout', default=None,
 )
 
 coreconfigitem(
     b'logtoprocess', b'commandexception', default=None,
 )
 coreconfigitem(
     b'logtoprocess', b'commandfinish', default=None,
 )
 coreconfigitem(
     b'logtoprocess', b'command', default=None,
 )
 coreconfigitem(
     b'logtoprocess', b'develwarn', default=None,
 )
 coreconfigitem(
     b'logtoprocess', b'uiblocked', default=None,
 )
 coreconfigitem(
     b'merge', b'checkunknown', default=b'abort',
 )
 coreconfigitem(
     b'merge', b'checkignored', default=b'abort',
 )
 coreconfigitem(
     b'experimental', b'merge.checkpathconflicts', default=False,
 )
 coreconfigitem(
     b'merge', b'followcopies', default=True,
 )
 coreconfigitem(
     b'merge', b'on-failure', default=b'continue',
 )
 coreconfigitem(
     b'merge', b'preferancestor', default=lambda: [b'*'], experimental=True,
 )
 coreconfigitem(
     b'merge', b'strict-capability-check', default=False,
 )
 coreconfigitem(
     b'merge-tools', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.args$',
     default=b"$local $base $other",
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.binary$', default=False, generic=True, priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.check$', default=list, generic=True, priority=-1,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.checkchanged$',
     default=False,
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.executable$',
     default=dynamicdefault,
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.fixeol$', default=False, generic=True, priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.gui$', default=False, generic=True, priority=-1,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.mergemarkers$',
     default=b'basic',
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.mergemarkertemplate$',
     default=dynamicdefault,  # take from ui.mergemarkertemplate
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.priority$', default=0, generic=True, priority=-1,
 )
 coreconfigitem(
     b'merge-tools',
     br'.*\.premerge$',
     default=dynamicdefault,
     generic=True,
     priority=-1,
 )
 coreconfigitem(
     b'merge-tools', br'.*\.symlink$', default=False, generic=True, priority=-1,
 )
 coreconfigitem(
     b'pager', b'attend-.*', default=dynamicdefault, generic=True,
 )
 coreconfigitem(
     b'pager', b'ignore', default=list,
 )
 coreconfigitem(
     b'pager', b'pager', default=dynamicdefault,
 )
 coreconfigitem(
     b'patch', b'eol', default=b'strict',
 )
 coreconfigitem(
     b'patch', b'fuzz', default=2,
 )
 coreconfigitem(
     b'paths', b'default', default=None,
 )
 coreconfigitem(
     b'paths', b'default-push', default=None,
 )
 coreconfigitem(
     b'paths', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'phases', b'checksubrepos', default=b'follow',
 )
 coreconfigitem(
     b'phases', b'new-commit', default=b'draft',
 )
 coreconfigitem(
     b'phases', b'publish', default=True,
 )
 coreconfigitem(
     b'profiling', b'enabled', default=False,
 )
 coreconfigitem(
     b'profiling', b'format', default=b'text',
 )
 coreconfigitem(
     b'profiling', b'freq', default=1000,
 )
 coreconfigitem(
     b'profiling', b'limit', default=30,
 )
 coreconfigitem(
     b'profiling', b'nested', default=0,
 )
 coreconfigitem(
     b'profiling', b'output', default=None,
 )
 coreconfigitem(
     b'profiling', b'showmax', default=0.999,
 )
 coreconfigitem(
     b'profiling', b'showmin', default=dynamicdefault,
 )
 coreconfigitem(
     b'profiling', b'showtime', default=True,
 )
 coreconfigitem(
     b'profiling', b'sort', default=b'inlinetime',
 )
 coreconfigitem(
     b'profiling', b'statformat', default=b'hotpath',
 )
 coreconfigitem(
     b'profiling', b'time-track', default=dynamicdefault,
 )
 coreconfigitem(
     b'profiling', b'type', default=b'stat',
 )
 coreconfigitem(
     b'progress', b'assume-tty', default=False,
 )
 coreconfigitem(
     b'progress', b'changedelay', default=1,
 )
 coreconfigitem(
     b'progress', b'clear-complete', default=True,
 )
 coreconfigitem(
     b'progress', b'debug', default=False,
 )
 coreconfigitem(
     b'progress', b'delay', default=3,
 )
 coreconfigitem(
     b'progress', b'disable', default=False,
 )
 coreconfigitem(
     b'progress', b'estimateinterval', default=60.0,
 )
 coreconfigitem(
     b'progress',
     b'format',
     default=lambda: [b'topic', b'bar', b'number', b'estimate'],
 )
 coreconfigitem(
     b'progress', b'refresh', default=0.1,
 )
 coreconfigitem(
     b'progress', b'width', default=dynamicdefault,
 )
 coreconfigitem(
     b'pull', b'confirm', default=False,
 )
 coreconfigitem(
     b'push', b'pushvars.server', default=False,
 )
 coreconfigitem(
     b'rewrite',
     b'backup-bundle',
     default=True,
     alias=[(b'ui', b'history-editing-backup')],
 )
 coreconfigitem(
     b'rewrite', b'update-timestamp', default=False,
 )
 coreconfigitem(
     b'storage', b'new-repo-backend', default=b'revlogv1', experimental=True,
 )
 coreconfigitem(
     b'storage',
     b'revlog.optimize-delta-parent-choice',
     default=True,
     alias=[(b'format', b'aggressivemergedeltas')],
 )
 coreconfigitem(
     b'storage', b'revlog.reuse-external-delta', default=True,
 )
 coreconfigitem(
     b'storage', b'revlog.reuse-external-delta-parent', default=None,
 )
 coreconfigitem(
     b'storage', b'revlog.zlib.level', default=None,
 )
 coreconfigitem(
     b'storage', b'revlog.zstd.level', default=None,
 )
 coreconfigitem(
     b'server', b'bookmarks-pushkey-compat', default=True,
 )
 coreconfigitem(
     b'server', b'bundle1', default=True,
 )
 coreconfigitem(
     b'server', b'bundle1gd', default=None,
 )
 coreconfigitem(
     b'server', b'bundle1.pull', default=None,
 )
 coreconfigitem(
     b'server', b'bundle1gd.pull', default=None,
 )
 coreconfigitem(
     b'server', b'bundle1.push', default=None,
 )
 coreconfigitem(
     b'server', b'bundle1gd.push', default=None,
 )
 coreconfigitem(
     b'server',
     b'bundle2.stream',
     default=True,
     alias=[(b'experimental', b'bundle2.stream')],
 )
 coreconfigitem(
     b'server', b'compressionengines', default=list,
 )
 coreconfigitem(
     b'server', b'concurrent-push-mode', default=b'check-related',
 )
 coreconfigitem(
     b'server', b'disablefullbundle', default=False,
 )
 coreconfigitem(
     b'server', b'maxhttpheaderlen', default=1024,
 )
 coreconfigitem(
     b'server', b'pullbundle', default=False,
 )
 coreconfigitem(
     b'server', b'preferuncompressed', default=False,
 )
 coreconfigitem(
     b'server', b'streamunbundle', default=False,
 )
 coreconfigitem(
     b'server', b'uncompressed', default=True,
 )
 coreconfigitem(
     b'server', b'uncompressedallowsecret', default=False,
 )
 coreconfigitem(
     b'server', b'view', default=b'served',
 )
 coreconfigitem(
     b'server', b'validate', default=False,
 )
 coreconfigitem(
     b'server', b'zliblevel', default=-1,
 )
 coreconfigitem(
     b'server', b'zstdlevel', default=3,
 )
 coreconfigitem(
     b'share', b'pool', default=None,
 )
 coreconfigitem(
     b'share', b'poolnaming', default=b'identity',
 )
 coreconfigitem(
     b'shelve', b'maxbackups', default=10,
 )
 coreconfigitem(
     b'smtp', b'host', default=None,
 )
 coreconfigitem(
     b'smtp', b'local_hostname', default=None,
 )
 coreconfigitem(
     b'smtp', b'password', default=None,
 )
 coreconfigitem(
     b'smtp', b'port', default=dynamicdefault,
 )
 coreconfigitem(
     b'smtp', b'tls', default=b'none',
 )
 coreconfigitem(
     b'smtp', b'username', default=None,
 )
 coreconfigitem(
     b'sparse', b'missingwarning', default=True, experimental=True,
 )
 coreconfigitem(
     b'subrepos',
     b'allowed',
     default=dynamicdefault,  # to make backporting simpler
 )
 coreconfigitem(
     b'subrepos', b'hg:allowed', default=dynamicdefault,
 )
 coreconfigitem(
     b'subrepos', b'git:allowed', default=dynamicdefault,
 )
 coreconfigitem(
     b'subrepos', b'svn:allowed', default=dynamicdefault,
 )
 coreconfigitem(
     b'templates', b'.*', default=None, generic=True,
 )
 coreconfigitem(
     b'templateconfig', b'.*', default=dynamicdefault, generic=True,
 )
 coreconfigitem(
     b'trusted', b'groups', default=list,
 )
 coreconfigitem(
     b'trusted', b'users', default=list,
 )
 coreconfigitem(
     b'ui', b'_usedassubrepo', default=False,
 )
 coreconfigitem(
     b'ui', b'allowemptycommit', default=False,
 )
 coreconfigitem(
     b'ui', b'archivemeta', default=True,
 )
 coreconfigitem(
     b'ui', b'askusername', default=False,
 )
 coreconfigitem(
     b'ui', b'clonebundlefallback', default=False,
 )
 coreconfigitem(
     b'ui', b'clonebundleprefers', default=list,
 )
 coreconfigitem(
     b'ui', b'clonebundles', default=True,
 )
 coreconfigitem(
     b'ui', b'color', default=b'auto',
 )
 coreconfigitem(
     b'ui', b'commitsubrepos', default=False,
 )
 coreconfigitem(
     b'ui', b'debug', default=False,
 )
 coreconfigitem(
     b'ui', b'debugger', default=None,
 )
 coreconfigitem(
     b'ui', b'editor', default=dynamicdefault,
 )
 coreconfigitem(
     b'ui', b'fallbackencoding', default=None,
 )
 coreconfigitem(
     b'ui', b'forcecwd', default=None,
 )
 coreconfigitem(
     b'ui', b'forcemerge', default=None,
 )
 coreconfigitem(
     b'ui', b'formatdebug', default=False,
 )
 coreconfigitem(
     b'ui', b'formatjson', default=False,
 )
 coreconfigitem(
     b'ui', b'formatted', default=None,
 )
 coreconfigitem(
     b'ui', b'graphnodetemplate', default=None,
 )
 coreconfigitem(
     b'ui', b'interactive', default=None,
 )
 coreconfigitem(
     b'ui', b'interface', default=None,
 )
 coreconfigitem(
     b'ui', b'interface.chunkselector', default=None,
 )
 coreconfigitem(
     b'ui', b'large-file-limit', default=10000000,
 )
 coreconfigitem(
     b'ui', b'logblockedtimes', default=False,
 )
 coreconfigitem(
     b'ui', b'logtemplate', default=None,
 )
 coreconfigitem(
     b'ui', b'merge', default=None,
 )
 coreconfigitem(
     b'ui', b'mergemarkers', default=b'basic',
 )
 coreconfigitem(
     b'ui',
     b'mergemarkertemplate',
     default=(
         b'{node|short} '
         b'{ifeq(tags, "tip", "", '
         b'ifeq(tags, "", "", "{tags} "))}'
         b'{if(bookmarks, "{bookmarks} ")}'
         b'{ifeq(branch, "default", "", "{branch} ")}'
         b'- {author|user}: {desc|firstline}'
     ),
 )
 coreconfigitem(
     b'ui', b'message-output', default=b'stdio',
 )
 coreconfigitem(
     b'ui', b'nontty', default=False,
 )
 coreconfigitem(
     b'ui', b'origbackuppath', default=None,
 )
 coreconfigitem(
     b'ui', b'paginate', default=True,
 )
 coreconfigitem(
     b'ui', b'patch', default=None,
 )
 coreconfigitem(
     b'ui', b'pre-merge-tool-output-template', default=None,
 )
 coreconfigitem(
     b'ui', b'portablefilenames', default=b'warn',
 )
 coreconfigitem(
     b'ui', b'promptecho', default=False,
 )
 coreconfigitem(
     b'ui', b'quiet', default=False,
 )
 coreconfigitem(
     b'ui', b'quietbookmarkmove', default=False,
 )
 coreconfigitem(
     b'ui', b'relative-paths', default=b'legacy',
 )
 coreconfigitem(
     b'ui', b'remotecmd', default=b'hg',
 )
 coreconfigitem(
     b'ui', b'report_untrusted', default=True,
 )
 coreconfigitem(
     b'ui', b'rollback', default=True,
 )
 coreconfigitem(
     b'ui', b'signal-safe-lock', default=True,
 )
 coreconfigitem(
     b'ui', b'slash', default=False,
 )
 coreconfigitem(
     b'ui', b'ssh', default=b'ssh',
 )
 coreconfigitem(
     b'ui', b'ssherrorhint', default=None,
 )
 coreconfigitem(
     b'ui', b'statuscopies', default=False,
 )
 coreconfigitem(
     b'ui', b'strict', default=False,
 )
 coreconfigitem(
     b'ui', b'style', default=b'',
 )
 coreconfigitem(
     b'ui', b'supportcontact', default=None,
 )
 coreconfigitem(
     b'ui', b'textwidth', default=78,
 )
 coreconfigitem(
     b'ui', b'timeout', default=b'600',
 )
 coreconfigitem(
     b'ui', b'timeout.warn', default=0,
 )
 coreconfigitem(
     b'ui', b'traceback', default=False,
 )
 coreconfigitem(
     b'ui', b'tweakdefaults', default=False,
 )
 coreconfigitem(b'ui', b'username', alias=[(b'ui', b'user')])
 coreconfigitem(
     b'ui', b'verbose', default=False,
 )
 coreconfigitem(
     b'verify', b'skipflags', default=None,
 )
 coreconfigitem(
     b'web', b'allowbz2', default=False,
 )
 coreconfigitem(
     b'web', b'allowgz', default=False,
 )
 coreconfigitem(
     b'web', b'allow-pull', alias=[(b'web', b'allowpull')], default=True,
 )
 coreconfigitem(
     b'web', b'allow-push', alias=[(b'web', b'allow_push')], default=list,
 )
 coreconfigitem(
     b'web', b'allowzip', default=False,
 )
 coreconfigitem(
     b'web', b'archivesubrepos', default=False,
 )
 coreconfigitem(
     b'web', b'cache', default=True,
 )
 coreconfigitem(
     b'web', b'comparisoncontext', default=5,
 )
 coreconfigitem(
     b'web', b'contact', default=None,
 )
 coreconfigitem(
     b'web', b'deny_push', default=list,
 )
 coreconfigitem(
     b'web', b'guessmime', default=False,
 )
 coreconfigitem(
     b'web', b'hidden', default=False,
 )
 coreconfigitem(
     b'web', b'labels', default=list,
 )
 coreconfigitem(
     b'web', b'logoimg', default=b'hglogo.png',
 )
 coreconfigitem(
     b'web', b'logourl', default=b'https://mercurial-scm.org/',
 )
 coreconfigitem(
     b'web', b'accesslog', default=b'-',
 )
 coreconfigitem(
     b'web', b'address', default=b'',
 )
 coreconfigitem(
     b'web', b'allow-archive', alias=[(b'web', b'allow_archive')], default=list,
 )
 coreconfigitem(
     b'web', b'allow_read', default=list,
 )
 coreconfigitem(
     b'web', b'baseurl', default=None,
 )
 coreconfigitem(
     b'web', b'cacerts', default=None,
 )
 coreconfigitem(
     b'web', b'certificate', default=None,
 )
 coreconfigitem(
     b'web', b'collapse', default=False,
 )
 coreconfigitem(
     b'web', b'csp', default=None,
 )
 coreconfigitem(
     b'web', b'deny_read', default=list,
 )
 coreconfigitem(
     b'web', b'descend', default=True,
 )
 coreconfigitem(
     b'web', b'description', default=b"",
 )
 coreconfigitem(
     b'web', b'encoding', default=lambda: encoding.encoding,
 )
 coreconfigitem(
     b'web', b'errorlog', default=b'-',
 )
 coreconfigitem(
     b'web', b'ipv6', default=False,
 )
 coreconfigitem(
     b'web', b'maxchanges', default=10,
 )
 coreconfigitem(
     b'web', b'maxfiles', default=10,
 )
 coreconfigitem(
     b'web', b'maxshortchanges', default=60,
 )
 coreconfigitem(
     b'web', b'motd', default=b'',
 )
 coreconfigitem(
     b'web', b'name', default=dynamicdefault,
 )
 coreconfigitem(
     b'web', b'port', default=8000,
 )
 coreconfigitem(
     b'web', b'prefix', default=b'',
 )
 coreconfigitem(
     b'web', b'push_ssl', default=True,
 )
 coreconfigitem(
     b'web', b'refreshinterval', default=20,
 )
 coreconfigitem(
     b'web', b'server-header', default=None,
 )
 coreconfigitem(
     b'web', b'static', default=None,
 )
 coreconfigitem(
     b'web', b'staticurl', default=None,
 )
 coreconfigitem(
     b'web', b'stripes', default=1,
 )
 coreconfigitem(
     b'web', b'style', default=b'paper',
 )
 coreconfigitem(
     b'web', b'templates', default=None,
 )
 coreconfigitem(
     b'web', b'view', default=b'served', experimental=True,
 )
 coreconfigitem(
     b'worker', b'backgroundclose', default=dynamicdefault,
 )
 # Windows defaults to a limit of 512 open files. A buffer of 128
 # should give us enough headway.
 coreconfigitem(
     b'worker', b'backgroundclosemaxqueue', default=384,
 )
 coreconfigitem(
     b'worker', b'backgroundcloseminfilecount', default=2048,
 )
 coreconfigitem(
     b'worker', b'backgroundclosethreadcount', default=4,
 )
 coreconfigitem(
     b'worker', b'enabled', default=True,
 )
 coreconfigitem(
     b'worker', b'numcpus', default=None,
 )
 
 # Rebase related configuration moved to core because other extension are doing
 # strange things. For example, shelve import the extensions to reuse some bit
 # without formally loading it.
 coreconfigitem(
     b'commands', b'rebase.requiredest', default=False,
 )
 coreconfigitem(
     b'experimental', b'rebaseskipobsolete', default=True,
 )
 coreconfigitem(
     b'rebase', b'singletransaction', default=False,
 )
 coreconfigitem(
     b'rebase', b'experimental.inmemory', default=False,
 )

mercurial/revlogutils/nodemap.py

@@ -1,625 +1,628
 # nodemap.py - nodemap related code and utilities
 #
 # Copyright 2019 Pierre-Yves David <[e-mail]>
 # Copyright 2019 George Racinet <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import os
 import re
 import struct
 
 from .. import (
     error,
     node as nodemod,
     util,
 )
 
 
 class NodeMap(dict):
     def __missing__(self, x):
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
 def persisted_data(revlog):
     """read the nodemap for a revlog from disk"""
     if revlog.nodemap_file is None:
         return None
     pdata = revlog.opener.tryread(revlog.nodemap_file)
     if not pdata:
         return None
     offset = 0
     (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size])
     if version != ONDISK_VERSION:
         return None
     offset += S_VERSION.size
     headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size])
     uid_size, tip_rev, data_length, data_unused, tip_node_size = headers
     offset += S_HEADER.size
     docket = NodeMapDocket(pdata[offset : offset + uid_size])
     offset += uid_size
     docket.tip_rev = tip_rev
     docket.tip_node = pdata[offset : offset + tip_node_size]
     docket.data_length = data_length
     docket.data_unused = data_unused
 
     filename = _rawdata_filepath(revlog, docket)
     use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
     try:
         with revlog.opener(filename) as fd:
             if use_mmap:
                 data = util.buffer(util.mmapread(fd, data_length))
             else:
                 data = fd.read(data_length)
     except OSError as e:
         if e.errno != errno.ENOENT:
             raise
     if len(data) < data_length:
         return None
     return docket, data
 
 
 def setup_persistent_nodemap(tr, revlog):
     """Install whatever is needed transaction side to persist a nodemap on disk
 
     (only actually persist the nodemap if this is relevant for this revlog)
     """
     if revlog._inline:
         return  # inlined revlog are too small for this to be relevant
     if revlog.nodemap_file is None:
         return  # we do not use persistent_nodemap on this revlog
 
     # we need to happen after the changelog finalization, in that use "cl-"
     callback_id = b"nm-revlog-persistent-nodemap-%s" % revlog.nodemap_file
     if tr.hasfinalize(callback_id):
         return  # no need to register again
     tr.addpending(
         callback_id, lambda tr: _persist_nodemap(tr, revlog, pending=True)
     )
     tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
 
 
 class _NoTransaction(object):
     """transaction like object to update the nodemap outside a transaction
     """
 
     def __init__(self):
         self._postclose = {}
 
     def addpostclose(self, callback_id, callback_func):
         self._postclose[callback_id] = callback_func
 
     def registertmp(self, *args, **kwargs):
         pass
 
     def addbackup(self, *args, **kwargs):
         pass
 
     def add(self, *args, **kwargs):
         pass
 
     def addabort(self, *args, **kwargs):
         pass
 
 
 def update_persistent_nodemap(revlog):
     """update the persistent nodemap right now
 
     To be used for updating the nodemap on disk outside of a normal transaction
     setup (eg, `debugupdatecache`).
     """
     notr = _NoTransaction()
     _persist_nodemap(notr, revlog)
     for k in sorted(notr._postclose):
         notr._postclose[k](None)
 
 
 def _persist_nodemap(tr, revlog, pending=False):
     """Write nodemap data on disk for a given revlog
     """
     if getattr(revlog, 'filteredrevs', ()):
         raise error.ProgrammingError(
             "cannot persist nodemap of a filtered changelog"
         )
     if revlog.nodemap_file is None:
         msg = "calling persist nodemap on a revlog without the feature enableb"
         raise error.ProgrammingError(msg)
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
     feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
     use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
     if can_incremental and ondisk_docket is not None:
         target_docket = revlog._nodemap_docket.copy()
         (
             src_docket,
             data_changed_count,
             data,
         ) = revlog.index.nodemap_data_incremental()
+        new_length = target_docket.data_length + len(data)
+        new_unused = target_docket.data_unused + data_changed_count
         if src_docket != target_docket:
             data = None
+        elif new_length <= (new_unused * 10):  # under 10% of unused data
+            data = None
         else:
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
-            new_length = target_docket.data_length + len(data)
             tr.add(datafile, target_docket.data_length)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
                 if feed_data:
                     if use_mmap:
                         fd.seek(0)
                         new_data = fd.read(new_length)
                     else:
                         fd.flush()
                         new_data = util.buffer(util.mmapread(fd, new_length))
             target_docket.data_length = new_length
-            target_docket.data_unused += data_changed_count
+            target_docket.data_unused = new_unused
 
     if data is None:
         # otherwise fallback to a full new export
         target_docket = NodeMapDocket()
         datafile = _rawdata_filepath(revlog, target_docket)
         if util.safehasattr(revlog.index, "nodemap_data_all"):
             data = revlog.index.nodemap_data_all()
         else:
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
 
         tryunlink = revlog.opener.tryunlink
 
         def abortck(tr):
             tryunlink(datafile)
 
         callback_id = b"delete-%s" % datafile
 
         # some flavor of the transaction abort does not cleanup new file, it
         # simply empty them.
         tr.addabort(callback_id, abortck)
         with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
             if feed_data:
                 if use_mmap:
                     new_data = data
                 else:
                     fd.flush()
                     new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     target_docket.tip_node = revlog.node(target_docket.tip_rev)
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
     # store vfs
     file_path = revlog.nodemap_file
     if pending:
         file_path += b'.a'
         tr.registertmp(file_path)
     else:
         tr.addbackup(file_path)
 
     with revlog.opener(file_path, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
     if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # search for old index file in all cases, some older process might have
     # left one behind.
     olds = _other_rawdata_filepath(revlog, target_docket)
     if olds:
         realvfs = getattr(revlog, '_realopener', revlog.opener)
 
         def cleanup(tr):
             for oldfile in olds:
                 realvfs.tryunlink(oldfile)
 
         callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
         tr.addpostclose(callback_id, cleanup)
 
 
 ### Nodemap docket file
 #
 # The nodemap data are stored on disk using 2 files:
 #
 # * a raw data files containing a persistent nodemap
 #   (see `Nodemap Trie` section)
 #
 # * a small "docket" file containing medatadata
 #
 # While the nodemap data can be multiple tens of megabytes, the "docket" is
 # small, it is easy to update it automatically or to duplicated its content
 # during a transaction.
 #
 # Multiple raw data can exist at the same time (The currently valid one and a
 # new one beind used by an in progress transaction). To accomodate this, the
 # filename hosting the raw data has a variable parts. The exact filename is
 # specified inside the "docket" file.
 #
 # The docket file contains information to find, qualify and validate the raw
 # data. Its content is currently very light, but it will expand as the on disk
 # nodemap gains the necessary features to be used in production.
 
 # version 0 is experimental, no BC garantee, do no use outside of tests.
 ONDISK_VERSION = 0
 S_VERSION = struct.Struct(">B")
 S_HEADER = struct.Struct(">BQQQQ")
 
 ID_SIZE = 8
 
 
 def _make_uid():
     """return a new unique identifier.
 
     The identifier is random and composed of ascii characters."""
     return nodemod.hex(os.urandom(ID_SIZE))
 
 
 class NodeMapDocket(object):
     """metadata associated with persistent nodemap data
 
     The persistent data may come from disk or be on their way to disk.
     """
 
     def __init__(self, uid=None):
         if uid is None:
             uid = _make_uid()
         # a unique identifier for the data file:
         #   - When new data are appended, it is preserved.
         #   - When a new data file is created, a new identifier is generated.
         self.uid = uid
         # the tipmost revision stored in the data file. This revision and all
         # revision before it are expected to be encoded in the data file.
         self.tip_rev = None
         # the node of that tipmost revision, if it mismatch the current index
         # data the docket is not valid for the current index and should be
         # discarded.
         #
         # note: this method is not perfect as some destructive operation could
         # preserve the same tip_rev + tip_node while altering lower revision.
         # However this multiple other caches have the same vulnerability (eg:
         # brancmap cache).
         self.tip_node = None
         # the size (in bytes) of the persisted data to encode the nodemap valid
         # for `tip_rev`.
         #   - data file shorter than this are corrupted,
         #   - any extra data should be ignored.
         self.data_length = None
         # the amount (in bytes) of "dead" data, still in the data file but no
         # longer used for the nodemap.
         self.data_unused = 0
 
     def copy(self):
         new = NodeMapDocket(uid=self.uid)
         new.tip_rev = self.tip_rev
         new.tip_node = self.tip_node
         new.data_length = self.data_length
         new.data_unused = self.data_unused
         return new
 
     def __cmp__(self, other):
         if self.uid < other.uid:
             return -1
         if self.uid > other.uid:
             return 1
         elif self.data_length < other.data_length:
             return -1
         elif self.data_length > other.data_length:
             return 1
         return 0
 
     def __eq__(self, other):
         return self.uid == other.uid and self.data_length == other.data_length
 
     def serialize(self):
         """return serialized bytes for a docket using the passed uid"""
         data = []
         data.append(S_VERSION.pack(ONDISK_VERSION))
         headers = (
             len(self.uid),
             self.tip_rev,
             self.data_length,
             self.data_unused,
             len(self.tip_node),
         )
         data.append(S_HEADER.pack(*headers))
         data.append(self.uid)
         data.append(self.tip_node)
         return b''.join(data)
 
 
 def _rawdata_filepath(revlog, docket):
     """The (vfs relative) nodemap's rawdata file for a given uid"""
     if revlog.nodemap_file.endswith(b'.n.a'):
         prefix = revlog.nodemap_file[:-4]
     else:
         prefix = revlog.nodemap_file[:-2]
     return b"%s-%s.nd" % (prefix, docket.uid)
 
 
 def _other_rawdata_filepath(revlog, docket):
     prefix = revlog.nodemap_file[:-2]
     pattern = re.compile(br"(^|/)%s-[0-9a-f]+\.nd$" % prefix)
     new_file_path = _rawdata_filepath(revlog, docket)
     new_file_name = revlog.opener.basename(new_file_path)
     dirpath = revlog.opener.dirname(new_file_path)
     others = []
     for f in revlog.opener.listdir(dirpath):
         if pattern.match(f) and f != new_file_name:
             others.append(f)
     return others
 
 
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
 # trie. This reference implementation is write only. The python version of this
 # is not expected to be actually used, since it wont provide performance
 # improvement over existing non-persistent C implementation.
 #
 # The nodemap is persisted as Trie using 4bits-address/16-entries block. each
 # revision can be adressed using its node shortest prefix.
 #
 # The trie is stored as a sequence of block. Each block contains 16 entries
 # (signed 64bit integer, big endian). Each entry can be one of the following:
 #
 #  * value >=  0 -> index of sub-block
 #  * value == -1 -> no value
 #  * value <  -1 -> a revision value: rev = -(value+10)
 #
 # The implementation focus on simplicity, not on performance. A Rust
 # implementation should provide a efficient version of the same binary
 # persistence. This reference python implementation is never meant to be
 # extensively use in production.
 
 
 def persistent_data(index):
     """return the persistent binary form for a nodemap for a given index
     """
     trie = _build_trie(index)
     return _persist_trie(trie)
 
 
 def update_persistent_data(index, root, max_idx, last_rev):
     """return the incremental update for persistent nodemap from a given index
     """
     changed_block, trie = _update_trie(index, root, last_rev)
     return (
         changed_block * S_BLOCK.size,
         _persist_trie(trie, existing_idx=max_idx),
     )
 
 
 S_BLOCK = struct.Struct(">" + ("l" * 16))
 
 NO_ENTRY = -1
 # rev 0 need to be -2 because 0 is used by block, -1 is a special value.
 REV_OFFSET = 2
 
 
 def _transform_rev(rev):
     """Return the number used to represent the rev in the tree.
 
     (or retrieve a rev number from such representation)
 
     Note that this is an involution, a function equal to its inverse (i.e.
     which gives the identity when applied to itself).
     """
     return -(rev + REV_OFFSET)
 
 
 def _to_int(hex_digit):
     """turn an hexadecimal digit into a proper integer"""
     return int(hex_digit, 16)
 
 
 class Block(dict):
     """represent a block of the Trie
 
     contains up to 16 entry indexed from 0 to 15"""
 
     def __init__(self):
         super(Block, self).__init__()
         # If this block exist on disk, here is its ID
         self.ondisk_id = None
 
     def __iter__(self):
         return iter(self.get(i) for i in range(16))
 
 
 def _build_trie(index):
     """build a nodemap trie
 
     The nodemap stores revision number for each unique prefix.
 
     Each block is a dictionary with keys in `[0, 15]`. Values are either
     another block or a revision number.
     """
     root = Block()
     for rev in range(len(index)):
         hex = nodemod.hex(index[rev][7])
         _insert_into_block(index, 0, root, rev, hex)
     return root
 
 
 def _update_trie(index, root, last_rev):
     """consume"""
     changed = 0
     for rev in range(last_rev + 1, len(index)):
         hex = nodemod.hex(index[rev][7])
         changed += _insert_into_block(index, 0, root, rev, hex)
     return changed, root
 
 
 def _insert_into_block(index, level, block, current_rev, current_hex):
     """insert a new revision in a block
 
     index: the index we are adding revision for
     level: the depth of the current block in the trie
     block: the block currently being considered
     current_rev: the revision number we are adding
     current_hex: the hexadecimal representation of the of that revision
     """
     changed = 1
     if block.ondisk_id is not None:
         block.ondisk_id = None
     hex_digit = _to_int(current_hex[level : level + 1])
     entry = block.get(hex_digit)
     if entry is None:
         # no entry, simply store the revision number
         block[hex_digit] = current_rev
     elif isinstance(entry, dict):
         # need to recurse to an underlying block
         changed += _insert_into_block(
             index, level + 1, entry, current_rev, current_hex
         )
     else:
         # collision with a previously unique prefix, inserting new
         # vertices to fit both entry.
         other_hex = nodemod.hex(index[entry][7])
         other_rev = entry
         new = Block()
         block[hex_digit] = new
         _insert_into_block(index, level + 1, new, other_rev, other_hex)
         _insert_into_block(index, level + 1, new, current_rev, current_hex)
     return changed
 
 
 def _persist_trie(root, existing_idx=None):
     """turn a nodemap trie into persistent binary data
 
     See `_build_trie` for nodemap trie structure"""
     block_map = {}
     if existing_idx is not None:
         base_idx = existing_idx + 1
     else:
         base_idx = 0
     chunks = []
     for tn in _walk_trie(root):
         if tn.ondisk_id is not None:
             block_map[id(tn)] = tn.ondisk_id
         else:
             block_map[id(tn)] = len(chunks) + base_idx
             chunks.append(_persist_block(tn, block_map))
     return b''.join(chunks)
 
 
 def _walk_trie(block):
     """yield all the block in a trie
 
     Children blocks are always yield before their parent block.
     """
     for (_, item) in sorted(block.items()):
         if isinstance(item, dict):
             for sub_block in _walk_trie(item):
                 yield sub_block
     yield block
 
 
 def _persist_block(block_node, block_map):
     """produce persistent binary data for a single block
 
     Children block are assumed to be already persisted and present in
     block_map.
     """
     data = tuple(_to_value(v, block_map) for v in block_node)
     return S_BLOCK.pack(*data)
 
 
 def _to_value(item, block_map):
     """persist any value as an integer"""
     if item is None:
         return NO_ENTRY
     elif isinstance(item, dict):
         return block_map[id(item)]
     else:
         return _transform_rev(item)
 
 
 def parse_data(data):
     """parse parse nodemap data into a nodemap Trie"""
     if (len(data) % S_BLOCK.size) != 0:
         msg = "nodemap data size is not a multiple of block size (%d): %d"
         raise error.Abort(msg % (S_BLOCK.size, len(data)))
     if not data:
         return Block(), None
     block_map = {}
     new_blocks = []
     for i in range(0, len(data), S_BLOCK.size):
         block = Block()
         block.ondisk_id = len(block_map)
         block_map[block.ondisk_id] = block
         block_data = data[i : i + S_BLOCK.size]
         values = S_BLOCK.unpack(block_data)
         new_blocks.append((block, values))
     for b, values in new_blocks:
         for idx, v in enumerate(values):
             if v == NO_ENTRY:
                 continue
             elif v >= 0:
                 b[idx] = block_map[v]
             else:
                 b[idx] = _transform_rev(v)
     return block, i // S_BLOCK.size
 
 
 # debug utility
 
 
 def check_data(ui, index, data):
     """verify that the provided nodemap data are valid for the given idex"""
     ret = 0
     ui.status((b"revision in index:   %d\n") % len(index))
     root, __ = parse_data(data)
     all_revs = set(_all_revisions(root))
     ui.status((b"revision in nodemap: %d\n") % len(all_revs))
     for r in range(len(index)):
         if r not in all_revs:
             msg = b"  revision missing from nodemap: %d\n" % r
             ui.write_err(msg)
             ret = 1
         else:
             all_revs.remove(r)
         nm_rev = _find_node(root, nodemod.hex(index[r][7]))
         if nm_rev is None:
             msg = b"  revision node does not match any entries: %d\n" % r
             ui.write_err(msg)
             ret = 1
         elif nm_rev != r:
             msg = (
                 b"  revision node does not match the expected revision: "
                 b"%d != %d\n" % (r, nm_rev)
             )
             ui.write_err(msg)
             ret = 1
 
     if all_revs:
         for r in sorted(all_revs):
             msg = b"  extra revision in  nodemap: %d\n" % r
             ui.write_err(msg)
         ret = 1
     return ret
 
 
 def _all_revisions(root):
     """return all revisions stored in a Trie"""
     for block in _walk_trie(root):
         for v in block:
             if v is None or isinstance(v, Block):
                 continue
             yield v
 
 
 def _find_node(block, node):
     """find the revision associated with a given node"""
     entry = block.get(_to_int(node[0:1]))
     if isinstance(entry, dict):
         return _find_node(entry, node[1:])
     return entry

tests/test-persistent-nodemap.t

@@ -1,413 +1,413
 ===================================
 Test the persistent on-disk nodemap
 ===================================
 
   $ hg init test-repo
   $ cd test-repo
   $ cat << EOF >> .hg/hgrc
   > [experimental]
   > exp-persistent-nodemap=yes
   > [devel]
   > persistent-nodemap=yes
   > EOF
   $ hg debugbuilddag .+5000
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5000
   tip-node: 06ddac466af534d365326c13c3879f97caca3cb1
   data-length: 122880
   data-unused: 0
   data-unused: 0.000%
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=70
 
 Simple lookup works
 
   $ ANYNODE=`hg log --template '{node|short}\n' --rev tip`
   $ hg log -r "$ANYNODE" --template '{rev}\n'
   5000
 
 
 #if rust
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????????????.nd: sha256=1e38e9ffaa45cad13f15c1a9880ad606f4241e8beea2f61b4d5365abadfb55f6 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=122880, sha256=1e38e9ffaa45cad13f15c1a9880ad606f4241e8beea2f61b4d5365abadfb55f6
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=122880, sha256=1e38e9ffaa45cad13f15c1a9880ad606f4241e8beea2f61b4d5365abadfb55f6
   0000: 00 00 00 76 00 00 01 65 00 00 00 95 00 00 01 34 |...v...e.......4|
   0010: 00 00 00 19 00 00 01 69 00 00 00 ab 00 00 00 4b |.......i.......K|
   0020: 00 00 00 07 00 00 01 4c 00 00 00 f8 00 00 00 8f |.......L........|
   0030: 00 00 00 c0 00 00 00 a7 00 00 00 89 00 00 01 46 |...............F|
   0040: 00 00 00 92 00 00 01 bc 00 00 00 71 00 00 00 ac |...........q....|
   0050: 00 00 00 af 00 00 00 b4 00 00 00 34 00 00 01 ca |...........4....|
   0060: 00 00 00 23 00 00 01 45 00 00 00 2d 00 00 00 b2 |...#...E...-....|
   0070: 00 00 00 56 00 00 01 0f 00 00 00 4e 00 00 02 4c |...V.......N...L|
   0080: 00 00 00 e7 00 00 00 cd 00 00 01 5b 00 00 00 78 |...........[...x|
   0090: 00 00 00 e3 00 00 01 8e 00 00 00 4f 00 00 00 b1 |...........O....|
   00a0: 00 00 00 30 00 00 00 11 00 00 00 25 00 00 00 d2 |...0.......%....|
   00b0: 00 00 00 ec 00 00 00 69 00 00 01 2b 00 00 01 2e |.......i...+....|
   00c0: 00 00 00 aa 00 00 00 15 00 00 00 3a 00 00 01 4e |...........:...N|
   00d0: 00 00 00 4d 00 00 00 9d 00 00 00 8e 00 00 00 a4 |...M............|
   00e0: 00 00 00 c3 00 00 00 eb 00 00 00 29 00 00 00 ad |...........)....|
   00f0: 00 00 01 3a 00 00 01 32 00 00 00 04 00 00 00 53 |...:...2.......S|
 
 
 #else
 
   $ f --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????????????.nd: sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7 (glob)
   $ hg debugnodemap --dump-new | f --sha256 --size
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   $ hg debugnodemap --dump-disk | f --sha256 --bytes=256 --hexdump --size
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
   0020: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0030: ff ff ff ff ff ff ed b3 ff ff ff ff ff ff ff ff |................|
   0040: ff ff ff ff ff ff ee 34 00 00 00 00 ff ff ff ff |.......4........|
   0050: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0060: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0070: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0080: ff ff ff ff ff ff f8 50 ff ff ff ff ff ff ff ff |.......P........|
   0090: ff ff ff ff ff ff ff ff ff ff ec c7 ff ff ff ff |................|
   00a0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00b0: ff ff ff ff ff ff fa be ff ff f2 fc ff ff ff ff |................|
   00c0: ff ff ff ff ff ff ef ea ff ff ff ff ff ff f9 17 |................|
   00d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   00f0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
 
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5001
   revision in nodemap: 5001
 
 add a new commit
 
   $ hg up
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ echo foo > foo
   $ hg add foo
   $ hg ci -m 'foo'
 
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5001
   tip-node: 2dd9b5258caa46469ff07d4a3da1eb3529a51f49
   data-length: 122880
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5001
   tip-node: 2dd9b5258caa46469ff07d4a3da1eb3529a51f49
   data-length: 123072
   data-unused: 192
   data-unused: 0.156%
 #endif
 
   $ f --size .hg/store/00changelog.n
   .hg/store/00changelog.n: size=70
 
 (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
 
 #if pure
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob)
 #endif
 
 #if rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=123072, sha256=ccc8a43310ace13812fcc648683e259346754ef934c12dd238cf9b7fadfe9a4b (glob)
 #endif
 
 #if no-pure no-rust
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob)
 #endif
 
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
 
 Test code path without mmap
 ---------------------------
 
   $ echo bar > bar
   $ hg add bar
   $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
 
   $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
   revision in index:   5003
   revision in nodemap: 5003
   $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
   revision in index:   5003
   revision in nodemap: 5003
 
 
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 123328
   data-unused: 384
   data-unused: 0.311%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=123328, sha256=10d26e9776b6596af0f89143a54eba8cc581e929c38242a02a7b0760698c6c70 (glob)
 #endif
 #if rust
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 123328
   data-unused: 384
   data-unused: 0.311%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=123328, sha256=081eec9eb6708f2bf085d939b4c97bc0b6762bc8336bc4b93838f7fffa1516bf (glob)
 #endif
 #if no-pure no-rust
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 122944
   data-unused: 0
   data-unused: 0.000%
   $ f --sha256 .hg/store/00changelog-*.nd --size
   .hg/store/00changelog-????????????????.nd: size=122944, sha256=755976b22b64ab680401b45395953504e64e7fa8c31ac570f58dee21e15f9bc0 (glob)
 #endif
 
 Test force warming the cache
 
   $ rm .hg/store/00changelog.n
   $ hg debugnodemap --metadata
   $ hg debugupdatecache
 #if pure
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 122944
   data-unused: 0
   data-unused: 0.000%
 #else
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 122944
   data-unused: 0
   data-unused: 0.000%
 #endif
 
 Check out of sync nodemap
 =========================
 
 First copy old data on the side.
 
   $ mkdir ../tmp-copies
   $ cp .hg/store/00changelog-????????????????.nd .hg/store/00changelog.n ../tmp-copies
 
 Nodemap lagging behind
 ----------------------
 
 make a new commit
 
   $ echo bar2 > bar
   $ hg ci -m 'bar2'
   $ NODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 If the nodemap is lagging behind, it can catch up fine
 
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5003
   tip-node: 5c049e9c4a4af159bdcd65dce1b6bf303a0da6cf
   data-length: 123200 (pure !)
   data-length: 123200 (rust !)
   data-length: 122944 (no-rust no-pure !)
   data-unused: 256 (pure !)
   data-unused: 256 (rust !)
   data-unused: 0 (no-rust no-pure !)
   data-unused: 0.208% (pure !)
   data-unused: 0.208% (rust !)
   data-unused: 0.000% (no-rust no-pure !)
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 122944
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$NODE" -T '{rev}\n'
   5003
 
 changelog altered
 -----------------
 
 If the nodemap is not gated behind a requirements, an unaware client can alter
 the repository so the revlog used to generate the nodemap is not longer
 compatible with the persistent nodemap. We need to detect that.
 
   $ hg up "$NODE~5"
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved
   $ echo bar > babar
   $ hg add babar
   $ hg ci -m 'babar'
   created new head
   $ OTHERNODE=`hg log -r tip -T '{node}\n'`
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5004
 
   $ hg --config extensions.strip= strip --rev "$NODE~1" --no-backup
 
 the nodemap should detect the changelog have been tampered with and recover.
 
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 42bf3068c7ddfdfded53c4eb11d02266faeebfee
   data-length: 123456 (pure !)
-  data-length: 246464 (rust !)
+  data-length: 123008 (rust !)
   data-length: 123008 (no-pure no-rust !)
   data-unused: 448 (pure !)
-  data-unused: 123904 (rust !)
+  data-unused: 0 (rust !)
   data-unused: 0 (no-pure no-rust !)
-  data-unused: 50.273% (rust !)
+  data-unused: 0.000% (rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cp -f ../tmp-copies/* .hg/store/
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5002
   tip-node: 6ce944fafcee85af91f29ea5b51654cc6101ad7e
   data-length: 122944
   data-unused: 0
   data-unused: 0.000%
   $ hg log -r "$OTHERNODE" -T '{rev}\n'
   5002
 
 Check transaction related property
 ==================================
 
 An up to date nodemap should be available to shell hooks,
 
   $ echo dsljfl > a
   $ hg add a
   $ hg ci -m a
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5003
   tip-node: c91af76d172f1053cca41b83f7c2e4e514fe2bcf
   data-length: 123008
   data-unused: 0
   data-unused: 0.000%
   $ echo babar2 > babar
   $ hg ci -m 'babar2' --config "hooks.pretxnclose.nodemap-test=hg debugnodemap --metadata"
   uid: ???????????????? (glob)
   tip-rev: 5004
   tip-node: ba87cd9559559e4b91b28cb140d003985315e031
   data-length: 123328 (pure !)
   data-length: 123328 (rust !)
   data-length: 123136 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.156% (pure !)
   data-unused: 0.156% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5004
   tip-node: ba87cd9559559e4b91b28cb140d003985315e031
   data-length: 123328 (pure !)
   data-length: 123328 (rust !)
   data-length: 123136 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.156% (pure !)
   data-unused: 0.156% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
 Another process does not see the pending nodemap content during run.
 
   $ PATH=$RUNTESTDIR/testlib/:$PATH
   $ echo qpoasp > a
   $ hg ci -m a2 \
   > --config "hooks.pretxnclose=wait-on-file 20 sync-repo-read sync-txn-pending" \
   > --config "hooks.txnclose=touch sync-txn-close" > output.txt 2>&1 &
 
 (read the repository while the commit transaction is pending)
 
   $ wait-on-file 20 sync-txn-pending && \
   > hg debugnodemap --metadata && \
   > wait-on-file 20 sync-txn-close sync-repo-read
   uid: ???????????????? (glob)
   tip-rev: 5004
   tip-node: ba87cd9559559e4b91b28cb140d003985315e031
   data-length: 123328 (pure !)
   data-length: 123328 (rust !)
   data-length: 123136 (no-pure no-rust !)
   data-unused: 192 (pure !)
   data-unused: 192 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.156% (pure !)
   data-unused: 0.156% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5005
   tip-node: bae4d45c759e30f1cb1a40e1382cf0e0414154db
   data-length: 123584 (pure !)
   data-length: 123584 (rust !)
   data-length: 123136 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.363% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
 
   $ cat output.txt
 
 Check that a failing transaction will properly revert the data
 
   $ echo plakfe > a
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????????????.nd: size=123584, sha256=8c6cef6fd3d3fac291968793ee19a4be6d0b8375e9508bd5c7d4a8879e8df180 (glob) (pure !)
   .hg/store/00changelog-????????????????.nd: size=123584, sha256=eb9e9a4bcafdb5e1344bc8a0cbb3288b2106413b8efae6265fb8a7973d7e97f9 (glob) (rust !)
   .hg/store/00changelog-????????????????.nd: size=123136, sha256=4f504f5a834db3811ced50ab3e9e80bcae3581bb0f9b13a7a9f94b7fc34bcebe (glob) (no-pure no-rust !)
   $ hg ci -m a3 --config "extensions.abort=$RUNTESTDIR/testlib/crash_transaction_late.py"
   transaction abort!
   rollback completed
   abort: This is a late abort
   [255]
   $ hg debugnodemap --metadata
   uid: ???????????????? (glob)
   tip-rev: 5005
   tip-node: bae4d45c759e30f1cb1a40e1382cf0e0414154db
   data-length: 123584 (pure !)
   data-length: 123584 (rust !)
   data-length: 123136 (no-pure no-rust !)
   data-unused: 448 (pure !)
   data-unused: 448 (rust !)
   data-unused: 0 (no-pure no-rust !)
   data-unused: 0.363% (pure !)
   data-unused: 0.363% (rust !)
   data-unused: 0.000% (no-pure no-rust !)
   $ f --size --sha256 .hg/store/00changelog-*.nd
   .hg/store/00changelog-????????????????.nd: size=123584, sha256=8c6cef6fd3d3fac291968793ee19a4be6d0b8375e9508bd5c7d4a8879e8df180 (glob) (pure !)
   .hg/store/00changelog-????????????????.nd: size=123584, sha256=eb9e9a4bcafdb5e1344bc8a0cbb3288b2106413b8efae6265fb8a7973d7e97f9 (glob) (rust !)
   .hg/store/00changelog-????????????????.nd: size=123136, sha256=4f504f5a834db3811ced50ab3e9e80bcae3581bb0f9b13a7a9f94b7fc34bcebe (glob) (no-pure no-rust !)