cleanup: typos, formatting

mercurial/bundle2.py

@@ -1,2676 +1,2676 b''
 # bundle2.py - generic container format to transmit arbitrary data.
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Handling of the new bundle2 format
 
 The goal of bundle2 is to act as an atomically packet to transmit a set of
 payloads in an application agnostic way. It consist in a sequence of "parts"
 that will be handed to and processed by the application layer.
 
 
 General format architecture
 ===========================
 
 The format is architectured as follow
 
  - magic string
  - stream level parameters
  - payload parts (any number)
  - end of stream marker.
 
 the Binary format
 ============================
 
 All numbers are unsigned and big-endian.
 
 stream level parameters
 ------------------------
 
 Binary format is as follow
 
 :params size: int32
 
   The total number of Bytes used by the parameters
 
 :params value: arbitrary number of Bytes
 
   A blob of `params size` containing the serialized version of all stream level
   parameters.
 
   The blob contains a space separated list of parameters. Parameters with value
   are stored in the form `<name>=<value>`. Both name and value are urlquoted.
 
   Empty name are obviously forbidden.
 
   Name MUST start with a letter. If this first letter is lower case, the
   parameter is advisory and can be safely ignored. However when the first
   letter is capital, the parameter is mandatory and the bundling process MUST
   stop if he is not able to proceed it.
 
   Stream parameters use a simple textual format for two main reasons:
 
   - Stream level parameters should remain simple and we want to discourage any
     crazy usage.
   - Textual data allow easy human inspection of a bundle2 header in case of
     troubles.
 
   Any Applicative level options MUST go into a bundle2 part instead.
 
 Payload part
 ------------------------
 
 Binary format is as follow
 
 :header size: int32
 
   The total number of Bytes used by the part header. When the header is empty
   (size = 0) this is interpreted as the end of stream marker.
 
 :header:
 
     The header defines how to interpret the part. It contains two piece of
     data: the part type, and the part parameters.
 
     The part type is used to route an application level handler, that can
     interpret payload.
 
     Part parameters are passed to the application level handler.  They are
     meant to convey information that will help the application level object to
     interpret the part payload.
 
     The binary format of the header is has follow
 
     :typesize: (one byte)
 
     :parttype: alphanumerical part name (restricted to [a-zA-Z0-9_:-]*)
 
     :partid: A 32bits integer (unique in the bundle) that can be used to refer
              to this part.
 
     :parameters:
 
         Part's parameter may have arbitrary content, the binary structure is::
 
             <mandatory-count><advisory-count><param-sizes><param-data>
 
         :mandatory-count: 1 byte, number of mandatory parameters
 
         :advisory-count:  1 byte, number of advisory parameters
 
         :param-sizes:
 
             N couple of bytes, where N is the total number of parameters. Each
             couple contains (<size-of-key>, <size-of-value) for one parameter.
 
         :param-data:
 
             A blob of bytes from which each parameter key and value can be
             retrieved using the list of size couples stored in the previous
             field.
 
             Mandatory parameters comes first, then the advisory ones.
 
             Each parameter's key MUST be unique within the part.
 
 :payload:
 
     payload is a series of `<chunksize><chunkdata>`.
 
     `chunksize` is an int32, `chunkdata` are plain bytes (as much as
     `chunksize` says)` The payload part is concluded by a zero size chunk.
 
     The current implementation always produces either zero or one chunk.
     This is an implementation limitation that will ultimately be lifted.
 
     `chunksize` can be negative to trigger special case processing. No such
     processing is in place yet.
 
 Bundle processing
 ============================
 
 Each part is processed in order using a "part handler". Handler are registered
 for a certain part type.
 
 The matching of a part to its handler is case insensitive. The case of the
 part type is used to know if a part is mandatory or advisory. If the Part type
 contains any uppercase char it is considered mandatory. When no handler is
 known for a Mandatory part, the process is aborted and an exception is raised.
 If the part is advisory and no handler is known, the part is ignored. When the
 process is aborted, the full bundle is still read from the stream to keep the
 channel usable. But none of the part read from an abort are processed. In the
 future, dropping the stream may become an option for channel we do not care to
 preserve.
 """
 
 
 import collections
 import errno
 import os
 import re
 import string
 import struct
 import sys
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from . import (
     bookmarks,
     changegroup,
     encoding,
     error,
     obsolete,
     phases,
     pushkey,
     pycompat,
     requirements,
     scmutil,
     streamclone,
     tags,
     url,
     util,
 )
 from .utils import (
     stringutil,
     urlutil,
 )
 from .interfaces import repository
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 _pack = struct.pack
 _unpack = struct.unpack
 
 _fstreamparamsize = b'>i'
 _fpartheadersize = b'>i'
 _fparttypesize = b'>B'
 _fpartid = b'>I'
 _fpayloadsize = b'>i'
 _fpartparamcount = b'>BB'
 
 preferedchunksize = 32768
 
 _parttypeforbidden = re.compile(b'[^a-zA-Z0-9_:-]')
 
 
 def outdebug(ui, message):
     """debug regarding output stream (bundling)"""
     if ui.configbool(b'devel', b'bundle2.debug'):
         ui.debug(b'bundle2-output: %s\n' % message)
 
 
 def indebug(ui, message):
     """debug on input stream (unbundling)"""
     if ui.configbool(b'devel', b'bundle2.debug'):
         ui.debug(b'bundle2-input: %s\n' % message)
 
 
 def validateparttype(parttype):
     """raise ValueError if a parttype contains invalid character"""
     if _parttypeforbidden.search(parttype):
         raise ValueError(parttype)
 
 
 def _makefpartparamsizes(nbparams):
     """return a struct format to read part parameter sizes
 
     The number parameters is variable so we need to build that format
     dynamically.
     """
     return b'>' + (b'BB' * nbparams)
 
 
 parthandlermapping = {}
 
 
 def parthandler(parttype, params=()):
     """decorator that register a function as a bundle2 part handler
 
     eg::
 
         @parthandler('myparttype', ('mandatory', 'param', 'handled'))
         def myparttypehandler(...):
             '''process a part of type "my part".'''
             ...
     """
     validateparttype(parttype)
 
     def _decorator(func):
         lparttype = parttype.lower()  # enforce lower case matching.
         assert lparttype not in parthandlermapping
         parthandlermapping[lparttype] = func
         func.params = frozenset(params)
         return func
 
     return _decorator
 
 
 class unbundlerecords:
     """keep record of what happens during and unbundle
 
     New records are added using `records.add('cat', obj)`. Where 'cat' is a
     category of record and obj is an arbitrary object.
 
     `records['cat']` will return all entries of this category 'cat'.
 
     Iterating on the object itself will yield `('category', obj)` tuples
     for all entries.
 
     All iterations happens in chronological order.
     """
 
     def __init__(self):
         self._categories = {}
         self._sequences = []
         self._replies = {}
 
     def add(self, category, entry, inreplyto=None):
         """add a new record of a given category.
 
         The entry can then be retrieved in the list returned by
         self['category']."""
         self._categories.setdefault(category, []).append(entry)
         self._sequences.append((category, entry))
         if inreplyto is not None:
             self.getreplies(inreplyto).add(category, entry)
 
     def getreplies(self, partid):
         """get the records that are replies to a specific part"""
         return self._replies.setdefault(partid, unbundlerecords())
 
     def __getitem__(self, cat):
         return tuple(self._categories.get(cat, ()))
 
     def __iter__(self):
         return iter(self._sequences)
 
     def __len__(self):
         return len(self._sequences)
 
     def __nonzero__(self):
         return bool(self._sequences)
 
     __bool__ = __nonzero__
 
 
 class bundleoperation:
     """an object that represents a single bundling process
 
     Its purpose is to carry unbundle-related objects and states.
 
     A new object should be created at the beginning of each bundle processing.
     The object is to be returned by the processing function.
 
     The object has very little content now it will ultimately contain:
     * an access to the repo the bundle is applied to,
     * a ui object,
     * a way to retrieve a transaction to add changes to the repo,
     * a way to record the result of processing each part,
     * a way to construct a bundle response when applicable.
     """
 
     def __init__(
         self,
         repo,
         transactiongetter,
         captureoutput=True,
         source=b'',
         remote=None,
     ):
         self.repo = repo
         # the peer object who produced this bundle if available
         self.remote = remote
         self.ui = repo.ui
         self.records = unbundlerecords()
         self.reply = None
         self.captureoutput = captureoutput
         self.hookargs = {}
         self._gettransaction = transactiongetter
         # carries value that can modify part behavior
         self.modes = {}
         self.source = source
 
     def gettransaction(self):
         transaction = self._gettransaction()
 
         if self.hookargs:
             # the ones added to the transaction supercede those added
             # to the operation.
             self.hookargs.update(transaction.hookargs)
             transaction.hookargs = self.hookargs
 
         # mark the hookargs as flushed.  further attempts to add to
         # hookargs will result in an abort.
         self.hookargs = None
 
         return transaction
 
     def addhookargs(self, hookargs):
         if self.hookargs is None:
             raise error.ProgrammingError(
                 b'attempted to add hookargs to '
                 b'operation after transaction started'
             )
         self.hookargs.update(hookargs)
 
 
 class TransactionUnavailable(RuntimeError):
     pass
 
 
 def _notransaction():
     """default method to get a transaction while processing a bundle
 
     Raise an exception to highlight the fact that no transaction was expected
     to be created"""
     raise TransactionUnavailable()
 
 
 def applybundle(repo, unbundler, tr, source, url=None, remote=None, **kwargs):
     # transform me into unbundler.apply() as soon as the freeze is lifted
     if isinstance(unbundler, unbundle20):
         tr.hookargs[b'bundle2'] = b'1'
         if source is not None and b'source' not in tr.hookargs:
             tr.hookargs[b'source'] = source
         if url is not None and b'url' not in tr.hookargs:
             tr.hookargs[b'url'] = url
         return processbundle(
             repo, unbundler, lambda: tr, source=source, remote=remote
         )
     else:
         # the transactiongetter won't be used, but we might as well set it
         op = bundleoperation(repo, lambda: tr, source=source, remote=remote)
         _processchangegroup(op, unbundler, tr, source, url, **kwargs)
         return op
 
 
 class partiterator:
     def __init__(self, repo, op, unbundler):
         self.repo = repo
         self.op = op
         self.unbundler = unbundler
         self.iterator = None
         self.count = 0
         self.current = None
 
     def __enter__(self):
         def func():
             itr = enumerate(self.unbundler.iterparts(), 1)
             for count, p in itr:
                 self.count = count
                 self.current = p
                 yield p
                 p.consume()
                 self.current = None
 
         self.iterator = func()
         return self.iterator
 
     def __exit__(self, type, exc, tb):
         if not self.iterator:
             return
 
         # Only gracefully abort in a normal exception situation. User aborts
         # like Ctrl+C throw a KeyboardInterrupt which is not a base Exception,
         # and should not gracefully cleanup.
         if isinstance(exc, Exception):
             # Any exceptions seeking to the end of the bundle at this point are
             # almost certainly related to the underlying stream being bad.
             # And, chances are that the exception we're handling is related to
             # getting in that bad state. So, we swallow the seeking error and
             # re-raise the original error.
             seekerror = False
             try:
                 if self.current:
                     # consume the part content to not corrupt the stream.
                     self.current.consume()
 
                 for part in self.iterator:
                     # consume the bundle content
                     part.consume()
             except Exception:
                 seekerror = True
 
             # Small hack to let caller code distinguish exceptions from bundle2
             # processing from processing the old format. This is mostly needed
             # to handle different return codes to unbundle according to the type
             # of bundle. We should probably clean up or drop this return code
             # craziness in a future version.
             exc.duringunbundle2 = True
             salvaged = []
             replycaps = None
             if self.op.reply is not None:
                 salvaged = self.op.reply.salvageoutput()
                 replycaps = self.op.reply.capabilities
             exc._replycaps = replycaps
             exc._bundle2salvagedoutput = salvaged
 
             # Re-raising from a variable loses the original stack. So only use
             # that form if we need to.
             if seekerror:
                 raise exc
 
         self.repo.ui.debug(
             b'bundle2-input-bundle: %i parts total\n' % self.count
         )
 
 
 def processbundle(
     repo,
     unbundler,
     transactiongetter=None,
     op=None,
     source=b'',
     remote=None,
 ):
     """This function process a bundle, apply effect to/from a repo
 
     It iterates over each part then searches for and uses the proper handling
     code to process the part. Parts are processed in order.
 
     Unknown Mandatory part will abort the process.
 
     It is temporarily possible to provide a prebuilt bundleoperation to the
     function. This is used to ensure output is properly propagated in case of
     an error during the unbundling. This output capturing part will likely be
     reworked and this ability will probably go away in the process.
     """
     if op is None:
         if transactiongetter is None:
             transactiongetter = _notransaction
         op = bundleoperation(
             repo,
             transactiongetter,
             source=source,
             remote=remote,
         )
     # todo:
     # - replace this is a init function soon.
     # - exception catching
     unbundler.params
     if repo.ui.debugflag:
         msg = [b'bundle2-input-bundle:']
         if unbundler.params:
             msg.append(b' %i params' % len(unbundler.params))
         if op._gettransaction is None or op._gettransaction is _notransaction:
             msg.append(b' no-transaction')
         else:
             msg.append(b' with-transaction')
         msg.append(b'\n')
         repo.ui.debug(b''.join(msg))
 
     processparts(repo, op, unbundler)
 
     return op
 
 
 def processparts(repo, op, unbundler):
     with partiterator(repo, op, unbundler) as parts:
         for part in parts:
             _processpart(op, part)
 
 
 def _processchangegroup(op, cg, tr, source, url, **kwargs):
     if op.remote is not None and op.remote.path is not None:
         remote_path = op.remote.path
         kwargs = kwargs.copy()
         kwargs['delta_base_reuse_policy'] = remote_path.delta_reuse_policy
     ret = cg.apply(op.repo, tr, source, url, **kwargs)
     op.records.add(
         b'changegroup',
         {
             b'return': ret,
         },
     )
     return ret
 
 
 def _gethandler(op, part):
     status = b'unknown'  # used by debug output
     try:
         handler = parthandlermapping.get(part.type)
         if handler is None:
             status = b'unsupported-type'
             raise error.BundleUnknownFeatureError(parttype=part.type)
         indebug(op.ui, b'found a handler for part %s' % part.type)
         unknownparams = part.mandatorykeys - handler.params
         if unknownparams:
             unknownparams = list(unknownparams)
             unknownparams.sort()
             status = b'unsupported-params (%s)' % b', '.join(unknownparams)
             raise error.BundleUnknownFeatureError(
                 parttype=part.type, params=unknownparams
             )
         status = b'supported'
     except error.BundleUnknownFeatureError as exc:
         if part.mandatory:  # mandatory parts
             raise
         indebug(op.ui, b'ignoring unsupported advisory part %s' % exc)
         return  # skip to part processing
     finally:
         if op.ui.debugflag:
             msg = [b'bundle2-input-part: "%s"' % part.type]
             if not part.mandatory:
                 msg.append(b' (advisory)')
             nbmp = len(part.mandatorykeys)
             nbap = len(part.params) - nbmp
             if nbmp or nbap:
                 msg.append(b' (params:')
                 if nbmp:
                     msg.append(b' %i mandatory' % nbmp)
                 if nbap:
                     msg.append(b' %i advisory' % nbmp)
                 msg.append(b')')
             msg.append(b' %s\n' % status)
             op.ui.debug(b''.join(msg))
 
     return handler
 
 
 def _processpart(op, part):
     """process a single part from a bundle
 
     The part is guaranteed to have been fully consumed when the function exits
     (even if an exception is raised)."""
     handler = _gethandler(op, part)
     if handler is None:
         return
 
     # handler is called outside the above try block so that we don't
     # risk catching KeyErrors from anything other than the
     # parthandlermapping lookup (any KeyError raised by handler()
     # itself represents a defect of a different variety).
     output = None
     if op.captureoutput and op.reply is not None:
         op.ui.pushbuffer(error=True, subproc=True)
         output = b''
     try:
         handler(op, part)
     finally:
         if output is not None:
             output = op.ui.popbuffer()
         if output:
             outpart = op.reply.newpart(b'output', data=output, mandatory=False)
             outpart.addparam(
                 b'in-reply-to', pycompat.bytestr(part.id), mandatory=False
             )
 
 
 def decodecaps(blob):
     """decode a bundle2 caps bytes blob into a dictionary
 
     The blob is a list of capabilities (one per line)
     Capabilities may have values using a line of the form::
 
         capability=value1,value2,value3
 
     The values are always a list."""
     caps = {}
     for line in blob.splitlines():
         if not line:
             continue
         if b'=' not in line:
             key, vals = line, ()
         else:
             key, vals = line.split(b'=', 1)
             vals = vals.split(b',')
         key = urlreq.unquote(key)
         vals = [urlreq.unquote(v) for v in vals]
         caps[key] = vals
     return caps
 
 
 def encodecaps(caps):
     """encode a bundle2 caps dictionary into a bytes blob"""
     chunks = []
     for ca in sorted(caps):
         vals = caps[ca]
         ca = urlreq.quote(ca)
         vals = [urlreq.quote(v) for v in vals]
         if vals:
             ca = b"%s=%s" % (ca, b','.join(vals))
         chunks.append(ca)
     return b'\n'.join(chunks)
 
 
 bundletypes = {
     b"": (b"", b'UN'),  # only when using unbundle on ssh and old http servers
     # since the unification ssh accepts a header but there
     # is no capability signaling it.
     b"HG20": (),  # special-cased below
     b"HG10UN": (b"HG10UN", b'UN'),
     b"HG10BZ": (b"HG10", b'BZ'),
     b"HG10GZ": (b"HG10GZ", b'GZ'),
 }
 
 # hgweb uses this list to communicate its preferred type
 bundlepriority = [b'HG10GZ', b'HG10BZ', b'HG10UN']
 
 
 class bundle20:
     """represent an outgoing bundle2 container
 
     Use the `addparam` method to add stream level parameter. and `newpart` to
     populate it. Then call `getchunks` to retrieve all the binary chunks of
     data that compose the bundle2 container."""
 
     _magicstring = b'HG20'
 
     def __init__(self, ui, capabilities=()):
         self.ui = ui
         self._params = []
         self._parts = []
         self.capabilities = dict(capabilities)
         self._compengine = util.compengines.forbundletype(b'UN')
         self._compopts = None
         # If compression is being handled by a consumer of the raw
         # data (e.g. the wire protocol), unsetting this flag tells
         # consumers that the bundle is best left uncompressed.
         self.prefercompressed = True
 
     def setcompression(self, alg, compopts=None):
         """setup core part compression to <alg>"""
         if alg in (None, b'UN'):
             return
         assert not any(n.lower() == b'compression' for n, v in self._params)
         self.addparam(b'Compression', alg)
         self._compengine = util.compengines.forbundletype(alg)
         self._compopts = compopts
 
     @property
     def nbparts(self):
         """total number of parts added to the bundler"""
         return len(self._parts)
 
     # methods used to defines the bundle2 content
     def addparam(self, name, value=None):
         """add a stream level parameter"""
         if not name:
             raise error.ProgrammingError(b'empty parameter name')
         if name[0:1] not in pycompat.bytestr(
             string.ascii_letters  # pytype: disable=wrong-arg-types
         ):
             raise error.ProgrammingError(
                 b'non letter first character: %s' % name
             )
         self._params.append((name, value))
 
     def addpart(self, part):
         """add a new part to the bundle2 container
 
         Parts contains the actual applicative payload."""
         assert part.id is None
         part.id = len(self._parts)  # very cheap counter
         self._parts.append(part)
 
     def newpart(self, typeid, *args, **kwargs):
         """create a new part and add it to the containers
 
         As the part is directly added to the containers. For now, this means
         that any failure to properly initialize the part after calling
         ``newpart`` should result in a failure of the whole bundling process.
 
         You can still fall back to manually create and add if you need better
         control."""
         part = bundlepart(typeid, *args, **kwargs)
         self.addpart(part)
         return part
 
     # methods used to generate the bundle2 stream
     def getchunks(self):
         if self.ui.debugflag:
             msg = [b'bundle2-output-bundle: "%s",' % self._magicstring]
             if self._params:
                 msg.append(b' (%i params)' % len(self._params))
             msg.append(b' %i parts total\n' % len(self._parts))
             self.ui.debug(b''.join(msg))
         outdebug(self.ui, b'start emission of %s stream' % self._magicstring)
         yield self._magicstring
         param = self._paramchunk()
         outdebug(self.ui, b'bundle parameter: %s' % param)
         yield _pack(_fstreamparamsize, len(param))
         if param:
             yield param
         for chunk in self._compengine.compressstream(
             self._getcorechunk(), self._compopts
         ):
             yield chunk
 
     def _paramchunk(self):
         """return a encoded version of all stream parameters"""
         blocks = []
         for par, value in self._params:
             par = urlreq.quote(par)
             if value is not None:
                 value = urlreq.quote(value)
                 par = b'%s=%s' % (par, value)
             blocks.append(par)
         return b' '.join(blocks)
 
     def _getcorechunk(self):
         """yield chunk for the core part of the bundle
 
         (all but headers and parameters)"""
         outdebug(self.ui, b'start of parts')
         for part in self._parts:
             outdebug(self.ui, b'bundle part: "%s"' % part.type)
             for chunk in part.getchunks(ui=self.ui):
                 yield chunk
         outdebug(self.ui, b'end of bundle')
         yield _pack(_fpartheadersize, 0)
 
     def salvageoutput(self):
         """return a list with a copy of all output parts in the bundle
 
         This is meant to be used during error handling to make sure we preserve
         server output"""
         salvaged = []
         for part in self._parts:
             if part.type.startswith(b'output'):
                 salvaged.append(part.copy())
         return salvaged
 
 
 class unpackermixin:
     """A mixin to extract bytes and struct data from a stream"""
 
     def __init__(self, fp):
         self._fp = fp
 
     def _unpack(self, format):
         """unpack this struct format from the stream
 
         This method is meant for internal usage by the bundle2 protocol only.
         They directly manipulate the low level stream including bundle2 level
         instruction.
 
         Do not use it to implement higher-level logic or methods."""
         data = self._readexact(struct.calcsize(format))
         return _unpack(format, data)
 
     def _readexact(self, size):
         """read exactly <size> bytes from the stream
 
         This method is meant for internal usage by the bundle2 protocol only.
         They directly manipulate the low level stream including bundle2 level
         instruction.
 
         Do not use it to implement higher-level logic or methods."""
         return changegroup.readexactly(self._fp, size)
 
 
 def getunbundler(ui, fp, magicstring=None):
     """return a valid unbundler object for a given magicstring"""
     if magicstring is None:
         magicstring = changegroup.readexactly(fp, 4)
     magic, version = magicstring[0:2], magicstring[2:4]
     if magic != b'HG':
         ui.debug(
             b"error: invalid magic: %r (version %r), should be 'HG'\n"
             % (magic, version)
         )
         raise error.Abort(_(b'not a Mercurial bundle'))
     unbundlerclass = formatmap.get(version)
     if unbundlerclass is None:
         raise error.Abort(_(b'unknown bundle version %s') % version)
     unbundler = unbundlerclass(ui, fp)
     indebug(ui, b'start processing of %s stream' % magicstring)
     return unbundler
 
 
 class unbundle20(unpackermixin):
     """interpret a bundle2 stream
 
     This class is fed with a binary stream and yields parts through its
     `iterparts` methods."""
 
     _magicstring = b'HG20'
 
     def __init__(self, ui, fp):
         """If header is specified, we do not read it out of the stream."""
         self.ui = ui
         self._compengine = util.compengines.forbundletype(b'UN')
         self._compressed = None
         super(unbundle20, self).__init__(fp)
 
     @util.propertycache
     def params(self):
         """dictionary of stream level parameters"""
         indebug(self.ui, b'reading bundle2 stream parameters')
         params = {}
         paramssize = self._unpack(_fstreamparamsize)[0]
         if paramssize < 0:
             raise error.BundleValueError(
                 b'negative bundle param size: %i' % paramssize
             )
         if paramssize:
             params = self._readexact(paramssize)
             params = self._processallparams(params)
         return params
 
     def _processallparams(self, paramsblock):
         """ """
         params = util.sortdict()
         for p in paramsblock.split(b' '):
             p = p.split(b'=', 1)
             p = [urlreq.unquote(i) for i in p]
             if len(p) < 2:
                 p.append(None)
             self._processparam(*p)
             params[p[0]] = p[1]
         return params
 
     def _processparam(self, name, value):
         """process a parameter, applying its effect if needed
 
         Parameter starting with a lower case letter are advisory and will be
         ignored when unknown.  Those starting with an upper case letter are
         mandatory and will this function will raise a KeyError when unknown.
 
         Note: no option are currently supported. Any input will be either
               ignored or failing.
         """
         if not name:
             raise ValueError('empty parameter name')
         if name[0:1] not in pycompat.bytestr(
             string.ascii_letters  # pytype: disable=wrong-arg-types
         ):
             raise ValueError('non letter first character: %s' % name)
         try:
             handler = b2streamparamsmap[name.lower()]
         except KeyError:
             if name[0:1].islower():
                 indebug(self.ui, b"ignoring unknown parameter %s" % name)
             else:
                 raise error.BundleUnknownFeatureError(params=(name,))
         else:
             handler(self, name, value)
 
     def _forwardchunks(self):
         """utility to transfer a bundle2 as binary
 
         This is made necessary by the fact the 'getbundle' command over 'ssh'
-        have no way to know then the reply end, relying on the bundle to be
+        have no way to know when the reply ends, relying on the bundle to be
         interpreted to know its end. This is terrible and we are sorry, but we
         needed to move forward to get general delta enabled.
         """
         yield self._magicstring
         assert 'params' not in vars(self)
         paramssize = self._unpack(_fstreamparamsize)[0]
         if paramssize < 0:
             raise error.BundleValueError(
                 b'negative bundle param size: %i' % paramssize
             )
         if paramssize:
             params = self._readexact(paramssize)
             self._processallparams(params)
             # The payload itself is decompressed below, so drop
             # the compression parameter passed down to compensate.
             outparams = []
             for p in params.split(b' '):
                 k, v = p.split(b'=', 1)
                 if k.lower() != b'compression':
                     outparams.append(p)
             outparams = b' '.join(outparams)
             yield _pack(_fstreamparamsize, len(outparams))
             yield outparams
         else:
             yield _pack(_fstreamparamsize, paramssize)
         # From there, payload might need to be decompressed
         self._fp = self._compengine.decompressorreader(self._fp)
         emptycount = 0
         while emptycount < 2:
             # so we can brainlessly loop
             assert _fpartheadersize == _fpayloadsize
             size = self._unpack(_fpartheadersize)[0]
             yield _pack(_fpartheadersize, size)
             if size:
                 emptycount = 0
             else:
                 emptycount += 1
                 continue
             if size == flaginterrupt:
                 continue
             elif size < 0:
                 raise error.BundleValueError(b'negative chunk size: %i')
             yield self._readexact(size)
 
     def iterparts(self, seekable=False):
         """yield all parts contained in the stream"""
         cls = seekableunbundlepart if seekable else unbundlepart
         # make sure param have been loaded
         self.params
         # From there, payload need to be decompressed
         self._fp = self._compengine.decompressorreader(self._fp)
         indebug(self.ui, b'start extraction of bundle2 parts')
         headerblock = self._readpartheader()
         while headerblock is not None:
             part = cls(self.ui, headerblock, self._fp)
             yield part
             # Ensure part is fully consumed so we can start reading the next
             # part.
             part.consume()
 
             headerblock = self._readpartheader()
         indebug(self.ui, b'end of bundle2 stream')
 
     def _readpartheader(self):
         """reads a part header size and return the bytes blob
 
         returns None if empty"""
         headersize = self._unpack(_fpartheadersize)[0]
         if headersize < 0:
             raise error.BundleValueError(
                 b'negative part header size: %i' % headersize
             )
         indebug(self.ui, b'part header size: %i' % headersize)
         if headersize:
             return self._readexact(headersize)
         return None
 
     def compressed(self):
         self.params  # load params
         return self._compressed
 
     def close(self):
         """close underlying file"""
         if util.safehasattr(self._fp, 'close'):
             return self._fp.close()
 
 
 formatmap = {b'20': unbundle20}
 
 b2streamparamsmap = {}
 
 
 def b2streamparamhandler(name):
     """register a handler for a stream level parameter"""
 
     def decorator(func):
         assert name not in formatmap
         b2streamparamsmap[name] = func
         return func
 
     return decorator
 
 
 @b2streamparamhandler(b'compression')
 def processcompression(unbundler, param, value):
     """read compression parameter and install payload decompression"""
     if value not in util.compengines.supportedbundletypes:
         raise error.BundleUnknownFeatureError(params=(param,), values=(value,))
     unbundler._compengine = util.compengines.forbundletype(value)
     if value is not None:
         unbundler._compressed = True
 
 
 class bundlepart:
     """A bundle2 part contains application level payload
 
     The part `type` is used to route the part to the application level
     handler.
 
     The part payload is contained in ``part.data``. It could be raw bytes or a
     generator of byte chunks.
 
     You can add parameters to the part using the ``addparam`` method.
     Parameters can be either mandatory (default) or advisory. Remote side
     should be able to safely ignore the advisory ones.
 
     Both data and parameters cannot be modified after the generation has begun.
     """
 
     def __init__(
         self,
         parttype,
         mandatoryparams=(),
         advisoryparams=(),
         data=b'',
         mandatory=True,
     ):
         validateparttype(parttype)
         self.id = None
         self.type = parttype
         self._data = data
         self._mandatoryparams = list(mandatoryparams)
         self._advisoryparams = list(advisoryparams)
         # checking for duplicated entries
         self._seenparams = set()
         for pname, __ in self._mandatoryparams + self._advisoryparams:
             if pname in self._seenparams:
                 raise error.ProgrammingError(b'duplicated params: %s' % pname)
             self._seenparams.add(pname)
         # status of the part's generation:
         # - None: not started,
         # - False: currently generated,
         # - True: generation done.
         self._generated = None
         self.mandatory = mandatory
 
     def __repr__(self):
         cls = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
         return '<%s object at %x; id: %s; type: %s; mandatory: %s>' % (
             cls,
             id(self),
             self.id,
             self.type,
             self.mandatory,
         )
 
     def copy(self):
         """return a copy of the part
 
         The new part have the very same content but no partid assigned yet.
         Parts with generated data cannot be copied."""
         assert not util.safehasattr(self.data, 'next')
         return self.__class__(
             self.type,
             self._mandatoryparams,
             self._advisoryparams,
             self._data,
             self.mandatory,
         )
 
     # methods used to defines the part content
     @property
     def data(self):
         return self._data
 
     @data.setter
     def data(self, data):
         if self._generated is not None:
             raise error.ReadOnlyPartError(b'part is being generated')
         self._data = data
 
     @property
     def mandatoryparams(self):
         # make it an immutable tuple to force people through ``addparam``
         return tuple(self._mandatoryparams)
 
     @property
     def advisoryparams(self):
         # make it an immutable tuple to force people through ``addparam``
         return tuple(self._advisoryparams)
 
     def addparam(self, name, value=b'', mandatory=True):
         """add a parameter to the part
 
         If 'mandatory' is set to True, the remote handler must claim support
         for this parameter or the unbundling will be aborted.
 
         The 'name' and 'value' cannot exceed 255 bytes each.
         """
         if self._generated is not None:
             raise error.ReadOnlyPartError(b'part is being generated')
         if name in self._seenparams:
             raise ValueError(b'duplicated params: %s' % name)
         self._seenparams.add(name)
         params = self._advisoryparams
         if mandatory:
             params = self._mandatoryparams
         params.append((name, value))
 
     # methods used to generates the bundle2 stream
     def getchunks(self, ui):
         if self._generated is not None:
             raise error.ProgrammingError(b'part can only be consumed once')
         self._generated = False
 
         if ui.debugflag:
             msg = [b'bundle2-output-part: "%s"' % self.type]
             if not self.mandatory:
                 msg.append(b' (advisory)')
             nbmp = len(self.mandatoryparams)
             nbap = len(self.advisoryparams)
             if nbmp or nbap:
                 msg.append(b' (params:')
                 if nbmp:
                     msg.append(b' %i mandatory' % nbmp)
                 if nbap:
                     msg.append(b' %i advisory' % nbmp)
                 msg.append(b')')
             if not self.data:
                 msg.append(b' empty payload')
             elif util.safehasattr(self.data, 'next') or util.safehasattr(
                 self.data, b'__next__'
             ):
                 msg.append(b' streamed payload')
             else:
                 msg.append(b' %i bytes payload' % len(self.data))
             msg.append(b'\n')
             ui.debug(b''.join(msg))
 
         #### header
         if self.mandatory:
             parttype = self.type.upper()
         else:
             parttype = self.type.lower()
         outdebug(ui, b'part %s: "%s"' % (pycompat.bytestr(self.id), parttype))
         ## parttype
         header = [
             _pack(_fparttypesize, len(parttype)),
             parttype,
             _pack(_fpartid, self.id),
         ]
         ## parameters
         # count
         manpar = self.mandatoryparams
         advpar = self.advisoryparams
         header.append(_pack(_fpartparamcount, len(manpar), len(advpar)))
         # size
         parsizes = []
         for key, value in manpar:
             parsizes.append(len(key))
             parsizes.append(len(value))
         for key, value in advpar:
             parsizes.append(len(key))
             parsizes.append(len(value))
         paramsizes = _pack(_makefpartparamsizes(len(parsizes) // 2), *parsizes)
         header.append(paramsizes)
         # key, value
         for key, value in manpar:
             header.append(key)
             header.append(value)
         for key, value in advpar:
             header.append(key)
             header.append(value)
         ## finalize header
         try:
             headerchunk = b''.join(header)
         except TypeError:
             raise TypeError(
                 'Found a non-bytes trying to '
                 'build bundle part header: %r' % header
             )
         outdebug(ui, b'header chunk size: %i' % len(headerchunk))
         yield _pack(_fpartheadersize, len(headerchunk))
         yield headerchunk
         ## payload
         try:
             for chunk in self._payloadchunks():
                 outdebug(ui, b'payload chunk size: %i' % len(chunk))
                 yield _pack(_fpayloadsize, len(chunk))
                 yield chunk
         except GeneratorExit:
             # GeneratorExit means that nobody is listening for our
             # results anyway, so just bail quickly rather than trying
             # to produce an error part.
             ui.debug(b'bundle2-generatorexit\n')
             raise
         except BaseException as exc:
             bexc = stringutil.forcebytestr(exc)
             # backup exception data for later
             ui.debug(
                 b'bundle2-input-stream-interrupt: encoding exception %s' % bexc
             )
             tb = sys.exc_info()[2]
             msg = b'unexpected error: %s' % bexc
             interpart = bundlepart(
                 b'error:abort', [(b'message', msg)], mandatory=False
             )
             interpart.id = 0
             yield _pack(_fpayloadsize, -1)
             for chunk in interpart.getchunks(ui=ui):
                 yield chunk
             outdebug(ui, b'closing payload chunk')
             # abort current part payload
             yield _pack(_fpayloadsize, 0)
             pycompat.raisewithtb(exc, tb)
         # end of payload
         outdebug(ui, b'closing payload chunk')
         yield _pack(_fpayloadsize, 0)
         self._generated = True
 
     def _payloadchunks(self):
         """yield chunks of a the part payload
 
         Exists to handle the different methods to provide data to a part."""
         # we only support fixed size data now.
         # This will be improved in the future.
         if util.safehasattr(self.data, 'next') or util.safehasattr(
             self.data, '__next__'
         ):
             buff = util.chunkbuffer(self.data)
             chunk = buff.read(preferedchunksize)
             while chunk:
                 yield chunk
                 chunk = buff.read(preferedchunksize)
         elif len(self.data):
             yield self.data
 
 
 flaginterrupt = -1
 
 
 class interrupthandler(unpackermixin):
     """read one part and process it with restricted capability
 
     This allows to transmit exception raised on the producer size during part
     iteration while the consumer is reading a part.
 
     Part processed in this manner only have access to a ui object,"""
 
     def __init__(self, ui, fp):
         super(interrupthandler, self).__init__(fp)
         self.ui = ui
 
     def _readpartheader(self):
         """reads a part header size and return the bytes blob
 
         returns None if empty"""
         headersize = self._unpack(_fpartheadersize)[0]
         if headersize < 0:
             raise error.BundleValueError(
                 b'negative part header size: %i' % headersize
             )
         indebug(self.ui, b'part header size: %i\n' % headersize)
         if headersize:
             return self._readexact(headersize)
         return None
 
     def __call__(self):
 
         self.ui.debug(
             b'bundle2-input-stream-interrupt: opening out of band context\n'
         )
         indebug(self.ui, b'bundle2 stream interruption, looking for a part.')
         headerblock = self._readpartheader()
         if headerblock is None:
             indebug(self.ui, b'no part found during interruption.')
             return
         part = unbundlepart(self.ui, headerblock, self._fp)
         op = interruptoperation(self.ui)
         hardabort = False
         try:
             _processpart(op, part)
         except (SystemExit, KeyboardInterrupt):
             hardabort = True
             raise
         finally:
             if not hardabort:
                 part.consume()
         self.ui.debug(
             b'bundle2-input-stream-interrupt: closing out of band context\n'
         )
 
 
 class interruptoperation:
     """A limited operation to be use by part handler during interruption
 
     It only have access to an ui object.
     """
 
     def __init__(self, ui):
         self.ui = ui
         self.reply = None
         self.captureoutput = False
 
     @property
     def repo(self):
         raise error.ProgrammingError(b'no repo access from stream interruption')
 
     def gettransaction(self):
         raise TransactionUnavailable(b'no repo access from stream interruption')
 
 
 def decodepayloadchunks(ui, fh):
     """Reads bundle2 part payload data into chunks.
 
     Part payload data consists of framed chunks. This function takes
     a file handle and emits those chunks.
     """
     dolog = ui.configbool(b'devel', b'bundle2.debug')
     debug = ui.debug
 
     headerstruct = struct.Struct(_fpayloadsize)
     headersize = headerstruct.size
     unpack = headerstruct.unpack
 
     readexactly = changegroup.readexactly
     read = fh.read
 
     chunksize = unpack(readexactly(fh, headersize))[0]
     indebug(ui, b'payload chunk size: %i' % chunksize)
 
     # changegroup.readexactly() is inlined below for performance.
     while chunksize:
         if chunksize >= 0:
             s = read(chunksize)
             if len(s) < chunksize:
                 raise error.Abort(
                     _(
                         b'stream ended unexpectedly '
                         b' (got %d bytes, expected %d)'
                     )
                     % (len(s), chunksize)
                 )
 
             yield s
         elif chunksize == flaginterrupt:
             # Interrupt "signal" detected. The regular stream is interrupted
             # and a bundle2 part follows. Consume it.
             interrupthandler(ui, fh)()
         else:
             raise error.BundleValueError(
                 b'negative payload chunk size: %s' % chunksize
             )
 
         s = read(headersize)
         if len(s) < headersize:
             raise error.Abort(
                 _(b'stream ended unexpectedly  (got %d bytes, expected %d)')
                 % (len(s), chunksize)
             )
 
         chunksize = unpack(s)[0]
 
         # indebug() inlined for performance.
         if dolog:
             debug(b'bundle2-input: payload chunk size: %i\n' % chunksize)
 
 
 class unbundlepart(unpackermixin):
     """a bundle part read from a bundle"""
 
     def __init__(self, ui, header, fp):
         super(unbundlepart, self).__init__(fp)
         self._seekable = util.safehasattr(fp, 'seek') and util.safehasattr(
             fp, 'tell'
         )
         self.ui = ui
         # unbundle state attr
         self._headerdata = header
         self._headeroffset = 0
         self._initialized = False
         self.consumed = False
         # part data
         self.id = None
         self.type = None
         self.mandatoryparams = None
         self.advisoryparams = None
         self.params = None
         self.mandatorykeys = ()
         self._readheader()
         self._mandatory = None
         self._pos = 0
 
     def _fromheader(self, size):
         """return the next <size> byte from the header"""
         offset = self._headeroffset
         data = self._headerdata[offset : (offset + size)]
         self._headeroffset = offset + size
         return data
 
     def _unpackheader(self, format):
         """read given format from header
 
         This automatically compute the size of the format to read."""
         data = self._fromheader(struct.calcsize(format))
         return _unpack(format, data)
 
     def _initparams(self, mandatoryparams, advisoryparams):
         """internal function to setup all logic related parameters"""
         # make it read only to prevent people touching it by mistake.
         self.mandatoryparams = tuple(mandatoryparams)
         self.advisoryparams = tuple(advisoryparams)
         # user friendly UI
         self.params = util.sortdict(self.mandatoryparams)
         self.params.update(self.advisoryparams)
         self.mandatorykeys = frozenset(p[0] for p in mandatoryparams)
 
     def _readheader(self):
         """read the header and setup the object"""
         typesize = self._unpackheader(_fparttypesize)[0]
         self.type = self._fromheader(typesize)
         indebug(self.ui, b'part type: "%s"' % self.type)
         self.id = self._unpackheader(_fpartid)[0]
         indebug(self.ui, b'part id: "%s"' % pycompat.bytestr(self.id))
         # extract mandatory bit from type
         self.mandatory = self.type != self.type.lower()
         self.type = self.type.lower()
         ## reading parameters
         # param count
         mancount, advcount = self._unpackheader(_fpartparamcount)
         indebug(self.ui, b'part parameters: %i' % (mancount + advcount))
         # param size
         fparamsizes = _makefpartparamsizes(mancount + advcount)
         paramsizes = self._unpackheader(fparamsizes)
         # make it a list of couple again
         paramsizes = list(zip(paramsizes[::2], paramsizes[1::2]))
         # split mandatory from advisory
         mansizes = paramsizes[:mancount]
         advsizes = paramsizes[mancount:]
         # retrieve param value
         manparams = []
         for key, value in mansizes:
             manparams.append((self._fromheader(key), self._fromheader(value)))
         advparams = []
         for key, value in advsizes:
             advparams.append((self._fromheader(key), self._fromheader(value)))
         self._initparams(manparams, advparams)
         ## part payload
         self._payloadstream = util.chunkbuffer(self._payloadchunks())
         # we read the data, tell it
         self._initialized = True
 
     def _payloadchunks(self):
         """Generator of decoded chunks in the payload."""
         return decodepayloadchunks(self.ui, self._fp)
 
     def consume(self):
         """Read the part payload until completion.
 
         By consuming the part data, the underlying stream read offset will
         be advanced to the next part (or end of stream).
         """
         if self.consumed:
             return
 
         chunk = self.read(32768)
         while chunk:
             self._pos += len(chunk)
             chunk = self.read(32768)
 
     def read(self, size=None):
         """read payload data"""
         if not self._initialized:
             self._readheader()
         if size is None:
             data = self._payloadstream.read()
         else:
             data = self._payloadstream.read(size)
         self._pos += len(data)
         if size is None or len(data) < size:
             if not self.consumed and self._pos:
                 self.ui.debug(
                     b'bundle2-input-part: total payload size %i\n' % self._pos
                 )
             self.consumed = True
         return data
 
 
 class seekableunbundlepart(unbundlepart):
     """A bundle2 part in a bundle that is seekable.
 
     Regular ``unbundlepart`` instances can only be read once. This class
     extends ``unbundlepart`` to enable bi-directional seeking within the
     part.
 
     Bundle2 part data consists of framed chunks. Offsets when seeking
     refer to the decoded data, not the offsets in the underlying bundle2
     stream.
 
     To facilitate quickly seeking within the decoded data, instances of this
     class maintain a mapping between offsets in the underlying stream and
     the decoded payload. This mapping will consume memory in proportion
     to the number of chunks within the payload (which almost certainly
     increases in proportion with the size of the part).
     """
 
     def __init__(self, ui, header, fp):
         # (payload, file) offsets for chunk starts.
         self._chunkindex = []
 
         super(seekableunbundlepart, self).__init__(ui, header, fp)
 
     def _payloadchunks(self, chunknum=0):
         '''seek to specified chunk and start yielding data'''
         if len(self._chunkindex) == 0:
             assert chunknum == 0, b'Must start with chunk 0'
             self._chunkindex.append((0, self._tellfp()))
         else:
             assert chunknum < len(self._chunkindex), (
                 b'Unknown chunk %d' % chunknum
             )
             self._seekfp(self._chunkindex[chunknum][1])
 
         pos = self._chunkindex[chunknum][0]
 
         for chunk in decodepayloadchunks(self.ui, self._fp):
             chunknum += 1
             pos += len(chunk)
             if chunknum == len(self._chunkindex):
                 self._chunkindex.append((pos, self._tellfp()))
 
             yield chunk
 
     def _findchunk(self, pos):
         '''for a given payload position, return a chunk number and offset'''
         for chunk, (ppos, fpos) in enumerate(self._chunkindex):
             if ppos == pos:
                 return chunk, 0
             elif ppos > pos:
                 return chunk - 1, pos - self._chunkindex[chunk - 1][0]
         raise ValueError(b'Unknown chunk')
 
     def tell(self):
         return self._pos
 
     def seek(self, offset, whence=os.SEEK_SET):
         if whence == os.SEEK_SET:
             newpos = offset
         elif whence == os.SEEK_CUR:
             newpos = self._pos + offset
         elif whence == os.SEEK_END:
             if not self.consumed:
                 # Can't use self.consume() here because it advances self._pos.
                 chunk = self.read(32768)
                 while chunk:
                     chunk = self.read(32768)
             newpos = self._chunkindex[-1][0] - offset
         else:
             raise ValueError(b'Unknown whence value: %r' % (whence,))
 
         if newpos > self._chunkindex[-1][0] and not self.consumed:
             # Can't use self.consume() here because it advances self._pos.
             chunk = self.read(32768)
             while chunk:
                 chunk = self.read(32668)
 
         if not 0 <= newpos <= self._chunkindex[-1][0]:
             raise ValueError(b'Offset out of range')
 
         if self._pos != newpos:
             chunk, internaloffset = self._findchunk(newpos)
             self._payloadstream = util.chunkbuffer(self._payloadchunks(chunk))
             adjust = self.read(internaloffset)
             if len(adjust) != internaloffset:
                 raise error.Abort(_(b'Seek failed\n'))
             self._pos = newpos
 
     def _seekfp(self, offset, whence=0):
         """move the underlying file pointer
 
         This method is meant for internal usage by the bundle2 protocol only.
         They directly manipulate the low level stream including bundle2 level
         instruction.
 
         Do not use it to implement higher-level logic or methods."""
         if self._seekable:
             return self._fp.seek(offset, whence)
         else:
             raise NotImplementedError(_(b'File pointer is not seekable'))
 
     def _tellfp(self):
         """return the file offset, or None if file is not seekable
 
         This method is meant for internal usage by the bundle2 protocol only.
         They directly manipulate the low level stream including bundle2 level
         instruction.
 
         Do not use it to implement higher-level logic or methods."""
         if self._seekable:
             try:
                 return self._fp.tell()
             except IOError as e:
                 if e.errno == errno.ESPIPE:
                     self._seekable = False
                 else:
                     raise
         return None
 
 
 # These are only the static capabilities.
 # Check the 'getrepocaps' function for the rest.
 capabilities = {
     b'HG20': (),
     b'bookmarks': (),
     b'error': (b'abort', b'unsupportedcontent', b'pushraced', b'pushkey'),
     b'listkeys': (),
     b'pushkey': (),
     b'digests': tuple(sorted(util.DIGESTS.keys())),
     b'remote-changegroup': (b'http', b'https'),
     b'hgtagsfnodes': (),
     b'phases': (b'heads',),
     b'stream': (b'v2',),
 }
 
 
 def getrepocaps(repo, allowpushback=False, role=None):
     """return the bundle2 capabilities for a given repo
 
     Exists to allow extensions (like evolution) to mutate the capabilities.
 
     The returned value is used for servers advertising their capabilities as
     well as clients advertising their capabilities to servers as part of
     bundle2 requests. The ``role`` argument specifies which is which.
     """
     if role not in (b'client', b'server'):
         raise error.ProgrammingError(b'role argument must be client or server')
 
     caps = capabilities.copy()
     caps[b'changegroup'] = tuple(
         sorted(changegroup.supportedincomingversions(repo))
     )
     if obsolete.isenabled(repo, obsolete.exchangeopt):
         supportedformat = tuple(b'V%i' % v for v in obsolete.formats)
         caps[b'obsmarkers'] = supportedformat
     if allowpushback:
         caps[b'pushback'] = ()
     cpmode = repo.ui.config(b'server', b'concurrent-push-mode')
     if cpmode == b'check-related':
         caps[b'checkheads'] = (b'related',)
     if b'phases' in repo.ui.configlist(b'devel', b'legacy.exchange'):
         caps.pop(b'phases')
 
     # Don't advertise stream clone support in server mode if not configured.
     if role == b'server':
         streamsupported = repo.ui.configbool(
             b'server', b'uncompressed', untrusted=True
         )
         featuresupported = repo.ui.configbool(b'server', b'bundle2.stream')
 
         if not streamsupported or not featuresupported:
             caps.pop(b'stream')
     # Else always advertise support on client, because payload support
     # should always be advertised.
 
     if repo.ui.configbool(b'experimental', b'stream-v3'):
         if b'stream' in caps:
             caps[b'stream'] += (b'v3-exp',)
 
     # b'rev-branch-cache is no longer advertised, but still supported
     # for legacy clients.
 
     return caps
 
 
 def bundle2caps(remote):
     """return the bundle capabilities of a peer as dict"""
     raw = remote.capable(b'bundle2')
     if not raw and raw != b'':
         return {}
     capsblob = urlreq.unquote(remote.capable(b'bundle2'))
     return decodecaps(capsblob)
 
 
 def obsmarkersversion(caps):
     """extract the list of supported obsmarkers versions from a bundle2caps dict"""
     obscaps = caps.get(b'obsmarkers', ())
     return [int(c[1:]) for c in obscaps if c.startswith(b'V')]
 
 
 def writenewbundle(
     ui,
     repo,
     source,
     filename,
     bundletype,
     outgoing,
     opts,
     vfs=None,
     compression=None,
     compopts=None,
     allow_internal=False,
 ):
     if bundletype.startswith(b'HG10'):
         cg = changegroup.makechangegroup(repo, outgoing, b'01', source)
         return writebundle(
             ui,
             cg,
             filename,
             bundletype,
             vfs=vfs,
             compression=compression,
             compopts=compopts,
         )
     elif not bundletype.startswith(b'HG20'):
         raise error.ProgrammingError(b'unknown bundle type: %s' % bundletype)
 
     # enforce that no internal phase are to be bundled
     bundled_internal = repo.revs(b"%ln and _internal()", outgoing.ancestorsof)
     if bundled_internal and not allow_internal:
         count = len(repo.revs(b'%ln and _internal()', outgoing.missing))
         msg = "backup bundle would contains %d internal changesets"
         msg %= count
         raise error.ProgrammingError(msg)
 
     caps = {}
     if opts.get(b'obsolescence', False):
         caps[b'obsmarkers'] = (b'V1',)
     if opts.get(b'streamv2'):
         caps[b'stream'] = [b'v2']
     elif opts.get(b'streamv3-exp'):
         caps[b'stream'] = [b'v3-exp']
     bundle = bundle20(ui, caps)
     bundle.setcompression(compression, compopts)
     _addpartsfromopts(ui, repo, bundle, source, outgoing, opts)
     chunkiter = bundle.getchunks()
 
     return changegroup.writechunks(ui, chunkiter, filename, vfs=vfs)
 
 
 def _addpartsfromopts(ui, repo, bundler, source, outgoing, opts):
     # We should eventually reconcile this logic with the one behind
     # 'exchange.getbundle2partsgenerator'.
     #
     # The type of input from 'getbundle' and 'writenewbundle' are a bit
     # different right now. So we keep them separated for now for the sake of
     # simplicity.
 
     # we might not always want a changegroup in such bundle, for example in
     # stream bundles
     if opts.get(b'changegroup', True):
         cgversion = opts.get(b'cg.version')
         if cgversion is None:
             cgversion = changegroup.safeversion(repo)
         cg = changegroup.makechangegroup(repo, outgoing, cgversion, source)
         part = bundler.newpart(b'changegroup', data=cg.getchunks())
         part.addparam(b'version', cg.version)
         if b'clcount' in cg.extras:
             part.addparam(
                 b'nbchanges', b'%d' % cg.extras[b'clcount'], mandatory=False
             )
         if opts.get(b'phases'):
             target_phase = phases.draft
             for head in outgoing.ancestorsof:
                 target_phase = max(target_phase, repo[head].phase())
             if target_phase > phases.draft:
                 part.addparam(
                     b'targetphase',
                     b'%d' % target_phase,
                     mandatory=False,
                 )
     if repository.REPO_FEATURE_SIDE_DATA in repo.features:
         part.addparam(b'exp-sidedata', b'1')
 
     if opts.get(b'streamv2', False):
         addpartbundlestream2(bundler, repo, stream=True)
 
     if opts.get(b'streamv3-exp', False):
         addpartbundlestream2(bundler, repo, stream=True)
 
     if opts.get(b'tagsfnodescache', True):
         addparttagsfnodescache(repo, bundler, outgoing)
 
     if opts.get(b'revbranchcache', True):
         addpartrevbranchcache(repo, bundler, outgoing)
 
     if opts.get(b'obsolescence', False):
         obsmarkers = repo.obsstore.relevantmarkers(outgoing.missing)
         buildobsmarkerspart(
             bundler,
             obsmarkers,
             mandatory=opts.get(b'obsolescence-mandatory', True),
         )
 
     if opts.get(b'phases', False):
         headsbyphase = phases.subsetphaseheads(repo, outgoing.missing)
         phasedata = phases.binaryencode(headsbyphase)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 def addparttagsfnodescache(repo, bundler, outgoing):
     # we include the tags fnode cache for the bundle changeset
     # (as an optional parts)
     cache = tags.hgtagsfnodescache(repo.unfiltered())
     chunks = []
 
     # .hgtags fnodes are only relevant for head changesets. While we could
     # transfer values for all known nodes, there will likely be little to
     # no benefit.
     #
     # We don't bother using a generator to produce output data because
     # a) we only have 40 bytes per head and even esoteric numbers of heads
     # consume little memory (1M heads is 40MB) b) we don't want to send the
     # part if we don't have entries and knowing if we have entries requires
     # cache lookups.
     for node in outgoing.ancestorsof:
         # Don't compute missing, as this may slow down serving.
         fnode = cache.getfnode(node, computemissing=False)
         if fnode:
             chunks.extend([node, fnode])
 
     if chunks:
         bundler.newpart(b'hgtagsfnodes', data=b''.join(chunks))
 
 
 def addpartrevbranchcache(repo, bundler, outgoing):
     # we include the rev branch cache for the bundle changeset
     # (as an optional parts)
     cache = repo.revbranchcache()
     cl = repo.unfiltered().changelog
     branchesdata = collections.defaultdict(lambda: (set(), set()))
     for node in outgoing.missing:
         branch, close = cache.branchinfo(cl.rev(node))
         branchesdata[branch][close].add(node)
 
     def generate():
         for branch, (nodes, closed) in sorted(branchesdata.items()):
             utf8branch = encoding.fromlocal(branch)
             yield rbcstruct.pack(len(utf8branch), len(nodes), len(closed))
             yield utf8branch
             for n in sorted(nodes):
                 yield n
             for n in sorted(closed):
                 yield n
 
     bundler.newpart(b'cache:rev-branch-cache', data=generate(), mandatory=False)
 
 
 def _formatrequirementsspec(requirements):
     requirements = [req for req in requirements if req != b"shared"]
     return urlreq.quote(b','.join(sorted(requirements)))
 
 
 def _formatrequirementsparams(requirements):
     requirements = _formatrequirementsspec(requirements)
     params = b"%s%s" % (urlreq.quote(b"requirements="), requirements)
     return params
 
 
 def format_remote_wanted_sidedata(repo):
     """Formats a repo's wanted sidedata categories into a bytestring for
     capabilities exchange."""
     wanted = b""
     if repo._wanted_sidedata:
         wanted = b','.join(
             pycompat.bytestr(c) for c in sorted(repo._wanted_sidedata)
         )
     return wanted
 
 
 def read_remote_wanted_sidedata(remote):
     sidedata_categories = remote.capable(b'exp-wanted-sidedata')
     return read_wanted_sidedata(sidedata_categories)
 
 
 def read_wanted_sidedata(formatted):
     if formatted:
         return set(formatted.split(b','))
     return set()
 
 
 def addpartbundlestream2(bundler, repo, **kwargs):
     if not kwargs.get('stream', False):
         return
 
     if not streamclone.allowservergeneration(repo):
         msg = _(b'stream data requested but server does not allow this feature')
         hint = _(b'the client seems buggy')
         raise error.Abort(msg, hint=hint)
     if not (b'stream' in bundler.capabilities):
         msg = _(
             b'stream data requested but supported streaming clone versions were not specified'
         )
         hint = _(b'the client seems buggy')
         raise error.Abort(msg, hint=hint)
     client_supported = set(bundler.capabilities[b'stream'])
     server_supported = set(getrepocaps(repo, role=b'client').get(b'stream', []))
     common_supported = client_supported & server_supported
     if not common_supported:
         msg = _(b'no common supported version with the client: %s; %s')
         str_server = b','.join(sorted(server_supported))
         str_client = b','.join(sorted(client_supported))
         msg %= (str_server, str_client)
         raise error.Abort(msg)
     version = max(common_supported)
 
     # Stream clones don't compress well. And compression undermines a
     # goal of stream clones, which is to be fast. Communicate the desire
     # to avoid compression to consumers of the bundle.
     bundler.prefercompressed = False
 
     # get the includes and excludes
     includepats = kwargs.get('includepats')
     excludepats = kwargs.get('excludepats')
 
     narrowstream = repo.ui.configbool(
         b'experimental', b'server.stream-narrow-clones'
     )
 
     if (includepats or excludepats) and not narrowstream:
         raise error.Abort(_(b'server does not support narrow stream clones'))
 
     includeobsmarkers = False
     if repo.obsstore:
         remoteversions = obsmarkersversion(bundler.capabilities)
         if not remoteversions:
             raise error.Abort(
                 _(
                     b'server has obsolescence markers, but client '
                     b'cannot receive them via stream clone'
                 )
             )
         elif repo.obsstore._version in remoteversions:
             includeobsmarkers = True
 
     if version == b"v2":
         filecount, bytecount, it = streamclone.generatev2(
             repo, includepats, excludepats, includeobsmarkers
         )
         requirements = streamclone.streamed_requirements(repo)
         requirements = _formatrequirementsspec(requirements)
         part = bundler.newpart(b'stream2', data=it)
         part.addparam(b'bytecount', b'%d' % bytecount, mandatory=True)
         part.addparam(b'filecount', b'%d' % filecount, mandatory=True)
         part.addparam(b'requirements', requirements, mandatory=True)
     elif version == b"v3-exp":
         it = streamclone.generatev3(
             repo, includepats, excludepats, includeobsmarkers
         )
         requirements = streamclone.streamed_requirements(repo)
         requirements = _formatrequirementsspec(requirements)
         part = bundler.newpart(b'stream3-exp', data=it)
         part.addparam(b'requirements', requirements, mandatory=True)
 
 
 def buildobsmarkerspart(bundler, markers, mandatory=True):
     """add an obsmarker part to the bundler with <markers>
 
     No part is created if markers is empty.
     Raises ValueError if the bundler doesn't support any known obsmarker format.
     """
     if not markers:
         return None
 
     remoteversions = obsmarkersversion(bundler.capabilities)
     version = obsolete.commonversion(remoteversions)
     if version is None:
         raise ValueError(b'bundler does not support common obsmarker format')
     stream = obsolete.encodemarkers(markers, True, version=version)
     return bundler.newpart(b'obsmarkers', data=stream, mandatory=mandatory)
 
 
 def writebundle(
     ui, cg, filename, bundletype, vfs=None, compression=None, compopts=None
 ):
     """Write a bundle file and return its filename.
 
     Existing files will not be overwritten.
     If no filename is specified, a temporary file is created.
     bz2 compression can be turned off.
     The bundle file will be deleted in case of errors.
     """
 
     if bundletype == b"HG20":
         bundle = bundle20(ui)
         bundle.setcompression(compression, compopts)
         part = bundle.newpart(b'changegroup', data=cg.getchunks())
         part.addparam(b'version', cg.version)
         if b'clcount' in cg.extras:
             part.addparam(
                 b'nbchanges', b'%d' % cg.extras[b'clcount'], mandatory=False
             )
         chunkiter = bundle.getchunks()
     else:
         # compression argument is only for the bundle2 case
         assert compression is None
         if cg.version != b'01':
             raise error.Abort(
                 _(b'old bundle types only supports v1 changegroups')
             )
 
         # HG20 is the case without 2 values to unpack, but is handled above.
         # pytype: disable=bad-unpacking
         header, comp = bundletypes[bundletype]
         # pytype: enable=bad-unpacking
 
         if comp not in util.compengines.supportedbundletypes:
             raise error.Abort(_(b'unknown stream compression type: %s') % comp)
         compengine = util.compengines.forbundletype(comp)
 
         def chunkiter():
             yield header
             for chunk in compengine.compressstream(cg.getchunks(), compopts):
                 yield chunk
 
         chunkiter = chunkiter()
 
     # parse the changegroup data, otherwise we will block
     # in case of sshrepo because we don't know the end of the stream
     return changegroup.writechunks(ui, chunkiter, filename, vfs=vfs)
 
 
 def combinechangegroupresults(op):
     """logic to combine 0 or more addchangegroup results into one"""
     results = [r.get(b'return', 0) for r in op.records[b'changegroup']]
     changedheads = 0
     result = 1
     for ret in results:
         # If any changegroup result is 0, return 0
         if ret == 0:
             result = 0
             break
         if ret < -1:
             changedheads += ret + 1
         elif ret > 1:
             changedheads += ret - 1
     if changedheads > 0:
         result = 1 + changedheads
     elif changedheads < 0:
         result = -1 + changedheads
     return result
 
 
 @parthandler(
     b'changegroup',
     (
         b'version',
         b'nbchanges',
         b'exp-sidedata',
         b'exp-wanted-sidedata',
         b'treemanifest',
         b'targetphase',
     ),
 )
 def handlechangegroup(op, inpart):
     """apply a changegroup part on the repo"""
     from . import localrepo
 
     tr = op.gettransaction()
     unpackerversion = inpart.params.get(b'version', b'01')
     # We should raise an appropriate exception here
     cg = changegroup.getunbundler(unpackerversion, inpart, None)
     # the source and url passed here are overwritten by the one contained in
     # the transaction.hookargs argument. So 'bundle2' is a placeholder
     nbchangesets = None
     if b'nbchanges' in inpart.params:
         nbchangesets = int(inpart.params.get(b'nbchanges'))
     if b'treemanifest' in inpart.params and not scmutil.istreemanifest(op.repo):
         if len(op.repo.changelog) != 0:
             raise error.Abort(
                 _(
                     b"bundle contains tree manifests, but local repo is "
                     b"non-empty and does not use tree manifests"
                 )
             )
         op.repo.requirements.add(requirements.TREEMANIFEST_REQUIREMENT)
         op.repo.svfs.options = localrepo.resolvestorevfsoptions(
             op.repo.ui, op.repo.requirements, op.repo.features
         )
         scmutil.writereporequirements(op.repo)
 
     extrakwargs = {}
     targetphase = inpart.params.get(b'targetphase')
     if targetphase is not None:
         extrakwargs['targetphase'] = int(targetphase)
 
     remote_sidedata = inpart.params.get(b'exp-wanted-sidedata')
     extrakwargs['sidedata_categories'] = read_wanted_sidedata(remote_sidedata)
 
     ret = _processchangegroup(
         op,
         cg,
         tr,
         op.source,
         b'bundle2',
         expectedtotal=nbchangesets,
         **extrakwargs
     )
     if op.reply is not None:
         # This is definitely not the final form of this
         # return. But one need to start somewhere.
         part = op.reply.newpart(b'reply:changegroup', mandatory=False)
         part.addparam(
             b'in-reply-to', pycompat.bytestr(inpart.id), mandatory=False
         )
         part.addparam(b'return', b'%i' % ret, mandatory=False)
     assert not inpart.read()
 
 
 _remotechangegroupparams = tuple(
     [b'url', b'size', b'digests']
     + [b'digest:%s' % k for k in util.DIGESTS.keys()]
 )
 
 
 @parthandler(b'remote-changegroup', _remotechangegroupparams)
 def handleremotechangegroup(op, inpart):
     """apply a bundle10 on the repo, given an url and validation information
 
     All the information about the remote bundle to import are given as
     parameters. The parameters include:
       - url: the url to the bundle10.
       - size: the bundle10 file size. It is used to validate what was
         retrieved by the client matches the server knowledge about the bundle.
       - digests: a space separated list of the digest types provided as
         parameters.
       - digest:<digest-type>: the hexadecimal representation of the digest with
         that name. Like the size, it is used to validate what was retrieved by
         the client matches what the server knows about the bundle.
 
     When multiple digest types are given, all of them are checked.
     """
     try:
         raw_url = inpart.params[b'url']
     except KeyError:
         raise error.Abort(_(b'remote-changegroup: missing "%s" param') % b'url')
     parsed_url = urlutil.url(raw_url)
     if parsed_url.scheme not in capabilities[b'remote-changegroup']:
         raise error.Abort(
             _(b'remote-changegroup does not support %s urls')
             % parsed_url.scheme
         )
 
     try:
         size = int(inpart.params[b'size'])
     except ValueError:
         raise error.Abort(
             _(b'remote-changegroup: invalid value for param "%s"') % b'size'
         )
     except KeyError:
         raise error.Abort(
             _(b'remote-changegroup: missing "%s" param') % b'size'
         )
 
     digests = {}
     for typ in inpart.params.get(b'digests', b'').split():
         param = b'digest:%s' % typ
         try:
             value = inpart.params[param]
         except KeyError:
             raise error.Abort(
                 _(b'remote-changegroup: missing "%s" param') % param
             )
         digests[typ] = value
 
     real_part = util.digestchecker(url.open(op.ui, raw_url), size, digests)
 
     tr = op.gettransaction()
     from . import exchange
 
     cg = exchange.readbundle(op.repo.ui, real_part, raw_url)
     if not isinstance(cg, changegroup.cg1unpacker):
         raise error.Abort(
             _(b'%s: not a bundle version 1.0') % urlutil.hidepassword(raw_url)
         )
     ret = _processchangegroup(op, cg, tr, op.source, b'bundle2')
     if op.reply is not None:
         # This is definitely not the final form of this
         # return. But one need to start somewhere.
         part = op.reply.newpart(b'reply:changegroup')
         part.addparam(
             b'in-reply-to', pycompat.bytestr(inpart.id), mandatory=False
         )
         part.addparam(b'return', b'%i' % ret, mandatory=False)
     try:
         real_part.validate()
     except error.Abort as e:
         raise error.Abort(
             _(b'bundle at %s is corrupted:\n%s')
             % (urlutil.hidepassword(raw_url), e.message)
         )
     assert not inpart.read()
 
 
 @parthandler(b'reply:changegroup', (b'return', b'in-reply-to'))
 def handlereplychangegroup(op, inpart):
     ret = int(inpart.params[b'return'])
     replyto = int(inpart.params[b'in-reply-to'])
     op.records.add(b'changegroup', {b'return': ret}, replyto)
 
 
 @parthandler(b'check:bookmarks')
 def handlecheckbookmarks(op, inpart):
     """check location of bookmarks
 
     This part is to be used to detect push race regarding bookmark, it
     contains binary encoded (bookmark, node) tuple. If the local state does
     not marks the one in the part, a PushRaced exception is raised
     """
     bookdata = bookmarks.binarydecode(op.repo, inpart)
 
     msgstandard = (
         b'remote repository changed while pushing - please try again '
         b'(bookmark "%s" move from %s to %s)'
     )
     msgmissing = (
         b'remote repository changed while pushing - please try again '
         b'(bookmark "%s" is missing, expected %s)'
     )
     msgexist = (
         b'remote repository changed while pushing - please try again '
         b'(bookmark "%s" set on %s, expected missing)'
     )
     for book, node in bookdata:
         currentnode = op.repo._bookmarks.get(book)
         if currentnode != node:
             if node is None:
                 finalmsg = msgexist % (book, short(currentnode))
             elif currentnode is None:
                 finalmsg = msgmissing % (book, short(node))
             else:
                 finalmsg = msgstandard % (
                     book,
                     short(node),
                     short(currentnode),
                 )
             raise error.PushRaced(finalmsg)
 
 
 @parthandler(b'check:heads')
 def handlecheckheads(op, inpart):
     """check that head of the repo did not change
 
     This is used to detect a push race when using unbundle.
     This replaces the "heads" argument of unbundle."""
     h = inpart.read(20)
     heads = []
     while len(h) == 20:
         heads.append(h)
         h = inpart.read(20)
     assert not h
     # Trigger a transaction so that we are guaranteed to have the lock now.
     if op.ui.configbool(b'experimental', b'bundle2lazylocking'):
         op.gettransaction()
     if sorted(heads) != sorted(op.repo.heads()):
         raise error.PushRaced(
             b'remote repository changed while pushing - please try again'
         )
 
 
 @parthandler(b'check:updated-heads')
 def handlecheckupdatedheads(op, inpart):
     """check for race on the heads touched by a push
 
     This is similar to 'check:heads' but focus on the heads actually updated
     during the push. If other activities happen on unrelated heads, it is
     ignored.
 
     This allow server with high traffic to avoid push contention as long as
     unrelated parts of the graph are involved."""
     h = inpart.read(20)
     heads = []
     while len(h) == 20:
         heads.append(h)
         h = inpart.read(20)
     assert not h
     # trigger a transaction so that we are guaranteed to have the lock now.
     if op.ui.configbool(b'experimental', b'bundle2lazylocking'):
         op.gettransaction()
 
     currentheads = set()
     for ls in op.repo.branchmap().iterheads():
         currentheads.update(ls)
 
     for h in heads:
         if h not in currentheads:
             raise error.PushRaced(
                 b'remote repository changed while pushing - '
                 b'please try again'
             )
 
 
 @parthandler(b'check:phases')
 def handlecheckphases(op, inpart):
     """check that phase boundaries of the repository did not change
 
     This is used to detect a push race.
     """
     phasetonodes = phases.binarydecode(inpart)
     unfi = op.repo.unfiltered()
     cl = unfi.changelog
     phasecache = unfi._phasecache
     msg = (
         b'remote repository changed while pushing - please try again '
         b'(%s is %s expected %s)'
     )
     for expectedphase, nodes in phasetonodes.items():
         for n in nodes:
             actualphase = phasecache.phase(unfi, cl.rev(n))
             if actualphase != expectedphase:
                 finalmsg = msg % (
                     short(n),
                     phases.phasenames[actualphase],
                     phases.phasenames[expectedphase],
                 )
                 raise error.PushRaced(finalmsg)
 
 
 @parthandler(b'output')
 def handleoutput(op, inpart):
     """forward output captured on the server to the client"""
     for line in inpart.read().splitlines():
         op.ui.status(_(b'remote: %s\n') % line)
 
 
 @parthandler(b'replycaps')
 def handlereplycaps(op, inpart):
     """Notify that a reply bundle should be created
 
     The payload contains the capabilities information for the reply"""
     caps = decodecaps(inpart.read())
     if op.reply is None:
         op.reply = bundle20(op.ui, caps)
 
 
 class AbortFromPart(error.Abort):
     """Sub-class of Abort that denotes an error from a bundle2 part."""
 
 
 @parthandler(b'error:abort', (b'message', b'hint'))
 def handleerrorabort(op, inpart):
     """Used to transmit abort error over the wire"""
     raise AbortFromPart(
         inpart.params[b'message'], hint=inpart.params.get(b'hint')
     )
 
 
 @parthandler(
     b'error:pushkey',
     (b'namespace', b'key', b'new', b'old', b'ret', b'in-reply-to'),
 )
 def handleerrorpushkey(op, inpart):
     """Used to transmit failure of a mandatory pushkey over the wire"""
     kwargs = {}
     for name in (b'namespace', b'key', b'new', b'old', b'ret'):
         value = inpart.params.get(name)
         if value is not None:
             kwargs[name] = value
     raise error.PushkeyFailed(
         inpart.params[b'in-reply-to'], **pycompat.strkwargs(kwargs)
     )
 
 
 @parthandler(b'error:unsupportedcontent', (b'parttype', b'params'))
 def handleerrorunsupportedcontent(op, inpart):
     """Used to transmit unknown content error over the wire"""
     kwargs = {}
     parttype = inpart.params.get(b'parttype')
     if parttype is not None:
         kwargs[b'parttype'] = parttype
     params = inpart.params.get(b'params')
     if params is not None:
         kwargs[b'params'] = params.split(b'\0')
 
     raise error.BundleUnknownFeatureError(**pycompat.strkwargs(kwargs))
 
 
 @parthandler(b'error:pushraced', (b'message',))
 def handleerrorpushraced(op, inpart):
     """Used to transmit push race error over the wire"""
     raise error.ResponseError(_(b'push failed:'), inpart.params[b'message'])
 
 
 @parthandler(b'listkeys', (b'namespace',))
 def handlelistkeys(op, inpart):
     """retrieve pushkey namespace content stored in a bundle2"""
     namespace = inpart.params[b'namespace']
     r = pushkey.decodekeys(inpart.read())
     op.records.add(b'listkeys', (namespace, r))
 
 
 @parthandler(b'pushkey', (b'namespace', b'key', b'old', b'new'))
 def handlepushkey(op, inpart):
     """process a pushkey request"""
     dec = pushkey.decode
     namespace = dec(inpart.params[b'namespace'])
     key = dec(inpart.params[b'key'])
     old = dec(inpart.params[b'old'])
     new = dec(inpart.params[b'new'])
     # Grab the transaction to ensure that we have the lock before performing the
     # pushkey.
     if op.ui.configbool(b'experimental', b'bundle2lazylocking'):
         op.gettransaction()
     ret = op.repo.pushkey(namespace, key, old, new)
     record = {b'namespace': namespace, b'key': key, b'old': old, b'new': new}
     op.records.add(b'pushkey', record)
     if op.reply is not None:
         rpart = op.reply.newpart(b'reply:pushkey')
         rpart.addparam(
             b'in-reply-to', pycompat.bytestr(inpart.id), mandatory=False
         )
         rpart.addparam(b'return', b'%i' % ret, mandatory=False)
     if inpart.mandatory and not ret:
         kwargs = {}
         for key in (b'namespace', b'key', b'new', b'old', b'ret'):
             if key in inpart.params:
                 kwargs[key] = inpart.params[key]
         raise error.PushkeyFailed(
             partid=b'%d' % inpart.id, **pycompat.strkwargs(kwargs)
         )
 
 
 @parthandler(b'bookmarks')
 def handlebookmark(op, inpart):
     """transmit bookmark information
 
     The part contains binary encoded bookmark information.
 
     The exact behavior of this part can be controlled by the 'bookmarks' mode
     on the bundle operation.
 
     When mode is 'apply' (the default) the bookmark information is applied as
     is to the unbundling repository. Make sure a 'check:bookmarks' part is
     issued earlier to check for push races in such update. This behavior is
     suitable for pushing.
 
     When mode is 'records', the information is recorded into the 'bookmarks'
     records of the bundle operation. This behavior is suitable for pulling.
     """
     changes = bookmarks.binarydecode(op.repo, inpart)
 
     pushkeycompat = op.repo.ui.configbool(
         b'server', b'bookmarks-pushkey-compat'
     )
     bookmarksmode = op.modes.get(b'bookmarks', b'apply')
 
     if bookmarksmode == b'apply':
         tr = op.gettransaction()
         bookstore = op.repo._bookmarks
         if pushkeycompat:
             allhooks = []
             for book, node in changes:
                 hookargs = tr.hookargs.copy()
                 hookargs[b'pushkeycompat'] = b'1'
                 hookargs[b'namespace'] = b'bookmarks'
                 hookargs[b'key'] = book
                 hookargs[b'old'] = hex(bookstore.get(book, b''))
                 hookargs[b'new'] = hex(node if node is not None else b'')
                 allhooks.append(hookargs)
 
             for hookargs in allhooks:
                 op.repo.hook(
                     b'prepushkey', throw=True, **pycompat.strkwargs(hookargs)
                 )
 
         for book, node in changes:
             if bookmarks.isdivergent(book):
                 msg = _(b'cannot accept divergent bookmark %s!') % book
                 raise error.Abort(msg)
 
         bookstore.applychanges(op.repo, op.gettransaction(), changes)
 
         if pushkeycompat:
 
             def runhook(unused_success):
                 for hookargs in allhooks:
                     op.repo.hook(b'pushkey', **pycompat.strkwargs(hookargs))
 
             op.repo._afterlock(runhook)
 
     elif bookmarksmode == b'records':
         for book, node in changes:
             record = {b'bookmark': book, b'node': node}
             op.records.add(b'bookmarks', record)
     else:
         raise error.ProgrammingError(
             b'unknown bookmark mode: %s' % bookmarksmode
         )
 
 
 @parthandler(b'phase-heads')
 def handlephases(op, inpart):
     """apply phases from bundle part to repo"""
     headsbyphase = phases.binarydecode(inpart)
     phases.updatephases(op.repo.unfiltered(), op.gettransaction, headsbyphase)
 
 
 @parthandler(b'reply:pushkey', (b'return', b'in-reply-to'))
 def handlepushkeyreply(op, inpart):
     """retrieve the result of a pushkey request"""
     ret = int(inpart.params[b'return'])
     partid = int(inpart.params[b'in-reply-to'])
     op.records.add(b'pushkey', {b'return': ret}, partid)
 
 
 @parthandler(b'obsmarkers')
 def handleobsmarker(op, inpart):
     """add a stream of obsmarkers to the repo"""
     tr = op.gettransaction()
     markerdata = inpart.read()
     if op.ui.config(b'experimental', b'obsmarkers-exchange-debug'):
         op.ui.writenoi18n(
             b'obsmarker-exchange: %i bytes received\n' % len(markerdata)
         )
     # The mergemarkers call will crash if marker creation is not enabled.
     # we want to avoid this if the part is advisory.
     if not inpart.mandatory and op.repo.obsstore.readonly:
         op.repo.ui.debug(
             b'ignoring obsolescence markers, feature not enabled\n'
         )
         return
     new = op.repo.obsstore.mergemarkers(tr, markerdata)
     op.repo.invalidatevolatilesets()
     op.records.add(b'obsmarkers', {b'new': new})
     if op.reply is not None:
         rpart = op.reply.newpart(b'reply:obsmarkers')
         rpart.addparam(
             b'in-reply-to', pycompat.bytestr(inpart.id), mandatory=False
         )
         rpart.addparam(b'new', b'%i' % new, mandatory=False)
 
 
 @parthandler(b'reply:obsmarkers', (b'new', b'in-reply-to'))
 def handleobsmarkerreply(op, inpart):
     """retrieve the result of a pushkey request"""
     ret = int(inpart.params[b'new'])
     partid = int(inpart.params[b'in-reply-to'])
     op.records.add(b'obsmarkers', {b'new': ret}, partid)
 
 
 @parthandler(b'hgtagsfnodes')
 def handlehgtagsfnodes(op, inpart):
     """Applies .hgtags fnodes cache entries to the local repo.
 
     Payload is pairs of 20 byte changeset nodes and filenodes.
     """
     # Grab the transaction so we ensure that we have the lock at this point.
     if op.ui.configbool(b'experimental', b'bundle2lazylocking'):
         op.gettransaction()
     cache = tags.hgtagsfnodescache(op.repo.unfiltered())
 
     count = 0
     while True:
         node = inpart.read(20)
         fnode = inpart.read(20)
         if len(node) < 20 or len(fnode) < 20:
             op.ui.debug(b'ignoring incomplete received .hgtags fnodes data\n')
             break
         cache.setfnode(node, fnode)
         count += 1
 
     cache.write()
     op.ui.debug(b'applied %i hgtags fnodes cache entries\n' % count)
 
 
 rbcstruct = struct.Struct(b'>III')
 
 
 @parthandler(b'cache:rev-branch-cache')
 def handlerbc(op, inpart):
     """Legacy part, ignored for compatibility with bundles from or
     for Mercurial before 5.7. Newer Mercurial computes the cache
     efficiently enough during unbundling that the additional transfer
     is unnecessary."""
 
 
 @parthandler(b'pushvars')
 def bundle2getvars(op, part):
     '''unbundle a bundle2 containing shellvars on the server'''
     # An option to disable unbundling on server-side for security reasons
     if op.ui.configbool(b'push', b'pushvars.server'):
         hookargs = {}
         for key, value in part.advisoryparams:
             key = key.upper()
             # We want pushed variables to have USERVAR_ prepended so we know
             # they came from the --pushvar flag.
             key = b"USERVAR_" + key
             hookargs[key] = value
         op.addhookargs(hookargs)
 
 
 @parthandler(b'stream2', (b'requirements', b'filecount', b'bytecount'))
 def handlestreamv2bundle(op, part):
 
     requirements = urlreq.unquote(part.params[b'requirements'])
     requirements = requirements.split(b',') if requirements else []
     filecount = int(part.params[b'filecount'])
     bytecount = int(part.params[b'bytecount'])
 
     repo = op.repo
     if len(repo):
         msg = _(b'cannot apply stream clone to non empty repository')
         raise error.Abort(msg)
 
     repo.ui.debug(b'applying stream bundle\n')
     streamclone.applybundlev2(repo, part, filecount, bytecount, requirements)
 
 
 @parthandler(b'stream3-exp', (b'requirements',))
 def handlestreamv3bundle(op, part):
     requirements = urlreq.unquote(part.params[b'requirements'])
     requirements = requirements.split(b',') if requirements else []
 
     repo = op.repo
     if len(repo):
         msg = _(b'cannot apply stream clone to non empty repository')
         raise error.Abort(msg)
 
     repo.ui.debug(b'applying stream bundle\n')
     streamclone.applybundlev3(repo, part, requirements)
 
 
 def widen_bundle(
     bundler, repo, oldmatcher, newmatcher, common, known, cgversion, ellipses
 ):
     """generates bundle2 for widening a narrow clone
 
     bundler is the bundle to which data should be added
     repo is the localrepository instance
     oldmatcher matches what the client already has
     newmatcher matches what the client needs (including what it already has)
     common is set of common heads between server and client
     known is a set of revs known on the client side (used in ellipses)
     cgversion is the changegroup version to send
     ellipses is boolean value telling whether to send ellipses data or not
 
     returns bundle2 of the data required for extending
     """
     commonnodes = set()
     cl = repo.changelog
     for r in repo.revs(b"::%ln", common):
         commonnodes.add(cl.node(r))
     if commonnodes:
         packer = changegroup.getbundler(
             cgversion,
             repo,
             oldmatcher=oldmatcher,
             matcher=newmatcher,
             fullnodes=commonnodes,
         )
         cgdata = packer.generate(
             {repo.nullid},
             list(commonnodes),
             False,
             b'narrow_widen',
             changelog=False,
         )
 
         part = bundler.newpart(b'changegroup', data=cgdata)
         part.addparam(b'version', cgversion)
         if scmutil.istreemanifest(repo):
             part.addparam(b'treemanifest', b'1')
     if repository.REPO_FEATURE_SIDE_DATA in repo.features:
         part.addparam(b'exp-sidedata', b'1')
         wanted = format_remote_wanted_sidedata(repo)
         part.addparam(b'exp-wanted-sidedata', wanted)
 
     return bundler

mercurial/upgrade_utils/actions.py

@@ -1,1122 +1,1121 b''
 # upgrade.py - functions for in place upgrade of Mercurial repository
 #
 # Copyright (c) 2016-present, Gregory Szorc
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 from ..i18n import _
 from .. import (
     error,
     localrepo,
     pycompat,
     requirements,
     revlog,
     util,
 )
 
 from ..utils import compression
 
 if pycompat.TYPE_CHECKING:
     from typing import (
         List,
         Type,
     )
 
 
 # list of requirements that request a clone of all revlog if added/removed
 RECLONES_REQUIREMENTS = {
     requirements.GENERALDELTA_REQUIREMENT,
     requirements.SPARSEREVLOG_REQUIREMENT,
     requirements.REVLOGV2_REQUIREMENT,
     requirements.CHANGELOGV2_REQUIREMENT,
 }
 
 
 def preservedrequirements(repo):
     preserved = {
         requirements.SHARED_REQUIREMENT,
         requirements.NARROW_REQUIREMENT,
     }
     return preserved & repo.requirements
 
 
 FORMAT_VARIANT = b'deficiency'
 OPTIMISATION = b'optimization'
 
 
 class improvement:
     """Represents an improvement that can be made as part of an upgrade."""
 
     ### The following attributes should be defined for each subclass:
 
     # Either ``FORMAT_VARIANT`` or ``OPTIMISATION``.
     # A format variant is where we change the storage format. Not all format
     # variant changes are an obvious problem.
     # An optimization is an action (sometimes optional) that
     # can be taken to further improve the state of the repository.
     type = None
 
     # machine-readable string uniquely identifying this improvement. it will be
     # mapped to an action later in the upgrade process.
     name = None
 
     # message intended for humans explaining the improvement in more detail,
     # including the implications of it ``FORMAT_VARIANT`` types, should be
     # worded
     # in the present tense.
     description = None
 
     # message intended for humans explaining what an upgrade addressing this
     # issue will do. should be worded in the future tense.
     upgrademessage = None
 
     # value of current Mercurial default for new repository
     default = None
 
     # Message intended for humans which will be shown post an upgrade
     # operation when the improvement will be added
     postupgrademessage = None
 
     # Message intended for humans which will be shown post an upgrade
     # operation in which this improvement was removed
     postdowngrademessage = None
 
     # By default we assume that every improvement touches requirements and all revlogs
 
     # Whether this improvement touches filelogs
     touches_filelogs = True
 
     # Whether this improvement touches manifests
     touches_manifests = True
 
     # Whether this improvement touches changelog
     touches_changelog = True
 
     # Whether this improvement changes repository requirements
     touches_requirements = True
 
     # Whether this improvement touches the dirstate
     touches_dirstate = False
 
     # Can this action be run on a share instead of its mains repository
     compatible_with_share = False
 
 
 allformatvariant = []  # type: List[Type['formatvariant']]
 
 
 def registerformatvariant(cls):
     allformatvariant.append(cls)
     return cls
 
 
 class formatvariant(improvement):
     """an improvement subclass dedicated to repository format"""
 
     type = FORMAT_VARIANT
 
     @staticmethod
     def fromrepo(repo):
         """current value of the variant in the repository"""
         raise NotImplementedError()
 
     @staticmethod
     def fromconfig(repo):
         """current value of the variant in the configuration"""
         raise NotImplementedError()
 
 
 class requirementformatvariant(formatvariant):
     """formatvariant based on a 'requirement' name.
 
     Many format variant are controlled by a 'requirement'. We define a small
     subclass to factor the code.
     """
 
     # the requirement that control this format variant
     _requirement = None
 
     @staticmethod
     def _newreporequirements(ui):
         return localrepo.newreporequirements(
             ui, localrepo.defaultcreateopts(ui)
         )
 
     @classmethod
     def fromrepo(cls, repo):
         assert cls._requirement is not None
         return cls._requirement in repo.requirements
 
     @classmethod
     def fromconfig(cls, repo):
         assert cls._requirement is not None
         return cls._requirement in cls._newreporequirements(repo.ui)
 
 
 @registerformatvariant
 class fncache(requirementformatvariant):
     name = b'fncache'
 
     _requirement = requirements.FNCACHE_REQUIREMENT
 
     default = True
 
     description = _(
         b'long and reserved filenames may not work correctly; '
         b'repository performance is sub-optimal'
     )
 
     upgrademessage = _(
         b'repository will be more resilient to storing '
         b'certain paths and performance of certain '
         b'operations should be improved'
     )
 
 
 @registerformatvariant
 class dirstatev2(requirementformatvariant):
     name = b'dirstate-v2'
     _requirement = requirements.DIRSTATE_V2_REQUIREMENT
 
     default = False
 
     description = _(
         b'version 1 of the dirstate file format requires '
         b'reading and parsing it all at once.\n'
         b'Version 2 has a better structure,'
         b'better information and lighter update mechanism'
     )
 
     upgrademessage = _(b'"hg status" will be faster')
 
     touches_filelogs = False
     touches_manifests = False
     touches_changelog = False
     touches_requirements = True
     touches_dirstate = True
     compatible_with_share = True
 
 
 @registerformatvariant
 class dirstatetrackedkey(requirementformatvariant):
     name = b'tracked-hint'
     _requirement = requirements.DIRSTATE_TRACKED_HINT_V1
 
     default = False
 
     description = _(
         b'Add a small file to help external tooling that watch the tracked set'
     )
 
     upgrademessage = _(
         b'external tools will be informated of potential change in the tracked set'
     )
 
     touches_filelogs = False
     touches_manifests = False
     touches_changelog = False
     touches_requirements = True
     touches_dirstate = True
     compatible_with_share = True
 
 
 @registerformatvariant
 class dotencode(requirementformatvariant):
     name = b'dotencode'
 
     _requirement = requirements.DOTENCODE_REQUIREMENT
 
     default = True
 
     description = _(
         b'storage of filenames beginning with a period or '
         b'space may not work correctly'
     )
 
     upgrademessage = _(
         b'repository will be better able to store files '
         b'beginning with a space or period'
     )
 
 
 @registerformatvariant
 class generaldelta(requirementformatvariant):
     name = b'generaldelta'
 
     _requirement = requirements.GENERALDELTA_REQUIREMENT
 
     default = True
 
     description = _(
         b'deltas within internal storage are unable to '
         b'choose optimal revisions; repository is larger and '
         b'slower than it could be; interaction with other '
         b'repositories may require extra network and CPU '
         b'resources, making "hg push" and "hg pull" slower'
     )
 
     upgrademessage = _(
         b'repository storage will be able to create '
         b'optimal deltas; new repository data will be '
         b'smaller and read times should decrease; '
         b'interacting with other repositories using this '
         b'storage model should require less network and '
         b'CPU resources, making "hg push" and "hg pull" '
         b'faster'
     )
 
 
 @registerformatvariant
 class sharesafe(requirementformatvariant):
     name = b'share-safe'
     _requirement = requirements.SHARESAFE_REQUIREMENT
 
     default = True
 
     description = _(
         b'old shared repositories do not share source repository '
         b'requirements and config. This leads to various problems '
         b'when the source repository format is upgraded or some new '
         b'extensions are enabled.'
     )
 
     upgrademessage = _(
         b'Upgrades a repository to share-safe format so that future '
         b'shares of this repository share its requirements and configs.'
     )
 
     postdowngrademessage = _(
         b'repository downgraded to not use share safe mode, '
-        b'existing shares will not work and needs to'
-        b' be reshared.'
+        b'existing shares will not work and need to be reshared.'
     )
 
     postupgrademessage = _(
         b'repository upgraded to share safe mode, existing'
         b' shares will still work in old non-safe mode. '
         b'Re-share existing shares to use them in safe mode'
         b' New shares will be created in safe mode.'
     )
 
     # upgrade only needs to change the requirements
     touches_filelogs = False
     touches_manifests = False
     touches_changelog = False
     touches_requirements = True
 
 
 @registerformatvariant
 class sparserevlog(requirementformatvariant):
     name = b'sparserevlog'
 
     _requirement = requirements.SPARSEREVLOG_REQUIREMENT
 
     default = True
 
     description = _(
         b'in order to limit disk reading and memory usage on older '
         b'version, the span of a delta chain from its root to its '
         b'end is limited, whatever the relevant data in this span. '
         b'This can severly limit Mercurial ability to build good '
         b'chain of delta resulting is much more storage space being '
         b'taken and limit reusability of on disk delta during '
         b'exchange.'
     )
 
     upgrademessage = _(
         b'Revlog supports delta chain with more unused data '
         b'between payload. These gaps will be skipped at read '
         b'time. This allows for better delta chains, making a '
         b'better compression and faster exchange with server.'
     )
 
 
 @registerformatvariant
 class persistentnodemap(requirementformatvariant):
     name = b'persistent-nodemap'
 
     _requirement = requirements.NODEMAP_REQUIREMENT
 
     default = False
 
     description = _(
         b'persist the node -> rev mapping on disk to speedup lookup'
     )
 
     upgrademessage = _(b'Speedup revision lookup by node id.')
 
 
 @registerformatvariant
 class copiessdc(requirementformatvariant):
     name = b'copies-sdc'
 
     _requirement = requirements.COPIESSDC_REQUIREMENT
 
     default = False
 
     description = _(b'Stores copies information alongside changesets.')
 
     upgrademessage = _(
-        b'Allows to use more efficient algorithm to deal with ' b'copy tracing.'
+        b'Allows to use more efficient algorithm to deal with copy tracing.'
     )
 
     touches_filelogs = False
     touches_manifests = False
 
 
 @registerformatvariant
 class revlogv2(requirementformatvariant):
     name = b'revlog-v2'
     _requirement = requirements.REVLOGV2_REQUIREMENT
     default = False
     description = _(b'Version 2 of the revlog.')
     upgrademessage = _(b'very experimental')
 
 
 @registerformatvariant
 class changelogv2(requirementformatvariant):
     name = b'changelog-v2'
     _requirement = requirements.CHANGELOGV2_REQUIREMENT
     default = False
     description = _(b'An iteration of the revlog focussed on changelog needs.')
     upgrademessage = _(b'quite experimental')
 
     touches_filelogs = False
     touches_manifests = False
 
 
 @registerformatvariant
 class removecldeltachain(formatvariant):
     name = b'plain-cl-delta'
 
     default = True
 
     description = _(
         b'changelog storage is using deltas instead of '
         b'raw entries; changelog reading and any '
         b'operation relying on changelog data are slower '
         b'than they could be'
     )
 
     upgrademessage = _(
         b'changelog storage will be reformated to '
         b'store raw entries; changelog reading will be '
         b'faster; changelog size may be reduced'
     )
 
     @staticmethod
     def fromrepo(repo):
         # Mercurial 4.0 changed changelogs to not use delta chains. Search for
         # changelogs with deltas.
         cl = repo.changelog
         chainbase = cl.chainbase
         return all(rev == chainbase(rev) for rev in cl)
 
     @staticmethod
     def fromconfig(repo):
         return True
 
 
 _has_zstd = (
     b'zstd' in util.compengines
     and util.compengines[b'zstd'].available()
     and util.compengines[b'zstd'].revlogheader()
 )
 
 
 @registerformatvariant
 class compressionengine(formatvariant):
     name = b'compression'
 
     if _has_zstd:
         default = b'zstd'
     else:
         default = b'zlib'
 
     description = _(
         b'Compresion algorithm used to compress data. '
         b'Some engine are faster than other'
     )
 
     upgrademessage = _(
         b'revlog content will be recompressed with the new algorithm.'
     )
 
     @classmethod
     def fromrepo(cls, repo):
         # we allow multiple compression engine requirement to co-exist because
         # strickly speaking, revlog seems to support mixed compression style.
         #
         # The compression used for new entries will be "the last one"
         compression = b'zlib'
         for req in repo.requirements:
             prefix = req.startswith
             if prefix(b'revlog-compression-') or prefix(b'exp-compression-'):
                 compression = req.split(b'-', 2)[2]
         return compression
 
     @classmethod
     def fromconfig(cls, repo):
         compengines = repo.ui.configlist(b'format', b'revlog-compression')
         # return the first valid value as the selection code would do
         for comp in compengines:
             if comp in util.compengines:
                 e = util.compengines[comp]
                 if e.available() and e.revlogheader():
                     return comp
 
         # no valide compression found lets display it all for clarity
         return b','.join(compengines)
 
 
 @registerformatvariant
 class compressionlevel(formatvariant):
     name = b'compression-level'
     default = b'default'
 
     description = _(b'compression level')
 
     upgrademessage = _(b'revlog content will be recompressed')
 
     @classmethod
     def fromrepo(cls, repo):
         comp = compressionengine.fromrepo(repo)
         level = None
         if comp == b'zlib':
             level = repo.ui.configint(b'storage', b'revlog.zlib.level')
         elif comp == b'zstd':
             level = repo.ui.configint(b'storage', b'revlog.zstd.level')
         if level is None:
             return b'default'
         return bytes(level)
 
     @classmethod
     def fromconfig(cls, repo):
         comp = compressionengine.fromconfig(repo)
         level = None
         if comp == b'zlib':
             level = repo.ui.configint(b'storage', b'revlog.zlib.level')
         elif comp == b'zstd':
             level = repo.ui.configint(b'storage', b'revlog.zstd.level')
         if level is None:
             return b'default'
         return bytes(level)
 
 
 def find_format_upgrades(repo):
     """returns a list of format upgrades which can be perform on the repo"""
     upgrades = []
 
     # We could detect lack of revlogv1 and store here, but they were added
     # in 0.9.2 and we don't support upgrading repos without these
     # requirements, so let's not bother.
 
     for fv in allformatvariant:
         if not fv.fromrepo(repo):
             upgrades.append(fv)
 
     return upgrades
 
 
 def find_format_downgrades(repo):
     """returns a list of format downgrades which will be performed on the repo
     because of disabled config option for them"""
 
     downgrades = []
 
     for fv in allformatvariant:
         if fv.name == b'compression':
             # If there is a compression change between repository
             # and config, destination repository compression will change
             # and current compression will be removed.
             if fv.fromrepo(repo) != fv.fromconfig(repo):
                 downgrades.append(fv)
             continue
         # format variant exist in repo but does not exist in new repository
         # config
         if fv.fromrepo(repo) and not fv.fromconfig(repo):
             downgrades.append(fv)
 
     return downgrades
 
 
 ALL_OPTIMISATIONS = []
 
 
 def register_optimization(obj):
     ALL_OPTIMISATIONS.append(obj)
     return obj
 
 
 class optimization(improvement):
     """an improvement subclass dedicated to optimizations"""
 
     type = OPTIMISATION
 
 
 @register_optimization
 class redeltaparents(optimization):
     name = b're-delta-parent'
 
     type = OPTIMISATION
 
     description = _(
         b'deltas within internal storage will be recalculated to '
         b'choose an optimal base revision where this was not '
         b'already done; the size of the repository may shrink and '
         b'various operations may become faster; the first time '
         b'this optimization is performed could slow down upgrade '
         b'execution considerably; subsequent invocations should '
         b'not run noticeably slower'
     )
 
     upgrademessage = _(
         b'deltas within internal storage will choose a new '
         b'base revision if needed'
     )
 
 
 @register_optimization
 class redeltamultibase(optimization):
     name = b're-delta-multibase'
 
     type = OPTIMISATION
 
     description = _(
         b'deltas within internal storage will be recalculated '
         b'against multiple base revision and the smallest '
         b'difference will be used; the size of the repository may '
         b'shrink significantly when there are many merges; this '
         b'optimization will slow down execution in proportion to '
         b'the number of merges in the repository and the amount '
         b'of files in the repository; this slow down should not '
         b'be significant unless there are tens of thousands of '
         b'files and thousands of merges'
     )
 
     upgrademessage = _(
         b'deltas within internal storage will choose an '
         b'optimal delta by computing deltas against multiple '
         b'parents; may slow down execution time '
         b'significantly'
     )
 
 
 @register_optimization
 class redeltaall(optimization):
     name = b're-delta-all'
 
     type = OPTIMISATION
 
     description = _(
         b'deltas within internal storage will always be '
         b'recalculated without reusing prior deltas; this will '
         b'likely make execution run several times slower; this '
         b'optimization is typically not needed'
     )
 
     upgrademessage = _(
         b'deltas within internal storage will be fully '
         b'recomputed; this will likely drastically slow down '
         b'execution time'
     )
 
 
 @register_optimization
 class redeltafulladd(optimization):
     name = b're-delta-fulladd'
 
     type = OPTIMISATION
 
     description = _(
         b'every revision will be re-added as if it was new '
         b'content. It will go through the full storage '
         b'mechanism giving extensions a chance to process it '
         b'(eg. lfs). This is similar to "re-delta-all" but even '
         b'slower since more logic is involved.'
     )
 
     upgrademessage = _(
         b'each revision will be added as new content to the '
         b'internal storage; this will likely drastically slow '
         b'down execution time, but some extensions might need '
         b'it'
     )
 
 
 def findoptimizations(repo):
     """Determine optimisation that could be used during upgrade"""
     # These are unconditionally added. There is logic later that figures out
     # which ones to apply.
     return list(ALL_OPTIMISATIONS)
 
 
 def determine_upgrade_actions(
     repo, format_upgrades, optimizations, sourcereqs, destreqs
 ):
     """Determine upgrade actions that will be performed.
 
     Given a list of improvements as returned by ``find_format_upgrades`` and
     ``findoptimizations``, determine the list of upgrade actions that
     will be performed.
 
     The role of this function is to filter improvements if needed, apply
     recommended optimizations from the improvements list that make sense,
     etc.
 
     Returns a list of action names.
     """
     newactions = []
 
     for d in format_upgrades:
         if util.safehasattr(d, '_requirement'):
             name = d._requirement
         else:
             name = None
 
         # If the action is a requirement that doesn't show up in the
         # destination requirements, prune the action.
         if name is not None and name not in destreqs:
             continue
 
         newactions.append(d)
 
     newactions.extend(
         o
         for o in sorted(optimizations, key=(lambda x: x.name))
         if o not in newactions
     )
 
     # FUTURE consider adding some optimizations here for certain transitions.
     # e.g. adding generaldelta could schedule parent redeltas.
 
     return newactions
 
 
 class BaseOperation:
     """base class that contains the minimum for an upgrade to work
 
     (this might need to be extended as the usage for subclass alternative to
     UpgradeOperation extends)
     """
 
     def __init__(
         self,
         new_requirements,
         backup_store,
     ):
         self.new_requirements = new_requirements
         # should this operation create a backup of the store
         self.backup_store = backup_store
 
 
 class UpgradeOperation(BaseOperation):
     """represent the work to be done during an upgrade"""
 
     def __init__(
         self,
         ui,
         new_requirements,
         current_requirements,
         upgrade_actions,
         removed_actions,
         revlogs_to_process,
         backup_store,
     ):
         super().__init__(
             new_requirements,
             backup_store,
         )
         self.ui = ui
         self.current_requirements = current_requirements
         # list of upgrade actions the operation will perform
         self.upgrade_actions = upgrade_actions
         self.removed_actions = removed_actions
         self.revlogs_to_process = revlogs_to_process
         # requirements which will be added by the operation
         self._added_requirements = (
             self.new_requirements - self.current_requirements
         )
         # requirements which will be removed by the operation
         self._removed_requirements = (
             self.current_requirements - self.new_requirements
         )
         # requirements which will be preserved by the operation
         self._preserved_requirements = (
             self.current_requirements & self.new_requirements
         )
         # optimizations which are not used and it's recommended that they
         # should use them
         all_optimizations = findoptimizations(None)
         self.unused_optimizations = [
             i for i in all_optimizations if i not in self.upgrade_actions
         ]
 
         # delta reuse mode of this upgrade operation
         upgrade_actions_names = self.upgrade_actions_names
         self.delta_reuse_mode = revlog.revlog.DELTAREUSEALWAYS
         if b're-delta-all' in upgrade_actions_names:
             self.delta_reuse_mode = revlog.revlog.DELTAREUSENEVER
         elif b're-delta-parent' in upgrade_actions_names:
             self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS
         elif b're-delta-multibase' in upgrade_actions_names:
             self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS
         elif b're-delta-fulladd' in upgrade_actions_names:
             self.delta_reuse_mode = revlog.revlog.DELTAREUSEFULLADD
 
         # should this operation force re-delta of both parents
         self.force_re_delta_both_parents = (
             b're-delta-multibase' in upgrade_actions_names
         )
 
     @property
     def upgrade_actions_names(self):
         return set([a.name for a in self.upgrade_actions])
 
     @property
     def requirements_only(self):
         # does the operation only touches repository requirement
         return (
             self.touches_requirements
             and not self.touches_filelogs
             and not self.touches_manifests
             and not self.touches_changelog
             and not self.touches_dirstate
         )
 
     @property
     def touches_filelogs(self):
         for a in self.upgrade_actions:
             # in optimisations, we re-process the revlogs again
             if a.type == OPTIMISATION:
                 return True
             elif a.touches_filelogs:
                 return True
         for a in self.removed_actions:
             if a.touches_filelogs:
                 return True
         return False
 
     @property
     def touches_manifests(self):
         for a in self.upgrade_actions:
             # in optimisations, we re-process the revlogs again
             if a.type == OPTIMISATION:
                 return True
             elif a.touches_manifests:
                 return True
         for a in self.removed_actions:
             if a.touches_manifests:
                 return True
         return False
 
     @property
     def touches_changelog(self):
         for a in self.upgrade_actions:
             # in optimisations, we re-process the revlogs again
             if a.type == OPTIMISATION:
                 return True
             elif a.touches_changelog:
                 return True
         for a in self.removed_actions:
             if a.touches_changelog:
                 return True
         return False
 
     @property
     def touches_requirements(self):
         for a in self.upgrade_actions:
             # optimisations are used to re-process revlogs and does not result
             # in a requirement being added or removed
             if a.type == OPTIMISATION:
                 pass
             elif a.touches_requirements:
                 return True
         for a in self.removed_actions:
             if a.touches_requirements:
                 return True
 
     @property
     def touches_dirstate(self):
         for a in self.upgrade_actions:
             # revlog optimisations do not affect the dirstate
             if a.type == OPTIMISATION:
                 pass
             elif a.touches_dirstate:
                 return True
         for a in self.removed_actions:
             if a.touches_dirstate:
                 return True
 
         return False
 
     def _write_labeled(self, l, label: bytes):
         """
         Utility function to aid writing of a list under one label
         """
         first = True
         for r in sorted(l):
             if not first:
                 self.ui.write(b', ')
             self.ui.write(r, label=label)
             first = False
 
     def print_requirements(self):
         self.ui.write(_(b'requirements\n'))
         self.ui.write(_(b'   preserved: '))
         self._write_labeled(
             self._preserved_requirements, b"upgrade-repo.requirement.preserved"
         )
         self.ui.write((b'\n'))
         if self._removed_requirements:
             self.ui.write(_(b'   removed: '))
             self._write_labeled(
                 self._removed_requirements, b"upgrade-repo.requirement.removed"
             )
             self.ui.write((b'\n'))
         if self._added_requirements:
             self.ui.write(_(b'   added: '))
             self._write_labeled(
                 self._added_requirements, b"upgrade-repo.requirement.added"
             )
             self.ui.write((b'\n'))
         self.ui.write(b'\n')
 
     def print_optimisations(self):
         optimisations = [
             a for a in self.upgrade_actions if a.type == OPTIMISATION
         ]
         optimisations.sort(key=lambda a: a.name)
         if optimisations:
             self.ui.write(_(b'optimisations: '))
             self._write_labeled(
                 [a.name for a in optimisations],
                 b"upgrade-repo.optimisation.performed",
             )
             self.ui.write(b'\n\n')
 
     def print_upgrade_actions(self):
         for a in self.upgrade_actions:
             self.ui.status(b'%s\n   %s\n\n' % (a.name, a.upgrademessage))
 
     def print_affected_revlogs(self):
         if not self.revlogs_to_process:
             self.ui.write((b'no revlogs to process\n'))
         else:
             self.ui.write((b'processed revlogs:\n'))
             for r in sorted(self.revlogs_to_process):
                 self.ui.write((b'  - %s\n' % r))
         self.ui.write((b'\n'))
 
     def print_unused_optimizations(self):
         for i in self.unused_optimizations:
             self.ui.status(_(b'%s\n   %s\n\n') % (i.name, i.description))
 
     def has_upgrade_action(self, name):
         """Check whether the upgrade operation will perform this action"""
         return name in self._upgrade_actions_names
 
     def print_post_op_messages(self):
         """print post upgrade operation warning messages"""
         for a in self.upgrade_actions:
             if a.postupgrademessage is not None:
                 self.ui.warn(b'%s\n' % a.postupgrademessage)
         for a in self.removed_actions:
             if a.postdowngrademessage is not None:
                 self.ui.warn(b'%s\n' % a.postdowngrademessage)
 
 
 ###  Code checking if a repository can got through the upgrade process at all. #
 
 
 def requiredsourcerequirements(repo):
     """Obtain requirements required to be present to upgrade a repo.
 
     An upgrade will not be allowed if the repository doesn't have the
     requirements returned by this function.
     """
     return {
         # Introduced in Mercurial 0.9.2.
         requirements.STORE_REQUIREMENT,
     }
 
 
 def blocksourcerequirements(repo):
     """Obtain requirements that will prevent an upgrade from occurring.
 
     An upgrade cannot be performed if the source repository contains a
     requirements in the returned set.
     """
     return {
         # This was a precursor to generaldelta and was never enabled by default.
         # It should (hopefully) not exist in the wild.
         b'parentdelta',
     }
 
 
 def check_revlog_version(reqs):
     """Check that the requirements contain at least one Revlog version"""
     all_revlogs = {
         requirements.REVLOGV1_REQUIREMENT,
         requirements.REVLOGV2_REQUIREMENT,
     }
     if not all_revlogs.intersection(reqs):
         msg = _(b'cannot upgrade repository; missing a revlog version')
         raise error.Abort(msg)
 
 
 def check_source_requirements(repo):
     """Ensure that no existing requirements prevent the repository upgrade"""
 
     check_revlog_version(repo.requirements)
     required = requiredsourcerequirements(repo)
     missingreqs = required - repo.requirements
     if missingreqs:
         msg = _(b'cannot upgrade repository; requirement missing: %s')
         missingreqs = b', '.join(sorted(missingreqs))
         raise error.Abort(msg % missingreqs)
 
     blocking = blocksourcerequirements(repo)
     blockingreqs = blocking & repo.requirements
     if blockingreqs:
         m = _(b'cannot upgrade repository; unsupported source requirement: %s')
         blockingreqs = b', '.join(sorted(blockingreqs))
         raise error.Abort(m % blockingreqs)
     # Upgrade should operate on the actual store, not the shared link.
 
     bad_share = (
         requirements.SHARED_REQUIREMENT in repo.requirements
         and requirements.SHARESAFE_REQUIREMENT not in repo.requirements
     )
     if bad_share:
         m = _(b'cannot upgrade repository; share repository without share-safe')
         h = _(b'check :hg:`help config.format.use-share-safe`')
         raise error.Abort(m, hint=h)
 
 
 ### Verify the validity of the planned requirement changes ####################
 
 
 def supportremovedrequirements(repo):
     """Obtain requirements that can be removed during an upgrade.
 
     If an upgrade were to create a repository that dropped a requirement,
     the dropped requirement must appear in the returned set for the upgrade
     to be allowed.
     """
     supported = {
         requirements.SPARSEREVLOG_REQUIREMENT,
         requirements.COPIESSDC_REQUIREMENT,
         requirements.NODEMAP_REQUIREMENT,
         requirements.SHARESAFE_REQUIREMENT,
         requirements.REVLOGV2_REQUIREMENT,
         requirements.CHANGELOGV2_REQUIREMENT,
         requirements.REVLOGV1_REQUIREMENT,
         requirements.DIRSTATE_TRACKED_HINT_V1,
         requirements.DIRSTATE_V2_REQUIREMENT,
     }
     for name in compression.compengines:
         engine = compression.compengines[name]
         if engine.available() and engine.revlogheader():
             supported.add(b'exp-compression-%s' % name)
             if engine.name() == b'zstd':
                 supported.add(b'revlog-compression-zstd')
     return supported
 
 
 def supporteddestrequirements(repo):
     """Obtain requirements that upgrade supports in the destination.
 
     If the result of the upgrade would have requirements not in this set,
     the upgrade is disallowed.
 
     Extensions should monkeypatch this to add their custom requirements.
     """
     supported = {
         requirements.CHANGELOGV2_REQUIREMENT,
         requirements.COPIESSDC_REQUIREMENT,
         requirements.DIRSTATE_TRACKED_HINT_V1,
         requirements.DIRSTATE_V2_REQUIREMENT,
         requirements.DOTENCODE_REQUIREMENT,
         requirements.FNCACHE_REQUIREMENT,
         requirements.GENERALDELTA_REQUIREMENT,
         requirements.NODEMAP_REQUIREMENT,
         requirements.REVLOGV1_REQUIREMENT,  # allowed in case of downgrade
         requirements.REVLOGV2_REQUIREMENT,
         requirements.SHARED_REQUIREMENT,
         requirements.SHARESAFE_REQUIREMENT,
         requirements.SPARSEREVLOG_REQUIREMENT,
         requirements.STORE_REQUIREMENT,
         requirements.TREEMANIFEST_REQUIREMENT,
         requirements.NARROW_REQUIREMENT,
     }
     for name in compression.compengines:
         engine = compression.compengines[name]
         if engine.available() and engine.revlogheader():
             supported.add(b'exp-compression-%s' % name)
             if engine.name() == b'zstd':
                 supported.add(b'revlog-compression-zstd')
     return supported
 
 
 def allowednewrequirements(repo):
     """Obtain requirements that can be added to a repository during upgrade.
 
     This is used to disallow proposed requirements from being added when
     they weren't present before.
 
     We use a list of allowed requirement additions instead of a list of known
     bad additions because the whitelist approach is safer and will prevent
     future, unknown requirements from accidentally being added.
     """
     supported = {
         requirements.DOTENCODE_REQUIREMENT,
         requirements.FNCACHE_REQUIREMENT,
         requirements.GENERALDELTA_REQUIREMENT,
         requirements.SPARSEREVLOG_REQUIREMENT,
         requirements.COPIESSDC_REQUIREMENT,
         requirements.NODEMAP_REQUIREMENT,
         requirements.SHARESAFE_REQUIREMENT,
         requirements.REVLOGV1_REQUIREMENT,
         requirements.REVLOGV2_REQUIREMENT,
         requirements.CHANGELOGV2_REQUIREMENT,
         requirements.DIRSTATE_TRACKED_HINT_V1,
         requirements.DIRSTATE_V2_REQUIREMENT,
     }
     for name in compression.compengines:
         engine = compression.compengines[name]
         if engine.available() and engine.revlogheader():
             supported.add(b'exp-compression-%s' % name)
             if engine.name() == b'zstd':
                 supported.add(b'revlog-compression-zstd')
     return supported
 
 
 def check_requirements_changes(repo, new_reqs):
     old_reqs = repo.requirements
     check_revlog_version(repo.requirements)
     support_removal = supportremovedrequirements(repo)
     no_remove_reqs = old_reqs - new_reqs - support_removal
     if no_remove_reqs:
         msg = _(b'cannot upgrade repository; requirement would be removed: %s')
         no_remove_reqs = b', '.join(sorted(no_remove_reqs))
         raise error.Abort(msg % no_remove_reqs)
 
     support_addition = allowednewrequirements(repo)
     no_add_reqs = new_reqs - old_reqs - support_addition
     if no_add_reqs:
         m = _(b'cannot upgrade repository; do not support adding requirement: ')
         no_add_reqs = b', '.join(sorted(no_add_reqs))
         raise error.Abort(m + no_add_reqs)
 
     supported = supporteddestrequirements(repo)
     unsupported_reqs = new_reqs - supported
     if unsupported_reqs:
         msg = _(
             b'cannot upgrade repository; do not support destination '
             b'requirement: %s'
         )
         unsupported_reqs = b', '.join(sorted(unsupported_reqs))
         raise error.Abort(msg % unsupported_reqs)

tests/test-share-safe.t

@@ -1,655 +1,655 b''
 setup
 
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > share =
   > [format]
   > use-share-safe = True
   > [storage]
   > revlog.persistent-nodemap.slow-path=allow
   > # enforce zlib to ensure we can upgrade to zstd later
   > [format]
   > revlog-compression=zlib
   > # we want to be able to enable it later
   > use-persistent-nodemap=no
   > EOF
 
 prepare source repo
 
   $ hg init source
   $ cd source
   $ cat .hg/requires
   dirstate-v2 (dirstate-v2 !)
   share-safe
   $ cat .hg/store/requires
   dotencode
   fncache
   generaldelta
   revlogv1
   sparserevlog
   store
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   share-safe
   sparserevlog
   store
 
   $ echo a > a
   $ hg ci -Aqm "added a"
   $ echo b > b
   $ hg ci -Aqm "added b"
 
   $ HGEDITOR=cat hg config --shared
   abort: repository is not shared; can't use --shared
   [10]
   $ cd ..
 
 Create a shared repo and check the requirements are shared and read correctly
   $ hg share source shared1
   updating working directory
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd shared1
   $ cat .hg/requires
   dirstate-v2 (dirstate-v2 !)
   share-safe
   shared
 
   $ hg debugrequirements -R ../source
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   share-safe
   sparserevlog
   store
 
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   share-safe
   shared
   sparserevlog
   store
 
   $ echo c > c
   $ hg ci -Aqm "added c"
 
 Check that config of the source repository is also loaded
 
   $ hg showconfig ui.curses
   [1]
 
   $ echo "[ui]" >> ../source/.hg/hgrc
   $ echo "curses=true" >> ../source/.hg/hgrc
 
   $ hg showconfig ui.curses
   true
 
 Test that extensions of source repository are also loaded
 
   $ hg debugextensions
   share
   $ hg extdiff -p echo
   hg: unknown command 'extdiff'
   'extdiff' is provided by the following extension:
   
       extdiff       command to allow external programs to compare revisions
   
   (use 'hg help extensions' for information on enabling extensions)
   [10]
 
   $ echo "[extensions]" >> ../source/.hg/hgrc
   $ echo "extdiff=" >> ../source/.hg/hgrc
 
   $ hg debugextensions -R ../source
   extdiff
   share
   $ hg extdiff -R ../source -p echo
 
 BROKEN: the command below will not work if config of shared source is not loaded
 on dispatch but debugextensions says that extension
 is loaded
   $ hg debugextensions
   extdiff
   share
 
   $ hg extdiff -p echo
 
 However, local .hg/hgrc should override the config set by share source
 
   $ echo "[ui]" >> .hg/hgrc
   $ echo "curses=false" >> .hg/hgrc
 
   $ hg showconfig ui.curses
   false
 
   $ HGEDITOR=cat hg config --shared
   [ui]
   curses=true
   [extensions]
   extdiff=
 
   $ HGEDITOR=cat hg config --local
   [ui]
   curses=false
 
 Testing that hooks set in source repository also runs in shared repo
 
   $ cd ../source
   $ cat <<EOF >> .hg/hgrc
   > [extensions]
   > hooklib=
   > [hooks]
   > pretxnchangegroup.reject_merge_commits = \
   >   python:hgext.hooklib.reject_merge_commits.hook
   > EOF
 
   $ cd ..
   $ hg clone source cloned
   updating to branch default
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd cloned
   $ hg up 0
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved
   $ echo bar > bar
   $ hg ci -Aqm "added bar"
   $ hg merge
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg ci -m "merge commit"
 
   $ hg push ../source
   pushing to ../source
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   error: pretxnchangegroup.reject_merge_commits hook failed: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   transaction abort!
   rollback completed
   abort: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   [255]
 
   $ hg push ../shared1
   pushing to ../shared1
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   error: pretxnchangegroup.reject_merge_commits hook failed: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   transaction abort!
   rollback completed
   abort: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   [255]
 
 Test that if share source config is untrusted, we dont read it
 
   $ cd ../shared1
 
   $ cat << EOF > $TESTTMP/untrusted.py
   > from mercurial import scmutil, util
   > def uisetup(ui):
   >     class untrustedui(ui.__class__):
   >         def _trusted(self, fp, f):
   >             if util.normpath(fp.name).endswith(b'source/.hg/hgrc'):
   >                 return False
   >             return super(untrustedui, self)._trusted(fp, f)
   >     ui.__class__ = untrustedui
   > EOF
 
   $ hg showconfig hooks
   hooks.pretxnchangegroup.reject_merge_commits=python:hgext.hooklib.reject_merge_commits.hook
 
   $ hg showconfig hooks --config extensions.untrusted=$TESTTMP/untrusted.py
   [1]
 
 Update the source repository format and check that shared repo works
 
   $ cd ../source
 
 Disable zstd related tests because its not present on pure version
 #if zstd
   $ echo "[format]" >> .hg/hgrc
   $ echo "revlog-compression=zstd" >> .hg/hgrc
 
   $ hg debugupgraderepo --run -q
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (dirstate-v2 !)
      added: revlog-compression-zstd
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ hg log -r .
   changeset:   1:5f6d8a4bf34a
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     added b
   
 #endif
   $ echo "[format]" >> .hg/hgrc
   $ echo "use-persistent-nodemap=True" >> .hg/hgrc
 
   $ hg debugupgraderepo --run -q -R ../shared1
   abort: cannot use these actions on a share repository: persistent-nodemap
   (upgrade the main repository directly)
   [255]
 
   $ hg debugupgraderepo --run -q
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (no-zstd no-dirstate-v2 !)
      preserved: dotencode, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, share-safe, sparserevlog, store (no-zstd dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlog-compression-zstd, revlogv1, share-safe, sparserevlog, store (zstd dirstate-v2 !)
      added: persistent-nodemap
   
   processed revlogs:
     - all-filelogs
     - changelog
     - manifest
   
   $ hg log -r .
   changeset:   1:5f6d8a4bf34a
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     added b
   
 
 Shared one should work
   $ cd ../shared1
   $ hg log -r .
   changeset:   2:155349b645be
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     added c
   
 
 Testing that nonsharedrc is loaded for source and not shared
 
   $ cd ../source
   $ touch .hg/hgrc-not-shared
   $ echo "[ui]" >> .hg/hgrc-not-shared
   $ echo "traceback=true" >> .hg/hgrc-not-shared
 
   $ hg showconfig ui.traceback
   true
 
   $ HGEDITOR=cat hg config --non-shared
   [ui]
   traceback=true
 
   $ cd ../shared1
   $ hg showconfig ui.traceback
   [1]
 
 Unsharing works
 
   $ hg unshare
 
 Test that source config is added to the shared one after unshare, and the config
 of current repo is still respected over the config which came from source config
   $ cd ../cloned
   $ hg push ../shared1
   pushing to ../shared1
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   error: pretxnchangegroup.reject_merge_commits hook failed: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   transaction abort!
   rollback completed
   abort: bcde3522682d rejected as merge on the same branch. Please consider rebase.
   [255]
   $ hg showconfig ui.curses -R ../shared1
   false
 
   $ cd ../
 
 Test that upgrading using debugupgraderepo works
 =================================================
 
   $ hg init non-share-safe --config format.use-share-safe=false
   $ cd non-share-safe
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   sparserevlog
   store
   $ echo foo > foo
   $ hg ci -Aqm 'added foo'
   $ echo bar > bar
   $ hg ci -Aqm 'added bar'
 
 Create a share before upgrading
 
   $ cd ..
   $ hg share non-share-safe nss-share
   updating working directory
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg debugrequirements -R nss-share
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   shared
   sparserevlog
   store
   $ cd non-share-safe
 
 Upgrade
 
   $ hg debugupgraderepo -q
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, sparserevlog, store (dirstate-v2 !)
      added: share-safe
   
   no revlogs to process
   
   $ hg debugupgraderepo --run
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, sparserevlog, store (dirstate-v2 !)
      added: share-safe
   
   share-safe
      Upgrades a repository to share-safe format so that future shares of this repository share its requirements and configs.
   
   no revlogs to process
   
   beginning upgrade...
   repository locked and read-only
   creating temporary repository to stage upgraded data: $TESTTMP/non-share-safe/.hg/upgrade.* (glob)
   (it is safe to interrupt this process any time before data migration completes)
   upgrading repository requirements
   removing temporary repository $TESTTMP/non-share-safe/.hg/upgrade.* (glob)
   repository upgraded to share safe mode, existing shares will still work in old non-safe mode. Re-share existing shares to use them in safe mode New shares will be created in safe mode.
 
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   share-safe
   sparserevlog
   store
 
   $ cat .hg/requires
   dirstate-v2 (dirstate-v2 !)
   share-safe
 
   $ cat .hg/store/requires
   dotencode
   fncache
   generaldelta
   revlogv1
   sparserevlog
   store
 
   $ hg log -GT "{node}: {desc}\n"
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
 Make sure existing shares dont work with default config
 
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share
   abort: version mismatch: source uses share-safe functionality while the current share does not
   (see `hg help config.format.use-share-safe` for more information)
   [255]
 
 
 Create a safe share from upgrade one
 
   $ cd ..
   $ hg share non-share-safe ss-share
   updating working directory
   2 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd ss-share
   $ hg log -GT "{node}: {desc}\n"
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
   $ cd ../non-share-safe
 
 Test that downgrading works too
 
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > share =
   > [format]
   > use-share-safe = False
   > EOF
 
   $ hg debugupgraderepo -q
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, sparserevlog, store (dirstate-v2 !)
      removed: share-safe
   
   no revlogs to process
   
   $ hg debugupgraderepo --run
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, sparserevlog, store (dirstate-v2 !)
      removed: share-safe
   
   no revlogs to process
   
   beginning upgrade...
   repository locked and read-only
   creating temporary repository to stage upgraded data: $TESTTMP/non-share-safe/.hg/upgrade.* (glob)
   (it is safe to interrupt this process any time before data migration completes)
   upgrading repository requirements
   removing temporary repository $TESTTMP/non-share-safe/.hg/upgrade.* (glob)
-  repository downgraded to not use share safe mode, existing shares will not work and needs to be reshared.
+  repository downgraded to not use share safe mode, existing shares will not work and need to be reshared.
 
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   sparserevlog
   store
 
   $ cat .hg/requires
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   sparserevlog
   store
 
   $ test -f .hg/store/requires
   [1]
 
   $ hg log -GT "{node}: {desc}\n"
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
 Make sure existing shares still works
 
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
   $ hg log -GT "{node}: {desc}\n" -R ../ss-share
   abort: share source does not support share-safe requirement
   (see `hg help config.format.use-share-safe` for more information)
   [255]
 
 Testing automatic downgrade of shares when config is set
 
   $ touch ../ss-share/.hg/wlock
   $ hg log -GT "{node}: {desc}\n" -R ../ss-share --config share.safe-mismatch.source-not-safe=downgrade-abort
   abort: failed to downgrade share, got error: Lock held
   (see `hg help config.format.use-share-safe` for more information)
   [255]
   $ rm ../ss-share/.hg/wlock
 
   $ cp -R ../ss-share ../ss-share-bck
   $ hg log -GT "{node}: {desc}\n" -R ../ss-share --config share.safe-mismatch.source-not-safe=downgrade-abort
   repository downgraded to not use share-safe mode
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
   $ rm -rf ../ss-share
   $ mv ../ss-share-bck ../ss-share
 
   $ hg log -GT "{node}: {desc}\n" -R ../ss-share --config share.safe-mismatch.source-not-safe=downgrade-abort --config share.safe-mismatch.source-not-safe:verbose-upgrade=no
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
   $ hg log -GT "{node}: {desc}\n" -R ../ss-share
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
 
 Testing automatic upgrade of shares when config is set
 
   $ hg debugupgraderepo -q --run --config format.use-share-safe=True
   upgrade will perform the following actions:
   
   requirements
      preserved: dotencode, fncache, generaldelta, revlogv1, sparserevlog, store (no-dirstate-v2 !)
      preserved: dotencode, use-dirstate-v2, fncache, generaldelta, revlogv1, sparserevlog, store (dirstate-v2 !)
      added: share-safe
   
   no revlogs to process
   
   repository upgraded to share safe mode, existing shares will still work in old non-safe mode. Re-share existing shares to use them in safe mode New shares will be created in safe mode.
   $ hg debugrequirements
   dotencode
   dirstate-v2 (dirstate-v2 !)
   fncache
   generaldelta
   revlogv1
   share-safe
   sparserevlog
   store
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share
   abort: version mismatch: source uses share-safe functionality while the current share does not
   (see `hg help config.format.use-share-safe` for more information)
   [255]
 
 Check that if lock is taken, upgrade fails but read operation are successful
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgra
   abort: share-safe mismatch with source.
   Unrecognized value 'upgra' of `share.safe-mismatch.source-safe` set.
   (see `hg help config.format.use-share-safe` for more information)
   [255]
   $ touch ../nss-share/.hg/wlock
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgrade-allow
   failed to upgrade share, got error: Lock held
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgrade-allow --config share.safe-mismatch.source-safe.warn=False
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgrade-abort
   abort: failed to upgrade share, got error: Lock held
   (see `hg help config.format.use-share-safe` for more information)
   [255]
 
   $ rm ../nss-share/.hg/wlock
   $ cp -R ../nss-share ../nss-share-bck
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgrade-abort
   repository upgraded to use share-safe mode
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
   $ rm -rf ../nss-share
   $ mv ../nss-share-bck ../nss-share
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share --config share.safe-mismatch.source-safe=upgrade-abort --config share.safe-mismatch.source-safe:verbose-upgrade=no
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
 Test that unshare works
 
   $ hg unshare -R ../nss-share
   $ hg log -GT "{node}: {desc}\n" -R ../nss-share
   @  f63db81e6dde1d9c78814167f77fb1fb49283f4f: added bar
   |
   o  f3ba8b99bb6f897c87bbc1c07b75c6ddf43a4f77: added foo
   
 
 Test automatique upgrade/downgrade of main-repository
 ------------------------------------------------------
 
 create an initial repository
 
   $ hg init auto-upgrade \
   > --config format.use-share-safe=no
   $ hg debugbuilddag -R auto-upgrade --new-file .+5
   $ hg -R auto-upgrade update
   6 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg debugformat -R auto-upgrade | grep share-safe
   share-safe:          no
 
 upgrade it to share-safe automatically
 
   $ hg status -R auto-upgrade \
   >     --config format.use-share-safe.automatic-upgrade-of-mismatching-repositories=yes \
   >     --config format.use-share-safe=yes
   automatically upgrading repository to the `share-safe` feature
   (see `hg help config.format.use-share-safe` for details)
   $ hg debugformat -R auto-upgrade | grep share-safe
   share-safe:         yes
 
 downgrade it from share-safe automatically
 
   $ hg status -R auto-upgrade \
   >     --config format.use-share-safe.automatic-upgrade-of-mismatching-repositories=yes \
   >     --config format.use-share-safe=no
   automatically downgrading repository from the `share-safe` feature
   (see `hg help config.format.use-share-safe` for details)
   $ hg debugformat -R auto-upgrade | grep share-safe
   share-safe:          no