changegroup: add v4 changegroup for revlog v2 exchange

This change only adds the required infrastructure for the new changegroup format
and does not do any actual exchange. This will be done in the next patches.

Differential Revision: https://phab.mercurial-scm.org/D10026

hgext/remotefilelog/remotefilelog.py

@@ -1,513 +1,516 b''
 # remotefilelog.py - filelog implementation where filelog history is stored
 #                    remotely
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import collections
 import os
 
 from mercurial.node import (
     bin,
     nullid,
     wdirfilenodeids,
     wdirid,
 )
 from mercurial.i18n import _
 from mercurial import (
     ancestor,
     error,
     mdiff,
     pycompat,
     revlog,
     util,
 )
 from mercurial.utils import storageutil
 from mercurial.revlogutils import flagutil
 
 from . import (
     constants,
     fileserverclient,
     shallowutil,
 )
 
 
 class remotefilelognodemap(object):
     def __init__(self, filename, store):
         self._filename = filename
         self._store = store
 
     def __contains__(self, node):
         missing = self._store.getmissing([(self._filename, node)])
         return not bool(missing)
 
     def __get__(self, node):
         if node not in self:
             raise KeyError(node)
         return node
 
 
 class remotefilelog(object):
 
     _generaldelta = True
     _flagserrorclass = error.RevlogError
 
     def __init__(self, opener, path, repo):
         self.opener = opener
         self.filename = path
         self.repo = repo
         self.nodemap = remotefilelognodemap(self.filename, repo.contentstore)
 
         self.version = 1
 
         self._flagprocessors = dict(flagutil.flagprocessors)
 
     def read(self, node):
         """returns the file contents at this node"""
         t = self.revision(node)
         if not t.startswith(b'\1\n'):
             return t
         s = t.index(b'\1\n', 2)
         return t[s + 2 :]
 
     def add(self, text, meta, transaction, linknode, p1=None, p2=None):
         # hash with the metadata, like in vanilla filelogs
         hashtext = shallowutil.createrevlogtext(
             text, meta.get(b'copy'), meta.get(b'copyrev')
         )
         node = storageutil.hashrevisionsha1(hashtext, p1, p2)
         return self.addrevision(
             hashtext, transaction, linknode, p1, p2, node=node
         )
 
     def _createfileblob(self, text, meta, flags, p1, p2, node, linknode):
         # text passed to "_createfileblob" does not include filelog metadata
         header = shallowutil.buildfileblobheader(len(text), flags)
         data = b"%s\0%s" % (header, text)
 
         realp1 = p1
         copyfrom = b""
         if meta and b'copy' in meta:
             copyfrom = meta[b'copy']
             realp1 = bin(meta[b'copyrev'])
 
         data += b"%s%s%s%s%s\0" % (node, realp1, p2, linknode, copyfrom)
 
         visited = set()
 
         pancestors = {}
         queue = []
         if realp1 != nullid:
             p1flog = self
             if copyfrom:
                 p1flog = remotefilelog(self.opener, copyfrom, self.repo)
 
             pancestors.update(p1flog.ancestormap(realp1))
             queue.append(realp1)
             visited.add(realp1)
         if p2 != nullid:
             pancestors.update(self.ancestormap(p2))
             queue.append(p2)
             visited.add(p2)
 
         ancestortext = b""
 
         # add the ancestors in topological order
         while queue:
             c = queue.pop(0)
             pa1, pa2, ancestorlinknode, pacopyfrom = pancestors[c]
 
             pacopyfrom = pacopyfrom or b''
             ancestortext += b"%s%s%s%s%s\0" % (
                 c,
                 pa1,
                 pa2,
                 ancestorlinknode,
                 pacopyfrom,
             )
 
             if pa1 != nullid and pa1 not in visited:
                 queue.append(pa1)
                 visited.add(pa1)
             if pa2 != nullid and pa2 not in visited:
                 queue.append(pa2)
                 visited.add(pa2)
 
         data += ancestortext
 
         return data
 
     def addrevision(
         self,
         text,
         transaction,
         linknode,
         p1,
         p2,
         cachedelta=None,
         node=None,
         flags=revlog.REVIDX_DEFAULT_FLAGS,
         sidedata=None,
     ):
         # text passed to "addrevision" includes hg filelog metadata header
         if node is None:
             node = storageutil.hashrevisionsha1(text, p1, p2)
 
         meta, metaoffset = storageutil.parsemeta(text)
         rawtext, validatehash = flagutil.processflagswrite(
             self,
             text,
             flags,
         )
         return self.addrawrevision(
             rawtext,
             transaction,
             linknode,
             p1,
             p2,
             node,
             flags,
             cachedelta,
             _metatuple=(meta, metaoffset),
         )
 
     def addrawrevision(
         self,
         rawtext,
         transaction,
         linknode,
         p1,
         p2,
         node,
         flags,
         cachedelta=None,
         _metatuple=None,
     ):
         if _metatuple:
             # _metatuple: used by "addrevision" internally by remotefilelog
             # meta was parsed confidently
             meta, metaoffset = _metatuple
         else:
             # not from self.addrevision, but something else (repo._filecommit)
             # calls addrawrevision directly. remotefilelog needs to get and
             # strip filelog metadata.
             # we don't have confidence about whether rawtext contains filelog
             # metadata or not (flag processor could replace it), so we just
             # parse it as best-effort.
             # in LFS (flags != 0)'s case, the best way is to call LFS code to
             # get the meta information, instead of storageutil.parsemeta.
             meta, metaoffset = storageutil.parsemeta(rawtext)
         if flags != 0:
             # when flags != 0, be conservative and do not mangle rawtext, since
             # a read flag processor expects the text not being mangled at all.
             metaoffset = 0
         if metaoffset:
             # remotefilelog fileblob stores copy metadata in its ancestortext,
             # not its main blob. so we need to remove filelog metadata
             # (containing copy information) from text.
             blobtext = rawtext[metaoffset:]
         else:
             blobtext = rawtext
         data = self._createfileblob(
             blobtext, meta, flags, p1, p2, node, linknode
         )
         self.repo.contentstore.addremotefilelognode(self.filename, node, data)
 
         return node
 
     def renamed(self, node):
         ancestors = self.repo.metadatastore.getancestors(self.filename, node)
         p1, p2, linknode, copyfrom = ancestors[node]
         if copyfrom:
             return (copyfrom, p1)
 
         return False
 
     def size(self, node):
         """return the size of a given revision"""
         return len(self.read(node))
 
     rawsize = size
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
 
         if node == nullid:
             return True
 
         nodetext = self.read(node)
         return nodetext != text
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     def __len__(self):
         if self.filename == b'.hgtags':
             # The length of .hgtags is used to fast path tag checking.
             # remotefilelog doesn't support .hgtags since the entire .hgtags
             # history is needed.  Use the excludepattern setting to make
             # .hgtags a normal filelog.
             return 0
 
         raise RuntimeError(b"len not supported")
 
     def heads(self):
         # Fake heads of the filelog to satisfy hgweb.
         return []
 
     def empty(self):
         return False
 
     def flags(self, node):
         if isinstance(node, int):
             raise error.ProgrammingError(
                 b'remotefilelog does not accept integer rev for flags'
             )
         store = self.repo.contentstore
         return store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)
 
     def parents(self, node):
         if node == nullid:
             return nullid, nullid
 
         ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
         p1, p2, linknode, copyfrom = ancestormap[node]
         if copyfrom:
             p1 = nullid
 
         return p1, p2
 
     def parentrevs(self, rev):
         # TODO(augie): this is a node and should be a rev, but for now
         # nothing in core seems to actually break.
         return self.parents(rev)
 
     def linknode(self, node):
         ancestormap = self.repo.metadatastore.getancestors(self.filename, node)
         p1, p2, linknode, copyfrom = ancestormap[node]
         return linknode
 
     def linkrev(self, node):
         return self.repo.unfiltered().changelog.rev(self.linknode(node))
 
     def emitrevisions(
         self,
         nodes,
         nodesorder=None,
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltaprevious=False,
         deltamode=None,
+        sidedata_helpers=None,
     ):
         # we don't use any of these parameters here
         del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
         del deltamode
         prevnode = None
         for node in nodes:
             p1, p2 = self.parents(node)
             if prevnode is None:
                 basenode = prevnode = p1
             if basenode == node:
                 basenode = nullid
             if basenode != nullid:
                 revision = None
                 delta = self.revdiff(basenode, node)
             else:
                 revision = self.rawdata(node)
                 delta = None
             yield revlog.revlogrevisiondelta(
                 node=node,
                 p1node=p1,
                 p2node=p2,
                 linknode=self.linknode(node),
                 basenode=basenode,
                 flags=self.flags(node),
                 baserevisionsize=None,
                 revision=revision,
                 delta=delta,
+                # Sidedata is not supported yet
+                sidedata=None,
             )
 
     def revdiff(self, node1, node2):
         return mdiff.textdiff(self.rawdata(node1), self.rawdata(node2))
 
     def lookup(self, node):
         if len(node) == 40:
             node = bin(node)
         if len(node) != 20:
             raise error.LookupError(
                 node, self.filename, _(b'invalid lookup input')
             )
 
         return node
 
     def rev(self, node):
         # This is a hack to make TortoiseHG work.
         return node
 
     def node(self, rev):
         # This is a hack.
         if isinstance(rev, int):
             raise error.ProgrammingError(
                 b'remotefilelog does not convert integer rev to node'
             )
         return rev
 
     def _processflags(self, text, flags, operation, raw=False):
         """deprecated entry point to access flag processors"""
         msg = b'_processflag(...) use the specialized variant'
         util.nouideprecwarn(msg, b'5.2', stacklevel=2)
         if raw:
             return text, flagutil.processflagsraw(self, text, flags)
         elif operation == b'read':
             return flagutil.processflagsread(self, text, flags)
         else:  # write operation
             return flagutil.processflagswrite(self, text, flags)
 
     def revision(self, node, raw=False):
         """returns the revlog contents at this node.
         this includes the meta data traditionally included in file revlogs.
         this is generally only used for bundling and communicating with vanilla
         hg clients.
         """
         if node == nullid:
             return b""
         if len(node) != 20:
             raise error.LookupError(
                 node, self.filename, _(b'invalid revision input')
             )
         if node == wdirid or node in wdirfilenodeids:
             raise error.WdirUnsupported
 
         store = self.repo.contentstore
         rawtext = store.get(self.filename, node)
         if raw:
             return rawtext
         flags = store.getmeta(self.filename, node).get(constants.METAKEYFLAG, 0)
         if flags == 0:
             return rawtext
         return flagutil.processflagsread(self, rawtext, flags)[0]
 
     def rawdata(self, node):
         return self.revision(node, raw=False)
 
     def _read(self, id):
         """reads the raw file blob from disk, cache, or server"""
         fileservice = self.repo.fileservice
         localcache = fileservice.localcache
         cachekey = fileserverclient.getcachekey(
             self.repo.name, self.filename, id
         )
         try:
             return localcache.read(cachekey)
         except KeyError:
             pass
 
         localkey = fileserverclient.getlocalkey(self.filename, id)
         localpath = os.path.join(self.localpath, localkey)
         try:
             return shallowutil.readfile(localpath)
         except IOError:
             pass
 
         fileservice.prefetch([(self.filename, id)])
         try:
             return localcache.read(cachekey)
         except KeyError:
             pass
 
         raise error.LookupError(id, self.filename, _(b'no node'))
 
     def ancestormap(self, node):
         return self.repo.metadatastore.getancestors(self.filename, node)
 
     def ancestor(self, a, b):
         if a == nullid or b == nullid:
             return nullid
 
         revmap, parentfunc = self._buildrevgraph(a, b)
         nodemap = {v: k for (k, v) in pycompat.iteritems(revmap)}
 
         ancs = ancestor.ancestors(parentfunc, revmap[a], revmap[b])
         if ancs:
             # choose a consistent winner when there's a tie
             return min(map(nodemap.__getitem__, ancs))
         return nullid
 
     def commonancestorsheads(self, a, b):
         """calculate all the heads of the common ancestors of nodes a and b"""
 
         if a == nullid or b == nullid:
             return nullid
 
         revmap, parentfunc = self._buildrevgraph(a, b)
         nodemap = {v: k for (k, v) in pycompat.iteritems(revmap)}
 
         ancs = ancestor.commonancestorsheads(parentfunc, revmap[a], revmap[b])
         return map(nodemap.__getitem__, ancs)
 
     def _buildrevgraph(self, a, b):
         """Builds a numeric revision graph for the given two nodes.
         Returns a node->rev map and a rev->[revs] parent function.
         """
         amap = self.ancestormap(a)
         bmap = self.ancestormap(b)
 
         # Union the two maps
         parentsmap = collections.defaultdict(list)
         allparents = set()
         for mapping in (amap, bmap):
             for node, pdata in pycompat.iteritems(mapping):
                 parents = parentsmap[node]
                 p1, p2, linknode, copyfrom = pdata
                 # Don't follow renames (copyfrom).
                 # remotefilectx.ancestor does that.
                 if p1 != nullid and not copyfrom:
                     parents.append(p1)
                     allparents.add(p1)
                 if p2 != nullid:
                     parents.append(p2)
                     allparents.add(p2)
 
         # Breadth first traversal to build linkrev graph
         parentrevs = collections.defaultdict(list)
         revmap = {}
         queue = collections.deque(
             ((None, n) for n in parentsmap if n not in allparents)
         )
         while queue:
             prevrev, current = queue.pop()
             if current in revmap:
                 if prevrev:
                     parentrevs[prevrev].append(revmap[current])
                 continue
 
             # Assign linkrevs in reverse order, so start at
             # len(parentsmap) and work backwards.
             currentrev = len(parentsmap) - len(revmap) - 1
             revmap[current] = currentrev
 
             if prevrev:
                 parentrevs[prevrev].append(currentrev)
 
             for parent in parentsmap.get(current):
                 queue.appendleft((currentrev, parent))
 
         return revmap, parentrevs.__getitem__
 
     def strip(self, minlink, transaction):
         pass
 
     # misc unused things
     def files(self):
         return []
 
     def checksize(self):
         return 0, 0

hgext/remotefilelog/shallowbundle.py

@@ -1,303 +1,307 b''
 # shallowbundle.py - bundle10 implementation for use with shallow repositories
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid
 from mercurial import (
     bundlerepo,
     changegroup,
     error,
     match,
     mdiff,
     pycompat,
 )
 from . import (
     constants,
     remotefilelog,
     shallowutil,
 )
 
 NoFiles = 0
 LocalFiles = 1
 AllFiles = 2
 
 
 def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None):
     if not isinstance(rlog, remotefilelog.remotefilelog):
         for c in super(cls, self).group(nodelist, rlog, lookup, units=units):
             yield c
         return
 
     if len(nodelist) == 0:
         yield self.close()
         return
 
     nodelist = shallowutil.sortnodes(nodelist, rlog.parents)
 
     # add the parent of the first rev
     p = rlog.parents(nodelist[0])[0]
     nodelist.insert(0, p)
 
     # build deltas
     for i in pycompat.xrange(len(nodelist) - 1):
         prev, curr = nodelist[i], nodelist[i + 1]
         linknode = lookup(curr)
         for c in self.nodechunk(rlog, curr, prev, linknode):
             yield c
 
     yield self.close()
 
 
 class shallowcg1packer(changegroup.cgpacker):
     def generate(self, commonrevs, clnodes, fastpathlinkrev, source, **kwargs):
         if shallowutil.isenabled(self._repo):
             fastpathlinkrev = False
 
         return super(shallowcg1packer, self).generate(
             commonrevs, clnodes, fastpathlinkrev, source, **kwargs
         )
 
     def group(self, nodelist, rlog, lookup, units=None, reorder=None):
         return shallowgroup(
             shallowcg1packer, self, nodelist, rlog, lookup, units=units
         )
 
-    def generatefiles(self, changedfiles, *args):
+    def generatefiles(self, changedfiles, *args, **kwargs):
         try:
             linknodes, commonrevs, source = args
         except ValueError:
             commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
         if shallowutil.isenabled(self._repo):
             repo = self._repo
             if isinstance(repo, bundlerepo.bundlerepository):
                 # If the bundle contains filelogs, we can't pull from it, since
                 # bundlerepo is heavily tied to revlogs. Instead require that
                 # the user use unbundle instead.
                 # Force load the filelog data.
                 bundlerepo.bundlerepository.file(repo, b'foo')
                 if repo._cgfilespos:
                     raise error.Abort(
                         b"cannot pull from full bundles",
                         hint=b"use `hg unbundle` instead",
                     )
                 return []
             filestosend = self.shouldaddfilegroups(source)
             if filestosend == NoFiles:
                 changedfiles = list(
                     [f for f in changedfiles if not repo.shallowmatch(f)]
                 )
 
-        return super(shallowcg1packer, self).generatefiles(changedfiles, *args)
+        return super(shallowcg1packer, self).generatefiles(
+            changedfiles, *args, **kwargs
+        )
 
     def shouldaddfilegroups(self, source):
         repo = self._repo
         if not shallowutil.isenabled(repo):
             return AllFiles
 
         if source == b"push" or source == b"bundle":
             return AllFiles
 
         caps = self._bundlecaps or []
         if source == b"serve" or source == b"pull":
             if constants.BUNDLE2_CAPABLITY in caps:
                 return LocalFiles
             else:
                 # Serving to a full repo requires us to serve everything
                 repo.ui.warn(_(b"pulling from a shallow repo\n"))
                 return AllFiles
 
         return NoFiles
 
     def prune(self, rlog, missing, commonrevs):
         if not isinstance(rlog, remotefilelog.remotefilelog):
             return super(shallowcg1packer, self).prune(
                 rlog, missing, commonrevs
             )
 
         repo = self._repo
         results = []
         for fnode in missing:
             fctx = repo.filectx(rlog.filename, fileid=fnode)
             if fctx.linkrev() not in commonrevs:
                 results.append(fnode)
         return results
 
     def nodechunk(self, revlog, node, prevnode, linknode):
         prefix = b''
         if prevnode == nullid:
             delta = revlog.rawdata(node)
             prefix = mdiff.trivialdiffheader(len(delta))
         else:
             # Actually uses remotefilelog.revdiff which works on nodes, not revs
             delta = revlog.revdiff(prevnode, node)
         p1, p2 = revlog.parents(node)
         flags = revlog.flags(node)
         meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags)
         meta += prefix
         l = len(meta) + len(delta)
         yield changegroup.chunkheader(l)
         yield meta
         yield delta
 
 
 def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
     if not shallowutil.isenabled(repo):
         return orig(repo, outgoing, version, source, *args, **kwargs)
 
     original = repo.shallowmatch
     try:
         # if serving, only send files the clients has patterns for
         if source == b'serve':
             bundlecaps = kwargs.get('bundlecaps')
             includepattern = None
             excludepattern = None
             for cap in bundlecaps or []:
                 if cap.startswith(b"includepattern="):
                     raw = cap[len(b"includepattern=") :]
                     if raw:
                         includepattern = raw.split(b'\0')
                 elif cap.startswith(b"excludepattern="):
                     raw = cap[len(b"excludepattern=") :]
                     if raw:
                         excludepattern = raw.split(b'\0')
             if includepattern or excludepattern:
                 repo.shallowmatch = match.match(
                     repo.root, b'', None, includepattern, excludepattern
                 )
             else:
                 repo.shallowmatch = match.always()
         return orig(repo, outgoing, version, source, *args, **kwargs)
     finally:
         repo.shallowmatch = original
 
 
-def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+def addchangegroupfiles(
+    orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
+):
     if not shallowutil.isenabled(repo):
-        return orig(repo, source, revmap, trp, expectedfiles, *args)
+        return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
 
     newfiles = 0
     visited = set()
     revisiondatas = {}
     queue = []
 
     # Normal Mercurial processes each file one at a time, adding all
     # the new revisions for that file at once. In remotefilelog a file
     # revision may depend on a different file's revision (in the case
     # of a rename/copy), so we must lay all revisions down across all
     # files in topological order.
 
     # read all the file chunks but don't add them
     progress = repo.ui.makeprogress(_(b'files'), total=expectedfiles)
     while True:
         chunkdata = source.filelogheader()
         if not chunkdata:
             break
         f = chunkdata[b"filename"]
         repo.ui.debug(b"adding %s revisions\n" % f)
         progress.increment()
 
         if not repo.shallowmatch(f):
             fl = repo.file(f)
             deltas = source.deltaiter()
             fl.addgroup(deltas, revmap, trp)
             continue
 
         chain = None
         while True:
             # returns: (node, p1, p2, cs, deltabase, delta, flags) or None
             revisiondata = source.deltachunk(chain)
             if not revisiondata:
                 break
 
             chain = revisiondata[0]
 
             revisiondatas[(f, chain)] = revisiondata
             queue.append((f, chain))
 
             if f not in visited:
                 newfiles += 1
                 visited.add(f)
 
         if chain is None:
             raise error.Abort(_(b"received file revlog group is empty"))
 
     processed = set()
 
     def available(f, node, depf, depnode):
         if depnode != nullid and (depf, depnode) not in processed:
             if not (depf, depnode) in revisiondatas:
                 # It's not in the changegroup, assume it's already
                 # in the repo
                 return True
             # re-add self to queue
             queue.insert(0, (f, node))
             # add dependency in front
             queue.insert(0, (depf, depnode))
             return False
         return True
 
     skipcount = 0
 
     # Prefetch the non-bundled revisions that we will need
     prefetchfiles = []
     for f, node in queue:
         revisiondata = revisiondatas[(f, node)]
         # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
         dependents = [revisiondata[1], revisiondata[2], revisiondata[4]]
 
         for dependent in dependents:
             if dependent == nullid or (f, dependent) in revisiondatas:
                 continue
             prefetchfiles.append((f, hex(dependent)))
 
     repo.fileservice.prefetch(prefetchfiles)
 
     # Apply the revisions in topological order such that a revision
     # is only written once it's deltabase and parents have been written.
     while queue:
         f, node = queue.pop(0)
         if (f, node) in processed:
             continue
 
         skipcount += 1
         if skipcount > len(queue) + 1:
             raise error.Abort(_(b"circular node dependency"))
 
         fl = repo.file(f)
 
         revisiondata = revisiondatas[(f, node)]
         # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
-        node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+        node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
 
         if not available(f, node, f, deltabase):
             continue
 
         base = fl.rawdata(deltabase)
         text = mdiff.patch(base, delta)
         if not isinstance(text, bytes):
             text = bytes(text)
 
         meta, text = shallowutil.parsemeta(text)
         if b'copy' in meta:
             copyfrom = meta[b'copy']
             copynode = bin(meta[b'copyrev'])
             if not available(f, node, copyfrom, copynode):
                 continue
 
         for p in [p1, p2]:
             if p != nullid:
                 if not available(f, node, f, p):
                     continue
 
         fl.add(text, meta, trp, linknode, p1, p2)
         processed.add((f, node))
         skipcount = 0
 
     progress.complete()
 
     return len(revisiondatas), newfiles

hgext/sqlitestore.py

@@ -1,1301 +1,1310 b''
 # sqlitestore.py - Storage backend that uses SQLite
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """store repository data in SQLite (EXPERIMENTAL)
 
 The sqlitestore extension enables the storage of repository data in SQLite.
 
 This extension is HIGHLY EXPERIMENTAL. There are NO BACKWARDS COMPATIBILITY
 GUARANTEES. This means that repositories created with this extension may
 only be usable with the exact version of this extension/Mercurial that was
 used. The extension attempts to enforce this in order to prevent repository
 corruption.
 
 In addition, several features are not yet supported or have known bugs:
 
 * Only some data is stored in SQLite. Changeset, manifest, and other repository
   data is not yet stored in SQLite.
 * Transactions are not robust. If the process is aborted at the right time
   during transaction close/rollback, the repository could be in an inconsistent
   state. This problem will diminish once all repository data is tracked by
   SQLite.
 * Bundle repositories do not work (the ability to use e.g.
   `hg -R <bundle-file> log` to automatically overlay a bundle on top of the
   existing repository).
 * Various other features don't work.
 
 This extension should work for basic clone/pull, update, and commit workflows.
 Some history rewriting operations may fail due to lack of support for bundle
 repositories.
 
 To use, activate the extension and set the ``storage.new-repo-backend`` config
 option to ``sqlite`` to enable new repositories to use SQLite for storage.
 """
 
 # To run the test suite with repos using SQLite by default, execute the
 # following:
 #
 # HGREPOFEATURES="sqlitestore" run-tests.py \
 #     --extra-config-opt extensions.sqlitestore= \
 #     --extra-config-opt storage.new-repo-backend=sqlite
 
 from __future__ import absolute_import
 
 import sqlite3
 import struct
 import threading
 import zlib
 
 from mercurial.i18n import _
 from mercurial.node import (
     nullid,
     nullrev,
     short,
 )
 from mercurial.thirdparty import attr
 from mercurial import (
     ancestor,
     dagop,
     encoding,
     error,
     extensions,
     localrepo,
     mdiff,
     pycompat,
     registrar,
     requirements,
     util,
     verify,
 )
 from mercurial.interfaces import (
     repository,
     util as interfaceutil,
 )
 from mercurial.utils import (
     hashutil,
     storageutil,
 )
 
 try:
     from mercurial import zstd
 
     zstd.__version__
 except ImportError:
     zstd = None
 
 configtable = {}
 configitem = registrar.configitem(configtable)
 
 # experimental config: storage.sqlite.compression
 configitem(
     b'storage',
     b'sqlite.compression',
     default=b'zstd' if zstd else b'zlib',
     experimental=True,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = b'ships-with-hg-core'
 
 REQUIREMENT = b'exp-sqlite-001'
 REQUIREMENT_ZSTD = b'exp-sqlite-comp-001=zstd'
 REQUIREMENT_ZLIB = b'exp-sqlite-comp-001=zlib'
 REQUIREMENT_NONE = b'exp-sqlite-comp-001=none'
 REQUIREMENT_SHALLOW_FILES = b'exp-sqlite-shallow-files'
 
 CURRENT_SCHEMA_VERSION = 1
 
 COMPRESSION_NONE = 1
 COMPRESSION_ZSTD = 2
 COMPRESSION_ZLIB = 3
 
 FLAG_CENSORED = 1
 FLAG_MISSING_P1 = 2
 FLAG_MISSING_P2 = 4
 
 CREATE_SCHEMA = [
     # Deltas are stored as content-indexed blobs.
     # compression column holds COMPRESSION_* constant for how the
     # delta is encoded.
     'CREATE TABLE delta ('
     '    id INTEGER PRIMARY KEY, '
     '    compression INTEGER NOT NULL, '
     '    hash BLOB UNIQUE ON CONFLICT ABORT, '
     '    delta BLOB NOT NULL '
     ')',
     # Tracked paths are denormalized to integers to avoid redundant
     # storage of the path name.
     'CREATE TABLE filepath ('
     '    id INTEGER PRIMARY KEY, '
     '    path BLOB NOT NULL '
     ')',
     'CREATE UNIQUE INDEX filepath_path ON filepath (path)',
     # We have a single table for all file revision data.
     # Each file revision is uniquely described by a (path, rev) and
     # (path, node).
     #
     # Revision data is stored as a pointer to the delta producing this
     # revision and the file revision whose delta should be applied before
     # that one. One can reconstruct the delta chain by recursively following
     # the delta base revision pointers until one encounters NULL.
     #
     # flags column holds bitwise integer flags controlling storage options.
     # These flags are defined by the FLAG_* constants.
     'CREATE TABLE fileindex ('
     '    id INTEGER PRIMARY KEY, '
     '    pathid INTEGER REFERENCES filepath(id), '
     '    revnum INTEGER NOT NULL, '
     '    p1rev INTEGER NOT NULL, '
     '    p2rev INTEGER NOT NULL, '
     '    linkrev INTEGER NOT NULL, '
     '    flags INTEGER NOT NULL, '
     '    deltaid INTEGER REFERENCES delta(id), '
     '    deltabaseid INTEGER REFERENCES fileindex(id), '
     '    node BLOB NOT NULL '
     ')',
     'CREATE UNIQUE INDEX fileindex_pathrevnum '
     '    ON fileindex (pathid, revnum)',
     'CREATE UNIQUE INDEX fileindex_pathnode ON fileindex (pathid, node)',
     # Provide a view over all file data for convenience.
     'CREATE VIEW filedata AS '
     'SELECT '
     '    fileindex.id AS id, '
     '    filepath.id AS pathid, '
     '    filepath.path AS path, '
     '    fileindex.revnum AS revnum, '
     '    fileindex.node AS node, '
     '    fileindex.p1rev AS p1rev, '
     '    fileindex.p2rev AS p2rev, '
     '    fileindex.linkrev AS linkrev, '
     '    fileindex.flags AS flags, '
     '    fileindex.deltaid AS deltaid, '
     '    fileindex.deltabaseid AS deltabaseid '
     'FROM filepath, fileindex '
     'WHERE fileindex.pathid=filepath.id',
     'PRAGMA user_version=%d' % CURRENT_SCHEMA_VERSION,
 ]
 
 
 def resolvedeltachain(db, pathid, node, revisioncache, stoprids, zstddctx=None):
     """Resolve a delta chain for a file node."""
 
     # TODO the "not in ({stops})" here is possibly slowing down the query
     # because it needs to perform the lookup on every recursive invocation.
     # This could possibly be faster if we created a temporary query with
     # baseid "poisoned" to null and limited the recursive filter to
     # "is not null".
     res = db.execute(
         'WITH RECURSIVE '
         '    deltachain(deltaid, baseid) AS ('
         '        SELECT deltaid, deltabaseid FROM fileindex '
         '            WHERE pathid=? AND node=? '
         '        UNION ALL '
         '        SELECT fileindex.deltaid, deltabaseid '
         '            FROM fileindex, deltachain '
         '            WHERE '
         '                fileindex.id=deltachain.baseid '
         '                AND deltachain.baseid IS NOT NULL '
         '                AND fileindex.id NOT IN ({stops}) '
         '    ) '
         'SELECT deltachain.baseid, compression, delta '
         'FROM deltachain, delta '
         'WHERE delta.id=deltachain.deltaid'.format(
             stops=','.join(['?'] * len(stoprids))
         ),
         tuple([pathid, node] + list(stoprids.keys())),
     )
 
     deltas = []
     lastdeltabaseid = None
 
     for deltabaseid, compression, delta in res:
         lastdeltabaseid = deltabaseid
 
         if compression == COMPRESSION_ZSTD:
             delta = zstddctx.decompress(delta)
         elif compression == COMPRESSION_NONE:
             delta = delta
         elif compression == COMPRESSION_ZLIB:
             delta = zlib.decompress(delta)
         else:
             raise SQLiteStoreError(
                 b'unhandled compression type: %d' % compression
             )
 
         deltas.append(delta)
 
     if lastdeltabaseid in stoprids:
         basetext = revisioncache[stoprids[lastdeltabaseid]]
     else:
         basetext = deltas.pop()
 
     deltas.reverse()
     fulltext = mdiff.patches(basetext, deltas)
 
     # SQLite returns buffer instances for blob columns on Python 2. This
     # type can propagate through the delta application layer. Because
     # downstream callers assume revisions are bytes, cast as needed.
     if not isinstance(fulltext, bytes):
         fulltext = bytes(delta)
 
     return fulltext
 
 
 def insertdelta(db, compression, hash, delta):
     try:
         return db.execute(
             'INSERT INTO delta (compression, hash, delta) VALUES (?, ?, ?)',
             (compression, hash, delta),
         ).lastrowid
     except sqlite3.IntegrityError:
         return db.execute(
             'SELECT id FROM delta WHERE hash=?', (hash,)
         ).fetchone()[0]
 
 
 class SQLiteStoreError(error.StorageError):
     pass
 
 
 @attr.s
 class revisionentry(object):
     rid = attr.ib()
     rev = attr.ib()
     node = attr.ib()
     p1rev = attr.ib()
     p2rev = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     linkrev = attr.ib()
     flags = attr.ib()
 
 
 @interfaceutil.implementer(repository.irevisiondelta)
 @attr.s(slots=True)
 class sqliterevisiondelta(object):
     node = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     basenode = attr.ib()
     flags = attr.ib()
     baserevisionsize = attr.ib()
     revision = attr.ib()
     delta = attr.ib()
     linknode = attr.ib(default=None)
 
 
 @interfaceutil.implementer(repository.iverifyproblem)
 @attr.s(frozen=True)
 class sqliteproblem(object):
     warning = attr.ib(default=None)
     error = attr.ib(default=None)
     node = attr.ib(default=None)
 
 
 @interfaceutil.implementer(repository.ifilestorage)
 class sqlitefilestore(object):
     """Implements storage for an individual tracked path."""
 
     def __init__(self, db, path, compression):
         self._db = db
         self._path = path
 
         self._pathid = None
 
         # revnum -> node
         self._revtonode = {}
         # node -> revnum
         self._nodetorev = {}
         # node -> data structure
         self._revisions = {}
 
         self._revisioncache = util.lrucachedict(10)
 
         self._compengine = compression
 
         if compression == b'zstd':
             self._cctx = zstd.ZstdCompressor(level=3)
             self._dctx = zstd.ZstdDecompressor()
         else:
             self._cctx = None
             self._dctx = None
 
         self._refreshindex()
 
     def _refreshindex(self):
         self._revtonode = {}
         self._nodetorev = {}
         self._revisions = {}
 
         res = list(
             self._db.execute(
                 'SELECT id FROM filepath WHERE path=?', (self._path,)
             )
         )
 
         if not res:
             self._pathid = None
             return
 
         self._pathid = res[0][0]
 
         res = self._db.execute(
             'SELECT id, revnum, node, p1rev, p2rev, linkrev, flags '
             'FROM fileindex '
             'WHERE pathid=? '
             'ORDER BY revnum ASC',
             (self._pathid,),
         )
 
         for i, row in enumerate(res):
             rid, rev, node, p1rev, p2rev, linkrev, flags = row
 
             if i != rev:
                 raise SQLiteStoreError(
                     _(b'sqlite database has inconsistent revision numbers')
                 )
 
             if p1rev == nullrev:
                 p1node = nullid
             else:
                 p1node = self._revtonode[p1rev]
 
             if p2rev == nullrev:
                 p2node = nullid
             else:
                 p2node = self._revtonode[p2rev]
 
             entry = revisionentry(
                 rid=rid,
                 rev=rev,
                 node=node,
                 p1rev=p1rev,
                 p2rev=p2rev,
                 p1node=p1node,
                 p2node=p2node,
                 linkrev=linkrev,
                 flags=flags,
             )
 
             self._revtonode[rev] = node
             self._nodetorev[node] = rev
             self._revisions[node] = entry
 
     # Start of ifileindex interface.
 
     def __len__(self):
         return len(self._revisions)
 
     def __iter__(self):
         return iter(pycompat.xrange(len(self._revisions)))
 
     def hasnode(self, node):
         if node == nullid:
             return False
 
         return node in self._nodetorev
 
     def revs(self, start=0, stop=None):
         return storageutil.iterrevs(
             len(self._revisions), start=start, stop=stop
         )
 
     def parents(self, node):
         if node == nullid:
             return nullid, nullid
 
         if node not in self._revisions:
             raise error.LookupError(node, self._path, _(b'no node'))
 
         entry = self._revisions[node]
         return entry.p1node, entry.p2node
 
     def parentrevs(self, rev):
         if rev == nullrev:
             return nullrev, nullrev
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         entry = self._revisions[self._revtonode[rev]]
         return entry.p1rev, entry.p2rev
 
     def rev(self, node):
         if node == nullid:
             return nullrev
 
         if node not in self._nodetorev:
             raise error.LookupError(node, self._path, _(b'no node'))
 
         return self._nodetorev[node]
 
     def node(self, rev):
         if rev == nullrev:
             return nullid
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         return self._revtonode[rev]
 
     def lookup(self, node):
         return storageutil.fileidlookup(self, node, self._path)
 
     def linkrev(self, rev):
         if rev == nullrev:
             return nullrev
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         entry = self._revisions[self._revtonode[rev]]
         return entry.linkrev
 
     def iscensored(self, rev):
         if rev == nullrev:
             return False
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         return self._revisions[self._revtonode[rev]].flags & FLAG_CENSORED
 
     def commonancestorsheads(self, node1, node2):
         rev1 = self.rev(node1)
         rev2 = self.rev(node2)
 
         ancestors = ancestor.commonancestorsheads(self.parentrevs, rev1, rev2)
         return pycompat.maplist(self.node, ancestors)
 
     def descendants(self, revs):
         # TODO we could implement this using a recursive SQL query, which
         # might be faster.
         return dagop.descendantrevs(revs, self.revs, self.parentrevs)
 
     def heads(self, start=None, stop=None):
         if start is None and stop is None:
             if not len(self):
                 return [nullid]
 
         startrev = self.rev(start) if start is not None else nullrev
         stoprevs = {self.rev(n) for n in stop or []}
 
         revs = dagop.headrevssubset(
             self.revs, self.parentrevs, startrev=startrev, stoprevs=stoprevs
         )
 
         return [self.node(rev) for rev in revs]
 
     def children(self, node):
         rev = self.rev(node)
 
         res = self._db.execute(
             'SELECT'
             '  node '
             '  FROM filedata '
             '  WHERE path=? AND (p1rev=? OR p2rev=?) '
             '  ORDER BY revnum ASC',
             (self._path, rev, rev),
         )
 
         return [row[0] for row in res]
 
     # End of ifileindex interface.
 
     # Start of ifiledata interface.
 
     def size(self, rev):
         if rev == nullrev:
             return 0
 
         if rev not in self._revtonode:
             raise IndexError(rev)
 
         node = self._revtonode[rev]
 
         if self.renamed(node):
             return len(self.read(node))
 
         return len(self.revision(node))
 
     def revision(self, node, raw=False, _verifyhash=True):
         if node in (nullid, nullrev):
             return b''
 
         if isinstance(node, int):
             node = self.node(node)
 
         if node not in self._nodetorev:
             raise error.LookupError(node, self._path, _(b'no node'))
 
         if node in self._revisioncache:
             return self._revisioncache[node]
 
         # Because we have a fulltext revision cache, we are able to
         # short-circuit delta chain traversal and decompression as soon as
         # we encounter a revision in the cache.
 
         stoprids = {self._revisions[n].rid: n for n in self._revisioncache}
 
         if not stoprids:
             stoprids[-1] = None
 
         fulltext = resolvedeltachain(
             self._db,
             self._pathid,
             node,
             self._revisioncache,
             stoprids,
             zstddctx=self._dctx,
         )
 
         # Don't verify hashes if parent nodes were rewritten, as the hash
         # wouldn't verify.
         if self._revisions[node].flags & (FLAG_MISSING_P1 | FLAG_MISSING_P2):
             _verifyhash = False
 
         if _verifyhash:
             self._checkhash(fulltext, node)
             self._revisioncache[node] = fulltext
 
         return fulltext
 
     def rawdata(self, *args, **kwargs):
         return self.revision(*args, **kwargs)
 
     def read(self, node):
         return storageutil.filtermetadata(self.revision(node))
 
     def renamed(self, node):
         return storageutil.filerevisioncopied(self, node)
 
     def cmp(self, node, fulltext):
         return not storageutil.filedataequivalent(self, node, fulltext)
 
     def emitrevisions(
         self,
         nodes,
         nodesorder=None,
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
     ):
         if nodesorder not in (b'nodes', b'storage', b'linear', None):
             raise error.ProgrammingError(
                 b'unhandled value for nodesorder: %s' % nodesorder
             )
 
         nodes = [n for n in nodes if n != nullid]
 
         if not nodes:
             return
 
         # TODO perform in a single query.
         res = self._db.execute(
             'SELECT revnum, deltaid FROM fileindex '
             'WHERE pathid=? '
             '    AND node in (%s)' % (','.join(['?'] * len(nodes))),
             tuple([self._pathid] + nodes),
         )
 
         deltabases = {}
 
         for rev, deltaid in res:
             res = self._db.execute(
                 'SELECT revnum from fileindex WHERE pathid=? AND deltaid=?',
                 (self._pathid, deltaid),
             )
             deltabases[rev] = res.fetchone()[0]
 
         # TODO define revdifffn so we can use delta from storage.
         for delta in storageutil.emitrevisions(
             self,
             nodes,
             nodesorder,
             sqliterevisiondelta,
             deltaparentfn=deltabases.__getitem__,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
         ):
 
             yield delta
 
     # End of ifiledata interface.
 
     # Start of ifilemutation interface.
 
     def add(self, filedata, meta, transaction, linkrev, p1, p2):
         if meta or filedata.startswith(b'\x01\n'):
             filedata = storageutil.packmeta(meta, filedata)
 
         rev = self.addrevision(filedata, transaction, linkrev, p1, p2)
         return self.node(rev)
 
     def addrevision(
         self,
         revisiondata,
         transaction,
         linkrev,
         p1,
         p2,
         node=None,
         flags=0,
         cachedelta=None,
     ):
         if flags:
             raise SQLiteStoreError(_(b'flags not supported on revisions'))
 
         validatehash = node is not None
         node = node or storageutil.hashrevisionsha1(revisiondata, p1, p2)
 
         if validatehash:
             self._checkhash(revisiondata, node, p1, p2)
 
         rev = self._nodetorev.get(node)
         if rev is not None:
             return rev
 
         rev = self._addrawrevision(
             node, revisiondata, transaction, linkrev, p1, p2
         )
 
         self._revisioncache[node] = revisiondata
         return rev
 
     def addgroup(
         self,
         deltas,
         linkmapper,
         transaction,
         addrevisioncb=None,
         duplicaterevisioncb=None,
         maybemissingparents=False,
     ):
         empty = True
 
-        for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
+        for (
+            node,
+            p1,
+            p2,
+            linknode,
+            deltabase,
+            delta,
+            wireflags,
+            sidedata,
+        ) in deltas:
             storeflags = 0
 
             if wireflags & repository.REVISION_FLAG_CENSORED:
                 storeflags |= FLAG_CENSORED
 
             if wireflags & ~repository.REVISION_FLAG_CENSORED:
                 raise SQLiteStoreError(b'unhandled revision flag')
 
             if maybemissingparents:
                 if p1 != nullid and not self.hasnode(p1):
                     p1 = nullid
                     storeflags |= FLAG_MISSING_P1
 
                 if p2 != nullid and not self.hasnode(p2):
                     p2 = nullid
                     storeflags |= FLAG_MISSING_P2
 
             baserev = self.rev(deltabase)
 
             # If base is censored, delta must be full replacement in a single
             # patch operation.
             if baserev != nullrev and self.iscensored(baserev):
                 hlen = struct.calcsize(b'>lll')
                 oldlen = len(self.rawdata(deltabase, _verifyhash=False))
                 newlen = len(delta) - hlen
 
                 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
                     raise error.CensoredBaseError(self._path, deltabase)
 
             if not (storeflags & FLAG_CENSORED) and storageutil.deltaiscensored(
                 delta, baserev, lambda x: len(self.rawdata(x))
             ):
                 storeflags |= FLAG_CENSORED
 
             linkrev = linkmapper(linknode)
 
             if node in self._revisions:
                 # Possibly reset parents to make them proper.
                 entry = self._revisions[node]
 
                 if entry.flags & FLAG_MISSING_P1 and p1 != nullid:
                     entry.p1node = p1
                     entry.p1rev = self._nodetorev[p1]
                     entry.flags &= ~FLAG_MISSING_P1
 
                     self._db.execute(
                         'UPDATE fileindex SET p1rev=?, flags=? WHERE id=?',
                         (self._nodetorev[p1], entry.flags, entry.rid),
                     )
 
                 if entry.flags & FLAG_MISSING_P2 and p2 != nullid:
                     entry.p2node = p2
                     entry.p2rev = self._nodetorev[p2]
                     entry.flags &= ~FLAG_MISSING_P2
 
                     self._db.execute(
                         'UPDATE fileindex SET p2rev=?, flags=? WHERE id=?',
                         (self._nodetorev[p1], entry.flags, entry.rid),
                     )
 
                 if duplicaterevisioncb:
                     duplicaterevisioncb(self, self.rev(node))
                 empty = False
                 continue
 
             if deltabase == nullid:
                 text = mdiff.patch(b'', delta)
                 storedelta = None
             else:
                 text = None
                 storedelta = (deltabase, delta)
 
             rev = self._addrawrevision(
                 node,
                 text,
                 transaction,
                 linkrev,
                 p1,
                 p2,
                 storedelta=storedelta,
                 flags=storeflags,
             )
 
             if addrevisioncb:
                 addrevisioncb(self, rev)
             empty = False
 
         return not empty
 
     def censorrevision(self, tr, censornode, tombstone=b''):
         tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
 
         # This restriction is cargo culted from revlogs and makes no sense for
         # SQLite, since columns can be resized at will.
         if len(tombstone) > len(self.rawdata(censornode)):
             raise error.Abort(
                 _(b'censor tombstone must be no longer than censored data')
             )
 
         # We need to replace the censored revision's data with the tombstone.
         # But replacing that data will have implications for delta chains that
         # reference it.
         #
         # While "better," more complex strategies are possible, we do something
         # simple: we find delta chain children of the censored revision and we
         # replace those incremental deltas with fulltexts of their corresponding
         # revision. Then we delete the now-unreferenced delta and original
         # revision and insert a replacement.
 
         # Find the delta to be censored.
         censoreddeltaid = self._db.execute(
             'SELECT deltaid FROM fileindex WHERE id=?',
             (self._revisions[censornode].rid,),
         ).fetchone()[0]
 
         # Find all its delta chain children.
         # TODO once we support storing deltas for !files, we'll need to look
         # for those delta chains too.
         rows = list(
             self._db.execute(
                 'SELECT id, pathid, node FROM fileindex '
                 'WHERE deltabaseid=? OR deltaid=?',
                 (censoreddeltaid, censoreddeltaid),
             )
         )
 
         for row in rows:
             rid, pathid, node = row
 
             fulltext = resolvedeltachain(
                 self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx
             )
 
             deltahash = hashutil.sha1(fulltext).digest()
 
             if self._compengine == b'zstd':
                 deltablob = self._cctx.compress(fulltext)
                 compression = COMPRESSION_ZSTD
             elif self._compengine == b'zlib':
                 deltablob = zlib.compress(fulltext)
                 compression = COMPRESSION_ZLIB
             elif self._compengine == b'none':
                 deltablob = fulltext
                 compression = COMPRESSION_NONE
             else:
                 raise error.ProgrammingError(
                     b'unhandled compression engine: %s' % self._compengine
                 )
 
             if len(deltablob) >= len(fulltext):
                 deltablob = fulltext
                 compression = COMPRESSION_NONE
 
             deltaid = insertdelta(self._db, compression, deltahash, deltablob)
 
             self._db.execute(
                 'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
                 'WHERE id=?',
                 (deltaid, rid),
             )
 
         # Now create the tombstone delta and replace the delta on the censored
         # node.
         deltahash = hashutil.sha1(tombstone).digest()
         tombstonedeltaid = insertdelta(
             self._db, COMPRESSION_NONE, deltahash, tombstone
         )
 
         flags = self._revisions[censornode].flags
         flags |= FLAG_CENSORED
 
         self._db.execute(
             'UPDATE fileindex SET flags=?, deltaid=?, deltabaseid=NULL '
             'WHERE pathid=? AND node=?',
             (flags, tombstonedeltaid, self._pathid, censornode),
         )
 
         self._db.execute('DELETE FROM delta WHERE id=?', (censoreddeltaid,))
 
         self._refreshindex()
         self._revisioncache.clear()
 
     def getstrippoint(self, minlink):
         return storageutil.resolvestripinfo(
             minlink,
             len(self) - 1,
             [self.rev(n) for n in self.heads()],
             self.linkrev,
             self.parentrevs,
         )
 
     def strip(self, minlink, transaction):
         if not len(self):
             return
 
         rev, _ignored = self.getstrippoint(minlink)
 
         if rev == len(self):
             return
 
         for rev in self.revs(rev):
             self._db.execute(
                 'DELETE FROM fileindex WHERE pathid=? AND node=?',
                 (self._pathid, self.node(rev)),
             )
 
         # TODO how should we garbage collect data in delta table?
 
         self._refreshindex()
 
     # End of ifilemutation interface.
 
     # Start of ifilestorage interface.
 
     def files(self):
         return []
 
     def storageinfo(
         self,
         exclusivefiles=False,
         sharedfiles=False,
         revisionscount=False,
         trackedsize=False,
         storedsize=False,
     ):
         d = {}
 
         if exclusivefiles:
             d[b'exclusivefiles'] = []
 
         if sharedfiles:
             # TODO list sqlite file(s) here.
             d[b'sharedfiles'] = []
 
         if revisionscount:
             d[b'revisionscount'] = len(self)
 
         if trackedsize:
             d[b'trackedsize'] = sum(
                 len(self.revision(node)) for node in self._nodetorev
             )
 
         if storedsize:
             # TODO implement this?
             d[b'storedsize'] = None
 
         return d
 
     def verifyintegrity(self, state):
         state[b'skipread'] = set()
 
         for rev in self:
             node = self.node(rev)
 
             try:
                 self.revision(node)
             except Exception as e:
                 yield sqliteproblem(
                     error=_(b'unpacking %s: %s') % (short(node), e), node=node
                 )
 
                 state[b'skipread'].add(node)
 
     # End of ifilestorage interface.
 
     def _checkhash(self, fulltext, node, p1=None, p2=None):
         if p1 is None and p2 is None:
             p1, p2 = self.parents(node)
 
         if node == storageutil.hashrevisionsha1(fulltext, p1, p2):
             return
 
         try:
             del self._revisioncache[node]
         except KeyError:
             pass
 
         if storageutil.iscensoredtext(fulltext):
             raise error.CensoredNodeError(self._path, node, fulltext)
 
         raise SQLiteStoreError(_(b'integrity check failed on %s') % self._path)
 
     def _addrawrevision(
         self,
         node,
         revisiondata,
         transaction,
         linkrev,
         p1,
         p2,
         storedelta=None,
         flags=0,
     ):
         if self._pathid is None:
             res = self._db.execute(
                 'INSERT INTO filepath (path) VALUES (?)', (self._path,)
             )
             self._pathid = res.lastrowid
 
         # For simplicity, always store a delta against p1.
         # TODO we need a lot more logic here to make behavior reasonable.
 
         if storedelta:
             deltabase, delta = storedelta
 
             if isinstance(deltabase, int):
                 deltabase = self.node(deltabase)
 
         else:
             assert revisiondata is not None
             deltabase = p1
 
             if deltabase == nullid:
                 delta = revisiondata
             else:
                 delta = mdiff.textdiff(
                     self.revision(self.rev(deltabase)), revisiondata
                 )
 
         # File index stores a pointer to its delta and the parent delta.
         # The parent delta is stored via a pointer to the fileindex PK.
         if deltabase == nullid:
             baseid = None
         else:
             baseid = self._revisions[deltabase].rid
 
         # Deltas are stored with a hash of their content. This allows
         # us to de-duplicate. The table is configured to ignore conflicts
         # and it is faster to just insert and silently noop than to look
         # first.
         deltahash = hashutil.sha1(delta).digest()
 
         if self._compengine == b'zstd':
             deltablob = self._cctx.compress(delta)
             compression = COMPRESSION_ZSTD
         elif self._compengine == b'zlib':
             deltablob = zlib.compress(delta)
             compression = COMPRESSION_ZLIB
         elif self._compengine == b'none':
             deltablob = delta
             compression = COMPRESSION_NONE
         else:
             raise error.ProgrammingError(
                 b'unhandled compression engine: %s' % self._compengine
             )
 
         # Don't store compressed data if it isn't practical.
         if len(deltablob) >= len(delta):
             deltablob = delta
             compression = COMPRESSION_NONE
 
         deltaid = insertdelta(self._db, compression, deltahash, deltablob)
 
         rev = len(self)
 
         if p1 == nullid:
             p1rev = nullrev
         else:
             p1rev = self._nodetorev[p1]
 
         if p2 == nullid:
             p2rev = nullrev
         else:
             p2rev = self._nodetorev[p2]
 
         rid = self._db.execute(
             'INSERT INTO fileindex ('
             '    pathid, revnum, node, p1rev, p2rev, linkrev, flags, '
             '    deltaid, deltabaseid) '
             '    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
             (
                 self._pathid,
                 rev,
                 node,
                 p1rev,
                 p2rev,
                 linkrev,
                 flags,
                 deltaid,
                 baseid,
             ),
         ).lastrowid
 
         entry = revisionentry(
             rid=rid,
             rev=rev,
             node=node,
             p1rev=p1rev,
             p2rev=p2rev,
             p1node=p1,
             p2node=p2,
             linkrev=linkrev,
             flags=flags,
         )
 
         self._nodetorev[node] = rev
         self._revtonode[rev] = node
         self._revisions[node] = entry
 
         return rev
 
 
 class sqliterepository(localrepo.localrepository):
     def cancopy(self):
         return False
 
     def transaction(self, *args, **kwargs):
         current = self.currenttransaction()
 
         tr = super(sqliterepository, self).transaction(*args, **kwargs)
 
         if current:
             return tr
 
         self._dbconn.execute('BEGIN TRANSACTION')
 
         def committransaction(_):
             self._dbconn.commit()
 
         tr.addfinalize(b'sqlitestore', committransaction)
 
         return tr
 
     @property
     def _dbconn(self):
         # SQLite connections can only be used on the thread that created
         # them. In most cases, this "just works." However, hgweb uses
         # multiple threads.
         tid = threading.current_thread().ident
 
         if self._db:
             if self._db[0] == tid:
                 return self._db[1]
 
         db = makedb(self.svfs.join(b'db.sqlite'))
         self._db = (tid, db)
 
         return db
 
 
 def makedb(path):
     """Construct a database handle for a database at path."""
 
     db = sqlite3.connect(encoding.strfromlocal(path))
     db.text_factory = bytes
 
     res = db.execute('PRAGMA user_version').fetchone()[0]
 
     # New database.
     if res == 0:
         for statement in CREATE_SCHEMA:
             db.execute(statement)
 
         db.commit()
 
     elif res == CURRENT_SCHEMA_VERSION:
         pass
 
     else:
         raise error.Abort(_(b'sqlite database has unrecognized version'))
 
     db.execute('PRAGMA journal_mode=WAL')
 
     return db
 
 
 def featuresetup(ui, supported):
     supported.add(REQUIREMENT)
 
     if zstd:
         supported.add(REQUIREMENT_ZSTD)
 
     supported.add(REQUIREMENT_ZLIB)
     supported.add(REQUIREMENT_NONE)
     supported.add(REQUIREMENT_SHALLOW_FILES)
     supported.add(requirements.NARROW_REQUIREMENT)
 
 
 def newreporequirements(orig, ui, createopts):
     if createopts[b'backend'] != b'sqlite':
         return orig(ui, createopts)
 
     # This restriction can be lifted once we have more confidence.
     if b'sharedrepo' in createopts:
         raise error.Abort(
             _(b'shared repositories not supported with SQLite store')
         )
 
     # This filtering is out of an abundance of caution: we want to ensure
     # we honor creation options and we do that by annotating exactly the
     # creation options we recognize.
     known = {
         b'narrowfiles',
         b'backend',
         b'shallowfilestore',
     }
 
     unsupported = set(createopts) - known
     if unsupported:
         raise error.Abort(
             _(b'SQLite store does not support repo creation option: %s')
             % b', '.join(sorted(unsupported))
         )
 
     # Since we're a hybrid store that still relies on revlogs, we fall back
     # to using the revlogv1 backend's storage requirements then adding our
     # own requirement.
     createopts[b'backend'] = b'revlogv1'
     requirements = orig(ui, createopts)
     requirements.add(REQUIREMENT)
 
     compression = ui.config(b'storage', b'sqlite.compression')
 
     if compression == b'zstd' and not zstd:
         raise error.Abort(
             _(
                 b'storage.sqlite.compression set to "zstd" but '
                 b'zstandard compression not available to this '
                 b'Mercurial install'
             )
         )
 
     if compression == b'zstd':
         requirements.add(REQUIREMENT_ZSTD)
     elif compression == b'zlib':
         requirements.add(REQUIREMENT_ZLIB)
     elif compression == b'none':
         requirements.add(REQUIREMENT_NONE)
     else:
         raise error.Abort(
             _(
                 b'unknown compression engine defined in '
                 b'storage.sqlite.compression: %s'
             )
             % compression
         )
 
     if createopts.get(b'shallowfilestore'):
         requirements.add(REQUIREMENT_SHALLOW_FILES)
 
     return requirements
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class sqlitefilestorage(object):
     """Repository file storage backed by SQLite."""
 
     def file(self, path):
         if path[0] == b'/':
             path = path[1:]
 
         if REQUIREMENT_ZSTD in self.requirements:
             compression = b'zstd'
         elif REQUIREMENT_ZLIB in self.requirements:
             compression = b'zlib'
         elif REQUIREMENT_NONE in self.requirements:
             compression = b'none'
         else:
             raise error.Abort(
                 _(
                     b'unable to determine what compression engine '
                     b'to use for SQLite storage'
                 )
             )
 
         return sqlitefilestore(self._dbconn, path, compression)
 
 
 def makefilestorage(orig, requirements, features, **kwargs):
     """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
     if REQUIREMENT in requirements:
         if REQUIREMENT_SHALLOW_FILES in requirements:
             features.add(repository.REPO_FEATURE_SHALLOW_FILE_STORAGE)
 
         return sqlitefilestorage
     else:
         return orig(requirements=requirements, features=features, **kwargs)
 
 
 def makemain(orig, ui, requirements, **kwargs):
     if REQUIREMENT in requirements:
         if REQUIREMENT_ZSTD in requirements and not zstd:
             raise error.Abort(
                 _(
                     b'repository uses zstandard compression, which '
                     b'is not available to this Mercurial install'
                 )
             )
 
         return sqliterepository
 
     return orig(requirements=requirements, **kwargs)
 
 
 def verifierinit(orig, self, *args, **kwargs):
     orig(self, *args, **kwargs)
 
     # We don't care that files in the store don't align with what is
     # advertised. So suppress these warnings.
     self.warnorphanstorefiles = False
 
 
 def extsetup(ui):
     localrepo.featuresetupfuncs.add(featuresetup)
     extensions.wrapfunction(
         localrepo, b'newreporequirements', newreporequirements
     )
     extensions.wrapfunction(localrepo, b'makefilestorage', makefilestorage)
     extensions.wrapfunction(localrepo, b'makemain', makemain)
     extensions.wrapfunction(verify.verifier, b'__init__', verifierinit)
 
 
 def reposetup(ui, repo):
     if isinstance(repo, sqliterepository):
         repo._db = None
 
     # TODO check for bundlerepository?

mercurial/bundlerepo.py

@@ -1,680 +1,680 b''
 # bundlerepo.py - repository class for viewing uncompressed bundles
 #
 # Copyright 2006, 2007 Benoit Boissinot <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Repository class for viewing uncompressed bundles.
 
 This provides a read-only repository interface to bundles as if they
 were part of the actual repository.
 """
 
 from __future__ import absolute_import
 
 import os
 import shutil
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
 )
 
 from . import (
     bundle2,
     changegroup,
     changelog,
     cmdutil,
     discovery,
     encoding,
     error,
     exchange,
     filelog,
     localrepo,
     manifest,
     mdiff,
     pathutil,
     phases,
     pycompat,
     revlog,
     util,
     vfs as vfsmod,
 )
 
 
 class bundlerevlog(revlog.revlog):
     def __init__(self, opener, indexfile, cgunpacker, linkmapper):
         # How it works:
         # To retrieve a revision, we need to know the offset of the revision in
         # the bundle (an unbundle object). We store this offset in the index
         # (start). The base of the delta is stored in the base field.
         #
         # To differentiate a rev in the bundle from a rev in the revlog, we
         # check revision against repotiprev.
         opener = vfsmod.readonlyvfs(opener)
         revlog.revlog.__init__(self, opener, indexfile)
         self.bundle = cgunpacker
         n = len(self)
         self.repotiprev = n - 1
         self.bundlerevs = set()  # used by 'bundle()' revset expression
         for deltadata in cgunpacker.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
 
             size = len(delta)
             start = cgunpacker.tell() - size
 
             if self.index.has_node(node):
                 # this can happen if two branches make the same change
                 self.bundlerevs.add(self.index.rev(node))
                 continue
             if cs == node:
                 linkrev = nullrev
             else:
                 linkrev = linkmapper(cs)
 
             for p in (p1, p2):
                 if not self.index.has_node(p):
                     raise error.LookupError(
                         p, self.indexfile, _(b"unknown parent")
                     )
 
             if not self.index.has_node(deltabase):
                 raise LookupError(
                     deltabase, self.indexfile, _(b'unknown delta base')
                 )
 
             baserev = self.rev(deltabase)
             # start, size, full unc. size, base (unused), link, p1, p2, node
             e = (
                 revlog.offset_type(start, flags),
                 size,
                 -1,
                 baserev,
                 linkrev,
                 self.rev(p1),
                 self.rev(p2),
                 node,
             )
             self.index.append(e)
             self.bundlerevs.add(n)
             n += 1
 
     def _chunk(self, rev, df=None):
         # Warning: in case of bundle, the diff is against what we stored as
         # delta base, not against rev - 1
         # XXX: could use some caching
         if rev <= self.repotiprev:
             return revlog.revlog._chunk(self, rev)
         self.bundle.seek(self.start(rev))
         return self.bundle.read(self.length(rev))
 
     def revdiff(self, rev1, rev2):
         """return or calculate a delta between two revisions"""
         if rev1 > self.repotiprev and rev2 > self.repotiprev:
             # hot path for bundle
             revb = self.index[rev2][3]
             if revb == rev1:
                 return self._chunk(rev2)
         elif rev1 <= self.repotiprev and rev2 <= self.repotiprev:
             return revlog.revlog.revdiff(self, rev1, rev2)
 
         return mdiff.textdiff(self.rawdata(rev1), self.rawdata(rev2))
 
     def _rawtext(self, node, rev, _df=None):
         if rev is None:
             rev = self.rev(node)
         validated = False
         rawtext = None
         chain = []
         iterrev = rev
         # reconstruct the revision if it is from a changegroup
         while iterrev > self.repotiprev:
             if self._revisioncache and self._revisioncache[1] == iterrev:
                 rawtext = self._revisioncache[2]
                 break
             chain.append(iterrev)
             iterrev = self.index[iterrev][3]
         if iterrev == nullrev:
             rawtext = b''
         elif rawtext is None:
             r = super(bundlerevlog, self)._rawtext(
                 self.node(iterrev), iterrev, _df=_df
             )
             __, rawtext, validated = r
         if chain:
             validated = False
         while chain:
             delta = self._chunk(chain.pop())
             rawtext = mdiff.patches(rawtext, [delta])
         return rev, rawtext, validated
 
     def addrevision(self, *args, **kwargs):
         raise NotImplementedError
 
     def addgroup(self, *args, **kwargs):
         raise NotImplementedError
 
     def strip(self, *args, **kwargs):
         raise NotImplementedError
 
     def checksize(self):
         raise NotImplementedError
 
 
 class bundlechangelog(bundlerevlog, changelog.changelog):
     def __init__(self, opener, cgunpacker):
         changelog.changelog.__init__(self, opener)
         linkmapper = lambda x: x
         bundlerevlog.__init__(
             self, opener, self.indexfile, cgunpacker, linkmapper
         )
 
 
 class bundlemanifest(bundlerevlog, manifest.manifestrevlog):
     def __init__(
         self, opener, cgunpacker, linkmapper, dirlogstarts=None, dir=b''
     ):
         manifest.manifestrevlog.__init__(self, opener, tree=dir)
         bundlerevlog.__init__(
             self, opener, self.indexfile, cgunpacker, linkmapper
         )
         if dirlogstarts is None:
             dirlogstarts = {}
             if self.bundle.version == b"03":
                 dirlogstarts = _getfilestarts(self.bundle)
         self._dirlogstarts = dirlogstarts
         self._linkmapper = linkmapper
 
     def dirlog(self, d):
         if d in self._dirlogstarts:
             self.bundle.seek(self._dirlogstarts[d])
             return bundlemanifest(
                 self.opener,
                 self.bundle,
                 self._linkmapper,
                 self._dirlogstarts,
                 dir=d,
             )
         return super(bundlemanifest, self).dirlog(d)
 
 
 class bundlefilelog(filelog.filelog):
     def __init__(self, opener, path, cgunpacker, linkmapper):
         filelog.filelog.__init__(self, opener, path)
         self._revlog = bundlerevlog(
             opener, self.indexfile, cgunpacker, linkmapper
         )
 
 
 class bundlepeer(localrepo.localpeer):
     def canpush(self):
         return False
 
 
 class bundlephasecache(phases.phasecache):
     def __init__(self, *args, **kwargs):
         super(bundlephasecache, self).__init__(*args, **kwargs)
         if util.safehasattr(self, 'opener'):
             self.opener = vfsmod.readonlyvfs(self.opener)
 
     def write(self):
         raise NotImplementedError
 
     def _write(self, fp):
         raise NotImplementedError
 
     def _updateroots(self, phase, newroots, tr):
         self.phaseroots[phase] = newroots
         self.invalidate()
         self.dirty = True
 
 
 def _getfilestarts(cgunpacker):
     filespos = {}
     for chunkdata in iter(cgunpacker.filelogheader, {}):
         fname = chunkdata[b'filename']
         filespos[fname] = cgunpacker.tell()
         for chunk in iter(lambda: cgunpacker.deltachunk(None), {}):
             pass
     return filespos
 
 
 class bundlerepository(object):
     """A repository instance that is a union of a local repo and a bundle.
 
     Instances represent a read-only repository composed of a local repository
     with the contents of a bundle file applied. The repository instance is
     conceptually similar to the state of a repository after an
     ``hg unbundle`` operation. However, the contents of the bundle are never
     applied to the actual base repository.
 
     Instances constructed directly are not usable as repository objects.
     Use instance() or makebundlerepository() to create instances.
     """
 
     def __init__(self, bundlepath, url, tempparent):
         self._tempparent = tempparent
         self._url = url
 
         self.ui.setconfig(b'phases', b'publish', False, b'bundlerepo')
 
         self.tempfile = None
         f = util.posixfile(bundlepath, b"rb")
         bundle = exchange.readbundle(self.ui, f, bundlepath)
 
         if isinstance(bundle, bundle2.unbundle20):
             self._bundlefile = bundle
             self._cgunpacker = None
 
             cgpart = None
             for part in bundle.iterparts(seekable=True):
                 if part.type == b'changegroup':
                     if cgpart:
                         raise NotImplementedError(
                             b"can't process multiple changegroups"
                         )
                     cgpart = part
 
                 self._handlebundle2part(bundle, part)
 
             if not cgpart:
                 raise error.Abort(_(b"No changegroups found"))
 
             # This is required to placate a later consumer, which expects
             # the payload offset to be at the beginning of the changegroup.
             # We need to do this after the iterparts() generator advances
             # because iterparts() will seek to end of payload after the
             # generator returns control to iterparts().
             cgpart.seek(0, os.SEEK_SET)
 
         elif isinstance(bundle, changegroup.cg1unpacker):
             if bundle.compressed():
                 f = self._writetempbundle(
                     bundle.read, b'.hg10un', header=b'HG10UN'
                 )
                 bundle = exchange.readbundle(self.ui, f, bundlepath, self.vfs)
 
             self._bundlefile = bundle
             self._cgunpacker = bundle
         else:
             raise error.Abort(
                 _(b'bundle type %s cannot be read') % type(bundle)
             )
 
         # dict with the mapping 'filename' -> position in the changegroup.
         self._cgfilespos = {}
 
         self.firstnewrev = self.changelog.repotiprev + 1
         phases.retractboundary(
             self,
             None,
             phases.draft,
             [ctx.node() for ctx in self[self.firstnewrev :]],
         )
 
     def _handlebundle2part(self, bundle, part):
         if part.type != b'changegroup':
             return
 
         cgstream = part
         version = part.params.get(b'version', b'01')
         legalcgvers = changegroup.supportedincomingversions(self)
         if version not in legalcgvers:
             msg = _(b'Unsupported changegroup version: %s')
             raise error.Abort(msg % version)
         if bundle.compressed():
             cgstream = self._writetempbundle(part.read, b'.cg%sun' % version)
 
         self._cgunpacker = changegroup.getunbundler(version, cgstream, b'UN')
 
     def _writetempbundle(self, readfn, suffix, header=b''):
         """Write a temporary file to disk"""
         fdtemp, temp = self.vfs.mkstemp(prefix=b"hg-bundle-", suffix=suffix)
         self.tempfile = temp
 
         with os.fdopen(fdtemp, 'wb') as fptemp:
             fptemp.write(header)
             while True:
                 chunk = readfn(2 ** 18)
                 if not chunk:
                     break
                 fptemp.write(chunk)
 
         return self.vfs.open(self.tempfile, mode=b"rb")
 
     @localrepo.unfilteredpropertycache
     def _phasecache(self):
         return bundlephasecache(self, self._phasedefaults)
 
     @localrepo.unfilteredpropertycache
     def changelog(self):
         # consume the header if it exists
         self._cgunpacker.changelogheader()
         c = bundlechangelog(self.svfs, self._cgunpacker)
         self.manstart = self._cgunpacker.tell()
         return c
 
     def _refreshchangelog(self):
         # changelog for bundle repo are not filecache, this method is not
         # applicable.
         pass
 
     @localrepo.unfilteredpropertycache
     def manifestlog(self):
         self._cgunpacker.seek(self.manstart)
         # consume the header if it exists
         self._cgunpacker.manifestheader()
         linkmapper = self.unfiltered().changelog.rev
         rootstore = bundlemanifest(self.svfs, self._cgunpacker, linkmapper)
         self.filestart = self._cgunpacker.tell()
 
         return manifest.manifestlog(
             self.svfs, self, rootstore, self.narrowmatch()
         )
 
     def _consumemanifest(self):
         """Consumes the manifest portion of the bundle, setting filestart so the
         file portion can be read."""
         self._cgunpacker.seek(self.manstart)
         self._cgunpacker.manifestheader()
         for delta in self._cgunpacker.deltaiter():
             pass
         self.filestart = self._cgunpacker.tell()
 
     @localrepo.unfilteredpropertycache
     def manstart(self):
         self.changelog
         return self.manstart
 
     @localrepo.unfilteredpropertycache
     def filestart(self):
         self.manifestlog
 
         # If filestart was not set by self.manifestlog, that means the
         # manifestlog implementation did not consume the manifests from the
         # changegroup (ex: it might be consuming trees from a separate bundle2
         # part instead). So we need to manually consume it.
         if 'filestart' not in self.__dict__:
             self._consumemanifest()
 
         return self.filestart
 
     def url(self):
         return self._url
 
     def file(self, f):
         if not self._cgfilespos:
             self._cgunpacker.seek(self.filestart)
             self._cgfilespos = _getfilestarts(self._cgunpacker)
 
         if f in self._cgfilespos:
             self._cgunpacker.seek(self._cgfilespos[f])
             linkmapper = self.unfiltered().changelog.rev
             return bundlefilelog(self.svfs, f, self._cgunpacker, linkmapper)
         else:
             return super(bundlerepository, self).file(f)
 
     def close(self):
         """Close assigned bundle file immediately."""
         self._bundlefile.close()
         if self.tempfile is not None:
             self.vfs.unlink(self.tempfile)
         if self._tempparent:
             shutil.rmtree(self._tempparent, True)
 
     def cancopy(self):
         return False
 
     def peer(self):
         return bundlepeer(self)
 
     def getcwd(self):
         return encoding.getcwd()  # always outside the repo
 
     # Check if parents exist in localrepo before setting
     def setparents(self, p1, p2=nullid):
         p1rev = self.changelog.rev(p1)
         p2rev = self.changelog.rev(p2)
         msg = _(b"setting parent to node %s that only exists in the bundle\n")
         if self.changelog.repotiprev < p1rev:
             self.ui.warn(msg % hex(p1))
         if self.changelog.repotiprev < p2rev:
             self.ui.warn(msg % hex(p2))
         return super(bundlerepository, self).setparents(p1, p2)
 
 
 def instance(ui, path, create, intents=None, createopts=None):
     if create:
         raise error.Abort(_(b'cannot create new bundle repository'))
     # internal config: bundle.mainreporoot
     parentpath = ui.config(b"bundle", b"mainreporoot")
     if not parentpath:
         # try to find the correct path to the working directory repo
         parentpath = cmdutil.findrepo(encoding.getcwd())
         if parentpath is None:
             parentpath = b''
     if parentpath:
         # Try to make the full path relative so we get a nice, short URL.
         # In particular, we don't want temp dir names in test outputs.
         cwd = encoding.getcwd()
         if parentpath == cwd:
             parentpath = b''
         else:
             cwd = pathutil.normasprefix(cwd)
             if parentpath.startswith(cwd):
                 parentpath = parentpath[len(cwd) :]
     u = util.url(path)
     path = u.localpath()
     if u.scheme == b'bundle':
         s = path.split(b"+", 1)
         if len(s) == 1:
             repopath, bundlename = parentpath, s[0]
         else:
             repopath, bundlename = s
     else:
         repopath, bundlename = parentpath, path
 
     return makebundlerepository(ui, repopath, bundlename)
 
 
 def makebundlerepository(ui, repopath, bundlepath):
     """Make a bundle repository object based on repo and bundle paths."""
     if repopath:
         url = b'bundle:%s+%s' % (util.expandpath(repopath), bundlepath)
     else:
         url = b'bundle:%s' % bundlepath
 
     # Because we can't make any guarantees about the type of the base
     # repository, we can't have a static class representing the bundle
     # repository. We also can't make any guarantees about how to even
     # call the base repository's constructor!
     #
     # So, our strategy is to go through ``localrepo.instance()`` to construct
     # a repo instance. Then, we dynamically create a new type derived from
     # both it and our ``bundlerepository`` class which overrides some
     # functionality. We then change the type of the constructed repository
     # to this new type and initialize the bundle-specific bits of it.
 
     try:
         repo = localrepo.instance(ui, repopath, create=False)
         tempparent = None
     except error.RepoError:
         tempparent = pycompat.mkdtemp()
         try:
             repo = localrepo.instance(ui, tempparent, create=True)
         except Exception:
             shutil.rmtree(tempparent)
             raise
 
     class derivedbundlerepository(bundlerepository, repo.__class__):
         pass
 
     repo.__class__ = derivedbundlerepository
     bundlerepository.__init__(repo, bundlepath, url, tempparent)
 
     return repo
 
 
 class bundletransactionmanager(object):
     def transaction(self):
         return None
 
     def close(self):
         raise NotImplementedError
 
     def release(self):
         raise NotImplementedError
 
 
 def getremotechanges(
     ui, repo, peer, onlyheads=None, bundlename=None, force=False
 ):
     """obtains a bundle of changes incoming from peer
 
     "onlyheads" restricts the returned changes to those reachable from the
       specified heads.
     "bundlename", if given, stores the bundle to this file path permanently;
       otherwise it's stored to a temp file and gets deleted again when you call
       the returned "cleanupfn".
     "force" indicates whether to proceed on unrelated repos.
 
     Returns a tuple (local, csets, cleanupfn):
 
     "local" is a local repo from which to obtain the actual incoming
       changesets; it is a bundlerepo for the obtained bundle when the
       original "peer" is remote.
     "csets" lists the incoming changeset node ids.
     "cleanupfn" must be called without arguments when you're done processing
       the changes; it closes both the original "peer" and the one returned
       here.
     """
     tmp = discovery.findcommonincoming(repo, peer, heads=onlyheads, force=force)
     common, incoming, rheads = tmp
     if not incoming:
         try:
             if bundlename:
                 os.unlink(bundlename)
         except OSError:
             pass
         return repo, [], peer.close
 
     commonset = set(common)
     rheads = [x for x in rheads if x not in commonset]
 
     bundle = None
     bundlerepo = None
     localrepo = peer.local()
     if bundlename or not localrepo:
         # create a bundle (uncompressed if peer repo is not local)
 
         # developer config: devel.legacy.exchange
         legexc = ui.configlist(b'devel', b'legacy.exchange')
         forcebundle1 = b'bundle2' not in legexc and b'bundle1' in legexc
         canbundle2 = (
             not forcebundle1
             and peer.capable(b'getbundle')
             and peer.capable(b'bundle2')
         )
         if canbundle2:
             with peer.commandexecutor() as e:
                 b2 = e.callcommand(
                     b'getbundle',
                     {
                         b'source': b'incoming',
                         b'common': common,
                         b'heads': rheads,
                         b'bundlecaps': exchange.caps20to10(
                             repo, role=b'client'
                         ),
                         b'cg': True,
                     },
                 ).result()
 
                 fname = bundle = changegroup.writechunks(
                     ui, b2._forwardchunks(), bundlename
                 )
         else:
             if peer.capable(b'getbundle'):
                 with peer.commandexecutor() as e:
                     cg = e.callcommand(
                         b'getbundle',
                         {
                             b'source': b'incoming',
                             b'common': common,
                             b'heads': rheads,
                         },
                     ).result()
             elif onlyheads is None and not peer.capable(b'changegroupsubset'):
                 # compat with older servers when pulling all remote heads
 
                 with peer.commandexecutor() as e:
                     cg = e.callcommand(
                         b'changegroup',
                         {
                             b'nodes': incoming,
                             b'source': b'incoming',
                         },
                     ).result()
 
                 rheads = None
             else:
                 with peer.commandexecutor() as e:
                     cg = e.callcommand(
                         b'changegroupsubset',
                         {
                             b'bases': incoming,
                             b'heads': rheads,
                             b'source': b'incoming',
                         },
                     ).result()
 
             if localrepo:
                 bundletype = b"HG10BZ"
             else:
                 bundletype = b"HG10UN"
             fname = bundle = bundle2.writebundle(ui, cg, bundlename, bundletype)
         # keep written bundle?
         if bundlename:
             bundle = None
         if not localrepo:
             # use the created uncompressed bundlerepo
             localrepo = bundlerepo = makebundlerepository(
                 repo.baseui, repo.root, fname
             )
 
             # this repo contains local and peer now, so filter out local again
             common = repo.heads()
     if localrepo:
         # Part of common may be remotely filtered
         # So use an unfiltered version
         # The discovery process probably need cleanup to avoid that
         localrepo = localrepo.unfiltered()
 
     csets = localrepo.changelog.findmissing(common, rheads)
 
     if bundlerepo:
         reponodes = [ctx.node() for ctx in bundlerepo[bundlerepo.firstnewrev :]]
 
         with peer.commandexecutor() as e:
             remotephases = e.callcommand(
                 b'listkeys',
                 {
                     b'namespace': b'phases',
                 },
             ).result()
 
         pullop = exchange.pulloperation(bundlerepo, peer, heads=reponodes)
         pullop.trmanager = bundletransactionmanager()
         exchange._pullapplyphases(pullop, remotephases)
 
     def cleanup():
         if bundlerepo:
             bundlerepo.close()
         if bundle:
             os.unlink(bundle)
         peer.close()
 
     return (localrepo, csets, cleanup)

mercurial/changegroup.py

@@ -1,1710 +1,1784 b''
 # changegroup.py - Mercurial changegroup manipulation functions
 #
 #  Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 import struct
 import weakref
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
     short,
 )
 from .pycompat import open
 
 from . import (
     error,
     match as matchmod,
     mdiff,
     phases,
     pycompat,
     requirements,
     scmutil,
     util,
 )
 
 from .interfaces import repository
+from .revlogutils import sidedata as sidedatamod
 
 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH")
 
 LFS_REQUIREMENT = b'lfs'
 
 readexactly = util.readexactly
 
 
 def getchunk(stream):
     """return the next chunk from stream as a string"""
     d = readexactly(stream, 4)
     l = struct.unpack(b">l", d)[0]
     if l <= 4:
         if l:
             raise error.Abort(_(b"invalid chunk length %d") % l)
         return b""
     return readexactly(stream, l - 4)
 
 
 def chunkheader(length):
     """return a changegroup chunk header (string)"""
     return struct.pack(b">l", length + 4)
 
 
 def closechunk():
     """return a changegroup chunk header (string) for a zero-length chunk"""
     return struct.pack(b">l", 0)
 
 
 def _fileheader(path):
     """Obtain a changegroup chunk header for a named path."""
     return chunkheader(len(path)) + path
 
 
 def writechunks(ui, chunks, filename, vfs=None):
     """Write chunks to a file and return its filename.
 
     The stream is assumed to be a bundle file.
     Existing files will not be overwritten.
     If no filename is specified, a temporary file is created.
     """
     fh = None
     cleanup = None
     try:
         if filename:
             if vfs:
                 fh = vfs.open(filename, b"wb")
             else:
                 # Increase default buffer size because default is usually
                 # small (4k is common on Linux).
                 fh = open(filename, b"wb", 131072)
         else:
             fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg")
             fh = os.fdopen(fd, "wb")
         cleanup = filename
         for c in chunks:
             fh.write(c)
         cleanup = None
         return filename
     finally:
         if fh is not None:
             fh.close()
         if cleanup is not None:
             if filename and vfs:
                 vfs.unlink(cleanup)
             else:
                 os.unlink(cleanup)
 
 
 class cg1unpacker(object):
     """Unpacker for cg1 changegroup streams.
 
     A changegroup unpacker handles the framing of the revision data in
     the wire format. Most consumers will want to use the apply()
     method to add the changes from the changegroup to a repository.
 
     If you're forwarding a changegroup unmodified to another consumer,
     use getchunks(), which returns an iterator of changegroup
     chunks. This is mostly useful for cases where you need to know the
     data stream has ended by observing the end of the changegroup.
 
     deltachunk() is useful only if you're applying delta data. Most
     consumers should prefer apply() instead.
 
     A few other public methods exist. Those are used only for
     bundlerepo and some debug commands - their use is discouraged.
     """
 
     deltaheader = _CHANGEGROUPV1_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = b'01'
     _grouplistcount = 1  # One list of files after the manifests
 
     def __init__(self, fh, alg, extras=None):
         if alg is None:
             alg = b'UN'
         if alg not in util.compengines.supportedbundletypes:
             raise error.Abort(_(b'unknown stream compression type: %s') % alg)
         if alg == b'BZ':
             alg = b'_truncatedBZ'
 
         compengine = util.compengines.forbundletype(alg)
         self._stream = compengine.decompressorreader(fh)
         self._type = alg
         self.extras = extras or {}
         self.callback = None
 
     # These methods (compressed, read, seek, tell) all appear to only
     # be used by bundlerepo, but it's a little hard to tell.
     def compressed(self):
         return self._type is not None and self._type != b'UN'
 
     def read(self, l):
         return self._stream.read(l)
 
     def seek(self, pos):
         return self._stream.seek(pos)
 
     def tell(self):
         return self._stream.tell()
 
     def close(self):
         return self._stream.close()
 
     def _chunklength(self):
         d = readexactly(self._stream, 4)
         l = struct.unpack(b">l", d)[0]
         if l <= 4:
             if l:
                 raise error.Abort(_(b"invalid chunk length %d") % l)
             return 0
         if self.callback:
             self.callback()
         return l - 4
 
     def changelogheader(self):
         """v10 does not have a changelog header chunk"""
         return {}
 
     def manifestheader(self):
         """v10 does not have a manifest header chunk"""
         return {}
 
     def filelogheader(self):
         """return the header of the filelogs chunk, v10 only has the filename"""
         l = self._chunklength()
         if not l:
             return {}
         fname = readexactly(self._stream, l)
         return {b'filename': fname}
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, cs = headertuple
         if prevnode is None:
             deltabase = p1
         else:
             deltabase = prevnode
         flags = 0
         return node, p1, p2, deltabase, cs, flags
 
     def deltachunk(self, prevnode):
         l = self._chunklength()
         if not l:
             return {}
         headerdata = readexactly(self._stream, self.deltaheadersize)
         header = self.deltaheader.unpack(headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
         node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
-        return (node, p1, p2, cs, deltabase, delta, flags)
+        # cg4 forward-compat
+        sidedata = {}
+        return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
 
         Used when you need to forward the binary stream to a file or another
         network API. To do so, it parse the changegroup data, otherwise it will
         block in case of sshrepo because it don't know the end of the stream.
         """
         # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
         # and a list of filelogs. For changegroup 3, we expect 4 parts:
         # changelog, manifestlog, a list of tree manifestlogs, and a list of
         # filelogs.
         #
         # Changelog and manifestlog parts are terminated with empty chunks. The
         # tree and file parts are a list of entry sections. Each entry section
         # is a series of chunks terminating in an empty chunk. The list of these
         # entry sections is terminated in yet another empty chunk, so we know
         # we've reached the end of the tree/file list when we reach an empty
         # chunk that was proceeded by no non-empty chunks.
 
         parts = 0
         while parts < 2 + self._grouplistcount:
             noentries = True
             while True:
                 chunk = getchunk(self)
                 if not chunk:
                     # The first two empty chunks represent the end of the
                     # changelog and the manifestlog portions. The remaining
                     # empty chunks represent either A) the end of individual
                     # tree or file entries in the file list, or B) the end of
                     # the entire list. It's the end of the entire list if there
                     # were no entries (i.e. noentries is True).
                     if parts < 2:
                         parts += 1
                     elif noentries:
                         parts += 1
                     break
                 noentries = False
                 yield chunkheader(len(chunk))
                 pos = 0
                 while pos < len(chunk):
                     next = pos + 2 ** 20
                     yield chunk[pos:next]
                     pos = next
             yield closechunk()
 
     def _unpackmanifests(self, repo, revmap, trp, prog):
         self.callback = prog.increment
         # no need to check for empty manifest group here:
         # if the result of the merge of 1 and 2 is the same in 3 and 4,
         # no new manifest will be created and the manifest group will
         # be empty during the pull
         self.manifestheader()
         deltas = self.deltaiter()
         repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp)
         prog.complete()
         self.callback = None
 
     def apply(
         self,
         repo,
         tr,
         srctype,
         url,
         targetphase=phases.draft,
         expectedtotal=None,
     ):
         """Add the changegroup returned by source.read() to this repo.
         srctype is a string like 'push', 'pull', or 'unbundle'.  url is
         the URL of the repo where this changegroup is coming from.
 
         Return an integer summarizing the change to this repo:
         - nothing changed or no source: 0
         - more heads than before: 1+added heads (2..n)
         - fewer heads than before: -1-removed heads (-2..-n)
         - number of heads stays the same: 1
         """
         repo = repo.unfiltered()
 
         def csmap(x):
             repo.ui.debug(b"add changeset %s\n" % short(x))
             return len(cl)
 
         def revmap(x):
             return cl.rev(x)
 
         try:
             # The transaction may already carry source information. In this
             # case we use the top level data. We overwrite the argument
             # because we need to use the top level value (if they exist)
             # in this function.
             srctype = tr.hookargs.setdefault(b'source', srctype)
             tr.hookargs.setdefault(b'url', url)
             repo.hook(
                 b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs)
             )
 
             # write changelog data to temp files so concurrent readers
             # will not see an inconsistent view
             cl = repo.changelog
             cl.delayupdate(tr)
             oldheads = set(cl.heads())
 
             trp = weakref.proxy(tr)
             # pull off the changeset group
             repo.ui.status(_(b"adding changesets\n"))
             clstart = len(cl)
             progress = repo.ui.makeprogress(
                 _(b'changesets'), unit=_(b'chunks'), total=expectedtotal
             )
             self.callback = progress.increment
 
             efilesset = set()
             duprevs = []
 
             def ondupchangelog(cl, rev):
                 if rev < clstart:
                     duprevs.append(rev)
 
             def onchangelog(cl, rev):
                 ctx = cl.changelogrevision(rev)
                 efilesset.update(ctx.files)
                 repo.register_changeset(rev, ctx)
 
             self.changelogheader()
             deltas = self.deltaiter()
             if not cl.addgroup(
                 deltas,
                 csmap,
                 trp,
                 alwayscache=True,
                 addrevisioncb=onchangelog,
                 duplicaterevisioncb=ondupchangelog,
             ):
                 repo.ui.develwarn(
                     b'applied empty changelog from changegroup',
                     config=b'warn-empty-changegroup',
                 )
             efiles = len(efilesset)
             clend = len(cl)
             changesets = clend - clstart
             progress.complete()
             del deltas
             # TODO Python 2.7 removal
             # del efilesset
             efilesset = None
             self.callback = None
 
             # pull off the manifest group
             repo.ui.status(_(b"adding manifests\n"))
             # We know that we'll never have more manifests than we had
             # changesets.
             progress = repo.ui.makeprogress(
                 _(b'manifests'), unit=_(b'chunks'), total=changesets
             )
             self._unpackmanifests(repo, revmap, trp, progress)
 
             needfiles = {}
             if repo.ui.configbool(b'server', b'validate'):
                 cl = repo.changelog
                 ml = repo.manifestlog
                 # validate incoming csets have their manifests
                 for cset in pycompat.xrange(clstart, clend):
                     mfnode = cl.changelogrevision(cset).manifest
                     mfest = ml[mfnode].readdelta()
                     # store file nodes we must see
                     for f, n in pycompat.iteritems(mfest):
                         needfiles.setdefault(f, set()).add(n)
 
             # process the files
             repo.ui.status(_(b"adding file changes\n"))
             newrevs, newfiles = _addchangegroupfiles(
                 repo, self, revmap, trp, efiles, needfiles
             )
 
             # making sure the value exists
             tr.changes.setdefault(b'changegroup-count-changesets', 0)
             tr.changes.setdefault(b'changegroup-count-revisions', 0)
             tr.changes.setdefault(b'changegroup-count-files', 0)
             tr.changes.setdefault(b'changegroup-count-heads', 0)
 
             # some code use bundle operation for internal purpose. They usually
             # set `ui.quiet` to do this outside of user sight. Size the report
             # of such operation now happens at the end of the transaction, that
             # ui.quiet has not direct effect on the output.
             #
             # To preserve this intend use an inelegant hack, we fail to report
             # the change if `quiet` is set. We should probably move to
             # something better, but this is a good first step to allow the "end
             # of transaction report" to pass tests.
             if not repo.ui.quiet:
                 tr.changes[b'changegroup-count-changesets'] += changesets
                 tr.changes[b'changegroup-count-revisions'] += newrevs
                 tr.changes[b'changegroup-count-files'] += newfiles
 
             deltaheads = 0
             if oldheads:
                 heads = cl.heads()
                 deltaheads += len(heads) - len(oldheads)
                 for h in heads:
                     if h not in oldheads and repo[h].closesbranch():
                         deltaheads -= 1
 
             # see previous comment about checking ui.quiet
             if not repo.ui.quiet:
                 tr.changes[b'changegroup-count-heads'] += deltaheads
             repo.invalidatevolatilesets()
 
             if changesets > 0:
                 if b'node' not in tr.hookargs:
                     tr.hookargs[b'node'] = hex(cl.node(clstart))
                     tr.hookargs[b'node_last'] = hex(cl.node(clend - 1))
                     hookargs = dict(tr.hookargs)
                 else:
                     hookargs = dict(tr.hookargs)
                     hookargs[b'node'] = hex(cl.node(clstart))
                     hookargs[b'node_last'] = hex(cl.node(clend - 1))
                 repo.hook(
                     b'pretxnchangegroup',
                     throw=True,
                     **pycompat.strkwargs(hookargs)
                 )
 
             added = pycompat.xrange(clstart, clend)
             phaseall = None
             if srctype in (b'push', b'serve'):
                 # Old servers can not push the boundary themselves.
                 # New servers won't push the boundary if changeset already
                 # exists locally as secret
                 #
                 # We should not use added here but the list of all change in
                 # the bundle
                 if repo.publishing():
                     targetphase = phaseall = phases.public
                 else:
                     # closer target phase computation
 
                     # Those changesets have been pushed from the
                     # outside, their phases are going to be pushed
                     # alongside. Therefor `targetphase` is
                     # ignored.
                     targetphase = phaseall = phases.draft
             if added:
                 phases.registernew(repo, tr, targetphase, added)
             if phaseall is not None:
                 if duprevs:
                     duprevs.extend(added)
                 else:
                     duprevs = added
                 phases.advanceboundary(repo, tr, phaseall, [], revs=duprevs)
                 duprevs = []
 
             if changesets > 0:
 
                 def runhooks(unused_success):
                     # These hooks run when the lock releases, not when the
                     # transaction closes. So it's possible for the changelog
                     # to have changed since we last saw it.
                     if clstart >= len(repo):
                         return
 
                     repo.hook(b"changegroup", **pycompat.strkwargs(hookargs))
 
                     for rev in added:
                         args = hookargs.copy()
                         args[b'node'] = hex(cl.node(rev))
                         del args[b'node_last']
                         repo.hook(b"incoming", **pycompat.strkwargs(args))
 
                     newheads = [h for h in repo.heads() if h not in oldheads]
                     repo.ui.log(
                         b"incoming",
                         b"%d incoming changes - new heads: %s\n",
                         len(added),
                         b', '.join([hex(c[:6]) for c in newheads]),
                     )
 
                 tr.addpostclose(
                     b'changegroup-runhooks-%020i' % clstart,
                     lambda tr: repo._afterlock(runhooks),
                 )
         finally:
             repo.ui.flush()
         # never return 0 here:
         if deltaheads < 0:
             ret = deltaheads - 1
         else:
             ret = deltaheads + 1
         return ret
 
     def deltaiter(self):
         """
         returns an iterator of the deltas in this changegroup
 
         Useful for passing to the underlying storage system to be stored.
         """
         chain = None
         for chunkdata in iter(lambda: self.deltachunk(chain), {}):
             # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
             yield chunkdata
             chain = chunkdata[0]
 
 
 class cg2unpacker(cg1unpacker):
     """Unpacker for cg2 streams.
 
     cg2 streams add support for generaldelta, so the delta header
     format is slightly different. All other features about the data
     remain the same.
     """
 
     deltaheader = _CHANGEGROUPV2_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = b'02'
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs = headertuple
         flags = 0
         return node, p1, p2, deltabase, cs, flags
 
 
 class cg3unpacker(cg2unpacker):
     """Unpacker for cg3 streams.
 
     cg3 streams add support for exchanging treemanifests and revlog
     flags. It adds the revlog flags to the delta header and an empty chunk
     separating manifests and files.
     """
 
     deltaheader = _CHANGEGROUPV3_DELTA_HEADER
     deltaheadersize = deltaheader.size
     version = b'03'
     _grouplistcount = 2  # One list of manifests and one list of files
 
     def _deltaheader(self, headertuple, prevnode):
         node, p1, p2, deltabase, cs, flags = headertuple
         return node, p1, p2, deltabase, cs, flags
 
     def _unpackmanifests(self, repo, revmap, trp, prog):
         super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
         for chunkdata in iter(self.filelogheader, {}):
             # If we get here, there are directory manifests in the changegroup
             d = chunkdata[b"filename"]
             repo.ui.debug(b"adding %s revisions\n" % d)
             deltas = self.deltaiter()
             if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp):
                 raise error.Abort(_(b"received dir revlog group is empty"))
 
 
+class cg4unpacker(cg3unpacker):
+    """Unpacker for cg4 streams.
+
+    cg4 streams add support for exchanging sidedata.
+    """
+
+    version = b'04'
+
+    def deltachunk(self, prevnode):
+        res = super(cg4unpacker, self).deltachunk(prevnode)
+        if not res:
+            return res
+
+        (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
+
+        sidedata_raw = getchunk(self._stream)
+        sidedata = {}
+        if len(sidedata_raw) > 0:
+            sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
+
+        return node, p1, p2, cs, deltabase, delta, flags, sidedata
+
+
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
         self._fh = fh
 
     def read(self, n):
         if self._h:
             d, self._h = self._h[:n], self._h[n:]
             if len(d) < n:
                 d += readexactly(self._fh, n - len(d))
             return d
         return readexactly(self._fh, n)
 
 
 def _revisiondeltatochunks(delta, headerfn):
     """Serialize a revisiondelta to changegroup chunks."""
 
     # The captured revision delta may be encoded as a delta against
     # a base revision or as a full revision. The changegroup format
     # requires that everything on the wire be deltas. So for full
     # revisions, we need to invent a header that says to rewrite
     # data.
 
     if delta.delta is not None:
         prefix, data = b'', delta.delta
     elif delta.basenode == nullid:
         data = delta.revision
         prefix = mdiff.trivialdiffheader(len(data))
     else:
         data = delta.revision
         prefix = mdiff.replacediffheader(delta.baserevisionsize, len(data))
 
     meta = headerfn(delta)
 
     yield chunkheader(len(meta) + len(prefix) + len(data))
     yield meta
     if prefix:
         yield prefix
     yield data
 
 
 def _sortnodesellipsis(store, nodes, cl, lookup):
     """Sort nodes for changegroup generation."""
     # Ellipses serving mode.
     #
     # In a perfect world, we'd generate better ellipsis-ified graphs
     # for non-changelog revlogs. In practice, we haven't started doing
     # that yet, so the resulting DAGs for the manifestlog and filelogs
     # are actually full of bogus parentage on all the ellipsis
     # nodes. This has the side effect that, while the contents are
     # correct, the individual DAGs might be completely out of whack in
     # a case like 882681bc3166 and its ancestors (back about 10
     # revisions or so) in the main hg repo.
     #
     # The one invariant we *know* holds is that the new (potentially
     # bogus) DAG shape will be valid if we order the nodes in the
     # order that they're introduced in dramatis personae by the
     # changelog, so what we do is we sort the non-changelog histories
     # by the order in which they are used by the changelog.
     key = lambda n: cl.rev(lookup(n))
     return sorted(nodes, key=key)
 
 
 def _resolvenarrowrevisioninfo(
     cl,
     store,
     ischangelog,
     rev,
     linkrev,
     linknode,
     clrevtolocalrev,
     fullclnodes,
     precomputedellipsis,
 ):
     linkparents = precomputedellipsis[linkrev]
 
     def local(clrev):
         """Turn a changelog revnum into a local revnum.
 
         The ellipsis dag is stored as revnums on the changelog,
         but when we're producing ellipsis entries for
         non-changelog revlogs, we need to turn those numbers into
         something local. This does that for us, and during the
         changelog sending phase will also expand the stored
         mappings as needed.
         """
         if clrev == nullrev:
             return nullrev
 
         if ischangelog:
             return clrev
 
         # Walk the ellipsis-ized changelog breadth-first looking for a
         # change that has been linked from the current revlog.
         #
         # For a flat manifest revlog only a single step should be necessary
         # as all relevant changelog entries are relevant to the flat
         # manifest.
         #
         # For a filelog or tree manifest dirlog however not every changelog
         # entry will have been relevant, so we need to skip some changelog
         # nodes even after ellipsis-izing.
         walk = [clrev]
         while walk:
             p = walk[0]
             walk = walk[1:]
             if p in clrevtolocalrev:
                 return clrevtolocalrev[p]
             elif p in fullclnodes:
                 walk.extend([pp for pp in cl.parentrevs(p) if pp != nullrev])
             elif p in precomputedellipsis:
                 walk.extend(
                     [pp for pp in precomputedellipsis[p] if pp != nullrev]
                 )
             else:
                 # In this case, we've got an ellipsis with parents
                 # outside the current bundle (likely an
                 # incremental pull). We "know" that we can use the
                 # value of this same revlog at whatever revision
                 # is pointed to by linknode. "Know" is in scare
                 # quotes because I haven't done enough examination
                 # of edge cases to convince myself this is really
                 # a fact - it works for all the (admittedly
                 # thorough) cases in our testsuite, but I would be
                 # somewhat unsurprised to find a case in the wild
                 # where this breaks down a bit. That said, I don't
                 # know if it would hurt anything.
                 for i in pycompat.xrange(rev, 0, -1):
                     if store.linkrev(i) == clrev:
                         return i
                 # We failed to resolve a parent for this node, so
                 # we crash the changegroup construction.
                 raise error.Abort(
                     b"unable to resolve parent while packing '%s' %r"
                     b' for changeset %r' % (store.indexfile, rev, clrev)
                 )
 
         return nullrev
 
     if not linkparents or (store.parentrevs(rev) == (nullrev, nullrev)):
         p1, p2 = nullrev, nullrev
     elif len(linkparents) == 1:
         (p1,) = sorted(local(p) for p in linkparents)
         p2 = nullrev
     else:
         p1, p2 = sorted(local(p) for p in linkparents)
 
     p1node, p2node = store.node(p1), store.node(p2)
 
     return p1node, p2node, linknode
 
 
 def deltagroup(
     repo,
     store,
     nodes,
     ischangelog,
     lookup,
     forcedeltaparentprev,
     topic=None,
     ellipses=False,
     clrevtolocalrev=None,
     fullclnodes=None,
     precomputedellipsis=None,
 ):
     """Calculate deltas for a set of revisions.
 
     Is a generator of ``revisiondelta`` instances.
 
     If topic is not None, progress detail will be generated using this
     topic name (e.g. changesets, manifests, etc).
     """
     if not nodes:
         return
 
     cl = repo.changelog
 
     if ischangelog:
         # `hg log` shows changesets in storage order. To preserve order
         # across clones, send out changesets in storage order.
         nodesorder = b'storage'
     elif ellipses:
         nodes = _sortnodesellipsis(store, nodes, cl, lookup)
         nodesorder = b'nodes'
     else:
         nodesorder = None
 
     # Perform ellipses filtering and revision massaging. We do this before
     # emitrevisions() because a) filtering out revisions creates less work
     # for emitrevisions() b) dropping revisions would break emitrevisions()'s
     # assumptions about delta choices and we would possibly send a delta
     # referencing a missing base revision.
     #
     # Also, calling lookup() has side-effects with regards to populating
     # data structures. If we don't call lookup() for each node or if we call
     # lookup() after the first pass through each node, things can break -
     # possibly intermittently depending on the python hash seed! For that
     # reason, we store a mapping of all linknodes during the initial node
     # pass rather than use lookup() on the output side.
     if ellipses:
         filtered = []
         adjustedparents = {}
         linknodes = {}
 
         for node in nodes:
             rev = store.rev(node)
             linknode = lookup(node)
             linkrev = cl.rev(linknode)
             clrevtolocalrev[linkrev] = rev
 
             # If linknode is in fullclnodes, it means the corresponding
             # changeset was a full changeset and is being sent unaltered.
             if linknode in fullclnodes:
                 linknodes[node] = linknode
 
             # If the corresponding changeset wasn't in the set computed
             # as relevant to us, it should be dropped outright.
             elif linkrev not in precomputedellipsis:
                 continue
 
             else:
                 # We could probably do this later and avoid the dict
                 # holding state. But it likely doesn't matter.
                 p1node, p2node, linknode = _resolvenarrowrevisioninfo(
                     cl,
                     store,
                     ischangelog,
                     rev,
                     linkrev,
                     linknode,
                     clrevtolocalrev,
                     fullclnodes,
                     precomputedellipsis,
                 )
 
                 adjustedparents[node] = (p1node, p2node)
                 linknodes[node] = linknode
 
             filtered.append(node)
 
         nodes = filtered
 
     # We expect the first pass to be fast, so we only engage the progress
     # meter for constructing the revision deltas.
     progress = None
     if topic is not None:
         progress = repo.ui.makeprogress(
             topic, unit=_(b'chunks'), total=len(nodes)
         )
 
     configtarget = repo.ui.config(b'devel', b'bundle.delta')
     if configtarget not in (b'', b'p1', b'full'):
         msg = _("""config "devel.bundle.delta" as unknown value: %s""")
         repo.ui.warn(msg % configtarget)
 
     deltamode = repository.CG_DELTAMODE_STD
     if forcedeltaparentprev:
         deltamode = repository.CG_DELTAMODE_PREV
     elif configtarget == b'p1':
         deltamode = repository.CG_DELTAMODE_P1
     elif configtarget == b'full':
         deltamode = repository.CG_DELTAMODE_FULL
 
     revisions = store.emitrevisions(
         nodes,
         nodesorder=nodesorder,
         revisiondata=True,
         assumehaveparentrevisions=not ellipses,
         deltamode=deltamode,
     )
 
     for i, revision in enumerate(revisions):
         if progress:
             progress.update(i + 1)
 
         if ellipses:
             linknode = linknodes[revision.node]
 
             if revision.node in adjustedparents:
                 p1node, p2node = adjustedparents[revision.node]
                 revision.p1node = p1node
                 revision.p2node = p2node
                 revision.flags |= repository.REVISION_FLAG_ELLIPSIS
 
         else:
             linknode = lookup(revision.node)
 
         revision.linknode = linknode
         yield revision
 
     if progress:
         progress.complete()
 
 
 class cgpacker(object):
     def __init__(
         self,
         repo,
         oldmatcher,
         matcher,
         version,
         builddeltaheader,
         manifestsend,
         forcedeltaparentprev=False,
         bundlecaps=None,
         ellipses=False,
         shallow=False,
         ellipsisroots=None,
         fullnodes=None,
+        remote_sidedata=None,
     ):
         """Given a source repo, construct a bundler.
 
         oldmatcher is a matcher that matches on files the client already has.
         These will not be included in the changegroup.
 
         matcher is a matcher that matches on files to include in the
         changegroup. Used to facilitate sparse changegroups.
 
         forcedeltaparentprev indicates whether delta parents must be against
         the previous revision in a delta group. This should only be used for
         compatibility with changegroup version 1.
 
         builddeltaheader is a callable that constructs the header for a group
         delta.
 
         manifestsend is a chunk to send after manifests have been fully emitted.
 
         ellipses indicates whether ellipsis serving mode is enabled.
 
         bundlecaps is optional and can be used to specify the set of
         capabilities which can be used to build the bundle. While bundlecaps is
         unused in core Mercurial, extensions rely on this feature to communicate
         capabilities to customize the changegroup packer.
 
         shallow indicates whether shallow data might be sent. The packer may
         need to pack file contents not introduced by the changes being packed.
 
         fullnodes is the set of changelog nodes which should not be ellipsis
         nodes. We store this rather than the set of nodes that should be
         ellipsis because for very large histories we expect this to be
         significantly smaller.
+
+        remote_sidedata is the set of sidedata categories wanted by the remote.
         """
         assert oldmatcher
         assert matcher
         self._oldmatcher = oldmatcher
         self._matcher = matcher
 
         self.version = version
         self._forcedeltaparentprev = forcedeltaparentprev
         self._builddeltaheader = builddeltaheader
         self._manifestsend = manifestsend
         self._ellipses = ellipses
 
         # Set of capabilities we can use to build the bundle.
         if bundlecaps is None:
             bundlecaps = set()
         self._bundlecaps = bundlecaps
         self._isshallow = shallow
         self._fullclnodes = fullnodes
 
         # Maps ellipsis revs to their roots at the changelog level.
         self._precomputedellipsis = ellipsisroots
 
         self._repo = repo
 
         if self._repo.ui.verbose and not self._repo.ui.debugflag:
             self._verbosenote = self._repo.ui.note
         else:
             self._verbosenote = lambda s: None
 
     def generate(
         self, commonrevs, clnodes, fastpathlinkrev, source, changelog=True
     ):
         """Yield a sequence of changegroup byte chunks.
         If changelog is False, changelog data won't be added to changegroup
         """
 
         repo = self._repo
         cl = repo.changelog
 
         self._verbosenote(_(b'uncompressed size of bundle content:\n'))
         size = 0
 
         clstate, deltas = self._generatechangelog(
             cl, clnodes, generate=changelog
         )
         for delta in deltas:
             for chunk in _revisiondeltatochunks(delta, self._builddeltaheader):
                 size += len(chunk)
                 yield chunk
 
         close = closechunk()
         size += len(close)
         yield closechunk()
 
         self._verbosenote(_(b'%8.i (changelog)\n') % size)
 
         clrevorder = clstate[b'clrevorder']
         manifests = clstate[b'manifests']
         changedfiles = clstate[b'changedfiles']
 
         # We need to make sure that the linkrev in the changegroup refers to
         # the first changeset that introduced the manifest or file revision.
         # The fastpath is usually safer than the slowpath, because the filelogs
         # are walked in revlog order.
         #
         # When taking the slowpath when the manifest revlog uses generaldelta,
         # the manifest may be walked in the "wrong" order. Without 'clrevorder',
         # we would get an incorrect linkrev (see fix in cc0ff93d0c0c).
         #
         # When taking the fastpath, we are only vulnerable to reordering
         # of the changelog itself. The changelog never uses generaldelta and is
         # never reordered. To handle this case, we simply take the slowpath,
         # which already has the 'clrevorder' logic. This was also fixed in
         # cc0ff93d0c0c.
 
         # Treemanifests don't work correctly with fastpathlinkrev
         # either, because we don't discover which directory nodes to
         # send along with files. This could probably be fixed.
         fastpathlinkrev = fastpathlinkrev and not scmutil.istreemanifest(repo)
 
         fnodes = {}  # needed file nodes
 
         size = 0
         it = self.generatemanifests(
             commonrevs,
             clrevorder,
             fastpathlinkrev,
             manifests,
             fnodes,
             source,
             clstate[b'clrevtomanifestrev'],
         )
 
         for tree, deltas in it:
             if tree:
-                assert self.version == b'03'
+                assert self.version in (b'03', b'04')
                 chunk = _fileheader(tree)
                 size += len(chunk)
                 yield chunk
 
             for delta in deltas:
                 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
                 for chunk in chunks:
                     size += len(chunk)
                     yield chunk
 
             close = closechunk()
             size += len(close)
             yield close
 
         self._verbosenote(_(b'%8.i (manifests)\n') % size)
         yield self._manifestsend
 
         mfdicts = None
         if self._ellipses and self._isshallow:
             mfdicts = [
                 (repo.manifestlog[n].read(), lr)
                 for (n, lr) in pycompat.iteritems(manifests)
             ]
 
         manifests.clear()
         clrevs = {cl.rev(x) for x in clnodes}
 
         it = self.generatefiles(
             changedfiles,
             commonrevs,
             source,
             mfdicts,
             fastpathlinkrev,
             fnodes,
             clrevs,
         )
 
         for path, deltas in it:
             h = _fileheader(path)
             size = len(h)
             yield h
 
             for delta in deltas:
                 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
                 for chunk in chunks:
                     size += len(chunk)
                     yield chunk
 
             close = closechunk()
             size += len(close)
             yield close
 
             self._verbosenote(_(b'%8.i  %s\n') % (size, path))
 
         yield closechunk()
 
         if clnodes:
             repo.hook(b'outgoing', node=hex(clnodes[0]), source=source)
 
     def _generatechangelog(self, cl, nodes, generate=True):
         """Generate data for changelog chunks.
 
         Returns a 2-tuple of a dict containing state and an iterable of
         byte chunks. The state will not be fully populated until the
         chunk stream has been fully consumed.
 
         if generate is False, the state will be fully populated and no chunk
         stream will be yielded
         """
         clrevorder = {}
         manifests = {}
         mfl = self._repo.manifestlog
         changedfiles = set()
         clrevtomanifestrev = {}
 
         state = {
             b'clrevorder': clrevorder,
             b'manifests': manifests,
             b'changedfiles': changedfiles,
             b'clrevtomanifestrev': clrevtomanifestrev,
         }
 
         if not (generate or self._ellipses):
             # sort the nodes in storage order
             nodes = sorted(nodes, key=cl.rev)
             for node in nodes:
                 c = cl.changelogrevision(node)
                 clrevorder[node] = len(clrevorder)
                 # record the first changeset introducing this manifest version
                 manifests.setdefault(c.manifest, node)
                 # Record a complete list of potentially-changed files in
                 # this manifest.
                 changedfiles.update(c.files)
 
             return state, ()
 
         # Callback for the changelog, used to collect changed files and
         # manifest nodes.
         # Returns the linkrev node (identity in the changelog case).
         def lookupcl(x):
             c = cl.changelogrevision(x)
             clrevorder[x] = len(clrevorder)
 
             if self._ellipses:
                 # Only update manifests if x is going to be sent. Otherwise we
                 # end up with bogus linkrevs specified for manifests and
                 # we skip some manifest nodes that we should otherwise
                 # have sent.
                 if (
                     x in self._fullclnodes
                     or cl.rev(x) in self._precomputedellipsis
                 ):
 
                     manifestnode = c.manifest
                     # Record the first changeset introducing this manifest
                     # version.
                     manifests.setdefault(manifestnode, x)
                     # Set this narrow-specific dict so we have the lowest
                     # manifest revnum to look up for this cl revnum. (Part of
                     # mapping changelog ellipsis parents to manifest ellipsis
                     # parents)
                     clrevtomanifestrev.setdefault(
                         cl.rev(x), mfl.rev(manifestnode)
                     )
                 # We can't trust the changed files list in the changeset if the
                 # client requested a shallow clone.
                 if self._isshallow:
                     changedfiles.update(mfl[c.manifest].read().keys())
                 else:
                     changedfiles.update(c.files)
             else:
                 # record the first changeset introducing this manifest version
                 manifests.setdefault(c.manifest, x)
                 # Record a complete list of potentially-changed files in
                 # this manifest.
                 changedfiles.update(c.files)
 
             return x
 
         gen = deltagroup(
             self._repo,
             cl,
             nodes,
             True,
             lookupcl,
             self._forcedeltaparentprev,
             ellipses=self._ellipses,
             topic=_(b'changesets'),
             clrevtolocalrev={},
             fullclnodes=self._fullclnodes,
             precomputedellipsis=self._precomputedellipsis,
         )
 
         return state, gen
 
     def generatemanifests(
         self,
         commonrevs,
         clrevorder,
         fastpathlinkrev,
         manifests,
         fnodes,
         source,
         clrevtolocalrev,
     ):
         """Returns an iterator of changegroup chunks containing manifests.
 
         `source` is unused here, but is used by extensions like remotefilelog to
         change what is sent based in pulls vs pushes, etc.
         """
         repo = self._repo
         mfl = repo.manifestlog
         tmfnodes = {b'': manifests}
 
         # Callback for the manifest, used to collect linkrevs for filelog
         # revisions.
         # Returns the linkrev node (collected in lookupcl).
         def makelookupmflinknode(tree, nodes):
             if fastpathlinkrev:
                 assert not tree
                 return (
                     manifests.__getitem__
                 )  # pytype: disable=unsupported-operands
 
             def lookupmflinknode(x):
                 """Callback for looking up the linknode for manifests.
 
                 Returns the linkrev node for the specified manifest.
 
                 SIDE EFFECT:
 
                 1) fclnodes gets populated with the list of relevant
                    file nodes if we're not using fastpathlinkrev
                 2) When treemanifests are in use, collects treemanifest nodes
                    to send
 
                 Note that this means manifests must be completely sent to
                 the client before you can trust the list of files and
                 treemanifests to send.
                 """
                 clnode = nodes[x]
                 mdata = mfl.get(tree, x).readfast(shallow=True)
                 for p, n, fl in mdata.iterentries():
                     if fl == b't':  # subdirectory manifest
                         subtree = tree + p + b'/'
                         tmfclnodes = tmfnodes.setdefault(subtree, {})
                         tmfclnode = tmfclnodes.setdefault(n, clnode)
                         if clrevorder[clnode] < clrevorder[tmfclnode]:
                             tmfclnodes[n] = clnode
                     else:
                         f = tree + p
                         fclnodes = fnodes.setdefault(f, {})
                         fclnode = fclnodes.setdefault(n, clnode)
                         if clrevorder[clnode] < clrevorder[fclnode]:
                             fclnodes[n] = clnode
                 return clnode
 
             return lookupmflinknode
 
         while tmfnodes:
             tree, nodes = tmfnodes.popitem()
 
             should_visit = self._matcher.visitdir(tree[:-1])
             if tree and not should_visit:
                 continue
 
             store = mfl.getstorage(tree)
 
             if not should_visit:
                 # No nodes to send because this directory is out of
                 # the client's view of the repository (probably
                 # because of narrow clones). Do this even for the root
                 # directory (tree=='')
                 prunednodes = []
             else:
                 # Avoid sending any manifest nodes we can prove the
                 # client already has by checking linkrevs. See the
                 # related comment in generatefiles().
                 prunednodes = self._prunemanifests(store, nodes, commonrevs)
 
             if tree and not prunednodes:
                 continue
 
             lookupfn = makelookupmflinknode(tree, nodes)
 
             deltas = deltagroup(
                 self._repo,
                 store,
                 prunednodes,
                 False,
                 lookupfn,
                 self._forcedeltaparentprev,
                 ellipses=self._ellipses,
                 topic=_(b'manifests'),
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis,
             )
 
             if not self._oldmatcher.visitdir(store.tree[:-1]):
                 yield tree, deltas
             else:
                 # 'deltas' is a generator and we need to consume it even if
                 # we are not going to send it because a side-effect is that
                 # it updates tmdnodes (via lookupfn)
                 for d in deltas:
                     pass
                 if not tree:
                     yield tree, []
 
     def _prunemanifests(self, store, nodes, commonrevs):
         if not self._ellipses:
             # In non-ellipses case and large repositories, it is better to
             # prevent calling of store.rev and store.linkrev on a lot of
             # nodes as compared to sending some extra data
             return nodes.copy()
         # This is split out as a separate method to allow filtering
         # commonrevs in extension code.
         #
         # TODO(augie): this shouldn't be required, instead we should
         # make filtering of revisions to send delegated to the store
         # layer.
         frev, flr = store.rev, store.linkrev
         return [n for n in nodes if flr(frev(n)) not in commonrevs]
 
     # The 'source' parameter is useful for extensions
     def generatefiles(
         self,
         changedfiles,
         commonrevs,
         source,
         mfdicts,
         fastpathlinkrev,
         fnodes,
         clrevs,
     ):
         changedfiles = [
             f
             for f in changedfiles
             if self._matcher(f) and not self._oldmatcher(f)
         ]
 
         if not fastpathlinkrev:
 
             def normallinknodes(unused, fname):
                 return fnodes.get(fname, {})
 
         else:
             cln = self._repo.changelog.node
 
             def normallinknodes(store, fname):
                 flinkrev = store.linkrev
                 fnode = store.node
                 revs = ((r, flinkrev(r)) for r in store)
                 return {fnode(r): cln(lr) for r, lr in revs if lr in clrevs}
 
         clrevtolocalrev = {}
 
         if self._isshallow:
             # In a shallow clone, the linknodes callback needs to also include
             # those file nodes that are in the manifests we sent but weren't
             # introduced by those manifests.
             commonctxs = [self._repo[c] for c in commonrevs]
             clrev = self._repo.changelog.rev
 
             def linknodes(flog, fname):
                 for c in commonctxs:
                     try:
                         fnode = c.filenode(fname)
                         clrevtolocalrev[c.rev()] = flog.rev(fnode)
                     except error.ManifestLookupError:
                         pass
                 links = normallinknodes(flog, fname)
                 if len(links) != len(mfdicts):
                     for mf, lr in mfdicts:
                         fnode = mf.get(fname, None)
                         if fnode in links:
                             links[fnode] = min(links[fnode], lr, key=clrev)
                         elif fnode:
                             links[fnode] = lr
                 return links
 
         else:
             linknodes = normallinknodes
 
         repo = self._repo
         progress = repo.ui.makeprogress(
             _(b'files'), unit=_(b'files'), total=len(changedfiles)
         )
         for i, fname in enumerate(sorted(changedfiles)):
             filerevlog = repo.file(fname)
             if not filerevlog:
                 raise error.Abort(
                     _(b"empty or missing file data for %s") % fname
                 )
 
             clrevtolocalrev.clear()
 
             linkrevnodes = linknodes(filerevlog, fname)
             # Lookup for filenodes, we collected the linkrev nodes above in the
             # fastpath case and with lookupmf in the slowpath case.
             def lookupfilelog(x):
                 return linkrevnodes[x]
 
             frev, flr = filerevlog.rev, filerevlog.linkrev
             # Skip sending any filenode we know the client already
             # has. This avoids over-sending files relatively
             # inexpensively, so it's not a problem if we under-filter
             # here.
             filenodes = [
                 n for n in linkrevnodes if flr(frev(n)) not in commonrevs
             ]
 
             if not filenodes:
                 continue
 
             progress.update(i + 1, item=fname)
 
             deltas = deltagroup(
                 self._repo,
                 filerevlog,
                 filenodes,
                 False,
                 lookupfilelog,
                 self._forcedeltaparentprev,
                 ellipses=self._ellipses,
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis,
             )
 
             yield fname, deltas
 
         progress.complete()
 
 
 def _makecg1packer(
     repo,
     oldmatcher,
     matcher,
     bundlecaps,
     ellipses=False,
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.linknode
     )
 
     return cgpacker(
         repo,
         oldmatcher,
         matcher,
         b'01',
         builddeltaheader=builddeltaheader,
         manifestsend=b'',
         forcedeltaparentprev=True,
         bundlecaps=bundlecaps,
         ellipses=ellipses,
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
     )
 
 
 def _makecg2packer(
     repo,
     oldmatcher,
     matcher,
     bundlecaps,
     ellipses=False,
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode
     )
 
     return cgpacker(
         repo,
         oldmatcher,
         matcher,
         b'02',
         builddeltaheader=builddeltaheader,
         manifestsend=b'',
         bundlecaps=bundlecaps,
         ellipses=ellipses,
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
     )
 
 
 def _makecg3packer(
     repo,
     oldmatcher,
     matcher,
     bundlecaps,
     ellipses=False,
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
     )
 
     return cgpacker(
         repo,
         oldmatcher,
         matcher,
         b'03',
         builddeltaheader=builddeltaheader,
         manifestsend=closechunk(),
         bundlecaps=bundlecaps,
         ellipses=ellipses,
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
     )
 
 
+def _makecg4packer(
+    repo,
+    oldmatcher,
+    matcher,
+    bundlecaps,
+    ellipses=False,
+    shallow=False,
+    ellipsisroots=None,
+    fullnodes=None,
+    remote_sidedata=None,
+):
+    # Same header func as cg3. Sidedata is in a separate chunk from the delta to
+    # differenciate "raw delta" and sidedata.
+    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
+        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
+    )
+
+    return cgpacker(
+        repo,
+        oldmatcher,
+        matcher,
+        b'04',
+        builddeltaheader=builddeltaheader,
+        manifestsend=closechunk(),
+        bundlecaps=bundlecaps,
+        ellipses=ellipses,
+        shallow=shallow,
+        ellipsisroots=ellipsisroots,
+        fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
+    )
+
+
 _packermap = {
     b'01': (_makecg1packer, cg1unpacker),
     # cg2 adds support for exchanging generaldelta
     b'02': (_makecg2packer, cg2unpacker),
     # cg3 adds support for exchanging revlog flags and treemanifests
     b'03': (_makecg3packer, cg3unpacker),
+    # ch4 adds support for exchanging sidedata
+    b'04': (_makecg4packer, cg4unpacker),
 }
 
 
 def allsupportedversions(repo):
     versions = set(_packermap.keys())
     needv03 = False
     if (
         repo.ui.configbool(b'experimental', b'changegroup3')
         or repo.ui.configbool(b'experimental', b'treemanifest')
         or scmutil.istreemanifest(repo)
     ):
         # we keep version 03 because we need to to exchange treemanifest data
         #
         # we also keep vresion 01 and 02, because it is possible for repo to
         # contains both normal and tree manifest at the same time. so using
         # older version to pull data is viable
         #
         # (or even to push subset of history)
         needv03 = True
-    if b'exp-sidedata-flag' in repo.requirements:
-        needv03 = True
-        # don't attempt to use 01/02 until we do sidedata cleaning
-        versions.discard(b'01')
-        versions.discard(b'02')
+    has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
+    if not has_revlogv2:
+        versions.discard(b'04')
     if not needv03:
         versions.discard(b'03')
     return versions
 
 
 # Changegroup versions that can be applied to the repo
 def supportedincomingversions(repo):
     return allsupportedversions(repo)
 
 
 # Changegroup versions that can be created from the repo
 def supportedoutgoingversions(repo):
     versions = allsupportedversions(repo)
     if scmutil.istreemanifest(repo):
         # Versions 01 and 02 support only flat manifests and it's just too
         # expensive to convert between the flat manifest and tree manifest on
         # the fly. Since tree manifests are hashed differently, all of history
         # would have to be converted. Instead, we simply don't even pretend to
         # support versions 01 and 02.
         versions.discard(b'01')
         versions.discard(b'02')
     if requirements.NARROW_REQUIREMENT in repo.requirements:
         # Versions 01 and 02 don't support revlog flags, and we need to
         # support that for stripping and unbundling to work.
         versions.discard(b'01')
         versions.discard(b'02')
     if LFS_REQUIREMENT in repo.requirements:
         # Versions 01 and 02 don't support revlog flags, and we need to
         # mark LFS entries with REVIDX_EXTSTORED.
         versions.discard(b'01')
         versions.discard(b'02')
 
     return versions
 
 
 def localversion(repo):
     # Finds the best version to use for bundles that are meant to be used
     # locally, such as those from strip and shelve, and temporary bundles.
     return max(supportedoutgoingversions(repo))
 
 
 def safeversion(repo):
     # Finds the smallest version that it's safe to assume clients of the repo
     # will support. For example, all hg versions that support generaldelta also
     # support changegroup 02.
     versions = supportedoutgoingversions(repo)
     if requirements.GENERALDELTA_REQUIREMENT in repo.requirements:
         versions.discard(b'01')
     assert versions
     return min(versions)
 
 
 def getbundler(
     version,
     repo,
     bundlecaps=None,
     oldmatcher=None,
     matcher=None,
     ellipses=False,
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     assert version in supportedoutgoingversions(repo)
 
     if matcher is None:
         matcher = matchmod.always()
     if oldmatcher is None:
         oldmatcher = matchmod.never()
 
     if version == b'01' and not matcher.always():
         raise error.ProgrammingError(
             b'version 01 changegroups do not support sparse file matchers'
         )
 
     if ellipses and version in (b'01', b'02'):
         raise error.Abort(
             _(
                 b'ellipsis nodes require at least cg3 on client and server, '
                 b'but negotiated version %s'
             )
             % version
         )
 
     # Requested files could include files not in the local store. So
     # filter those out.
     matcher = repo.narrowmatch(matcher)
 
     fn = _packermap[version][0]
     return fn(
         repo,
         oldmatcher,
         matcher,
         bundlecaps,
         ellipses=ellipses,
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
     )
 
 
 def getunbundler(version, fh, alg, extras=None):
     return _packermap[version][1](fh, alg, extras=extras)
 
 
 def _changegroupinfo(repo, nodes, source):
     if repo.ui.verbose or source == b'bundle':
         repo.ui.status(_(b"%d changesets found\n") % len(nodes))
     if repo.ui.debugflag:
         repo.ui.debug(b"list of changesets:\n")
         for node in nodes:
             repo.ui.debug(b"%s\n" % hex(node))
 
 
 def makechangegroup(
     repo, outgoing, version, source, fastpath=False, bundlecaps=None
 ):
     cgstream = makestream(
         repo,
         outgoing,
         version,
         source,
         fastpath=fastpath,
         bundlecaps=bundlecaps,
     )
     return getunbundler(
         version,
         util.chunkbuffer(cgstream),
         None,
         {b'clcount': len(outgoing.missing)},
     )
 
 
 def makestream(
     repo,
     outgoing,
     version,
     source,
     fastpath=False,
     bundlecaps=None,
     matcher=None,
+    remote_sidedata=None,
 ):
-    bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher)
+    bundler = getbundler(
+        version,
+        repo,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
+    )
 
     repo = repo.unfiltered()
     commonrevs = outgoing.common
     csets = outgoing.missing
     heads = outgoing.ancestorsof
     # We go through the fast path if we get told to, or if all (unfiltered
     # heads have been requested (since we then know there all linkrevs will
     # be pulled by the client).
     heads.sort()
     fastpathlinkrev = fastpath or (
         repo.filtername is None and heads == sorted(repo.heads())
     )
 
     repo.hook(b'preoutgoing', throw=True, source=source)
     _changegroupinfo(repo, csets, source)
     return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
 
 
 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
     revisions = 0
     files = 0
     progress = repo.ui.makeprogress(
         _(b'files'), unit=_(b'files'), total=expectedfiles
     )
     for chunkdata in iter(source.filelogheader, {}):
         files += 1
         f = chunkdata[b"filename"]
         repo.ui.debug(b"adding %s revisions\n" % f)
         progress.increment()
         fl = repo.file(f)
         o = len(fl)
         try:
             deltas = source.deltaiter()
             if not fl.addgroup(deltas, revmap, trp):
                 raise error.Abort(_(b"received file revlog group is empty"))
         except error.CensoredBaseError as e:
             raise error.Abort(_(b"received delta base is censored: %s") % e)
         revisions += len(fl) - o
         if f in needfiles:
             needs = needfiles[f]
             for new in pycompat.xrange(o, len(fl)):
                 n = fl.node(new)
                 if n in needs:
                     needs.remove(n)
                 else:
                     raise error.Abort(_(b"received spurious file revlog entry"))
             if not needs:
                 del needfiles[f]
     progress.complete()
 
     for f, needs in pycompat.iteritems(needfiles):
         fl = repo.file(f)
         for n in needs:
             try:
                 fl.rev(n)
             except error.LookupError:
                 raise error.Abort(
                     _(b'missing file data for %s:%s - run hg verify')
                     % (f, hex(n))
                 )
 
     return revisions, files

mercurial/debugcommands.py

@@ -1,4773 +1,4773 b''
 # debugcommands.py - command processing for debug* commands
 #
 # Copyright 2005-2016 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import codecs
 import collections
 import difflib
 import errno
 import glob
 import operator
 import os
 import platform
 import random
 import re
 import socket
 import ssl
 import stat
 import string
 import subprocess
 import sys
 import time
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
     short,
 )
 from .pycompat import (
     getattr,
     open,
 )
 from . import (
     bundle2,
     bundlerepo,
     changegroup,
     cmdutil,
     color,
     context,
     copies,
     dagparser,
     encoding,
     error,
     exchange,
     extensions,
     filemerge,
     filesetlang,
     formatter,
     hg,
     httppeer,
     localrepo,
     lock as lockmod,
     logcmdutil,
     mergestate as mergestatemod,
     metadata,
     obsolete,
     obsutil,
     pathutil,
     phases,
     policy,
     pvec,
     pycompat,
     registrar,
     repair,
     repoview,
     revlog,
     revset,
     revsetlang,
     scmutil,
     setdiscovery,
     simplemerge,
     sshpeer,
     sslutil,
     streamclone,
     strip,
     tags as tagsmod,
     templater,
     treediscovery,
     upgrade,
     url as urlmod,
     util,
     vfs as vfsmod,
     wireprotoframing,
     wireprotoserver,
     wireprotov2peer,
 )
 from .utils import (
     cborutil,
     compression,
     dateutil,
     procutil,
     stringutil,
 )
 
 from .revlogutils import (
     deltas as deltautil,
     nodemap,
     sidedata,
 )
 
 release = lockmod.release
 
 table = {}
 table.update(strip.command._table)
 command = registrar.command(table)
 
 
 @command(b'debugancestor', [], _(b'[INDEX] REV1 REV2'), optionalrepo=True)
 def debugancestor(ui, repo, *args):
     """find the ancestor revision of two revisions in a given index"""
     if len(args) == 3:
         index, rev1, rev2 = args
         r = revlog.revlog(vfsmod.vfs(encoding.getcwd(), audit=False), index)
         lookup = r.lookup
     elif len(args) == 2:
         if not repo:
             raise error.Abort(
                 _(b'there is no Mercurial repository here (.hg not found)')
             )
         rev1, rev2 = args
         r = repo.changelog
         lookup = repo.lookup
     else:
         raise error.Abort(_(b'either two or three arguments required'))
     a = r.ancestor(lookup(rev1), lookup(rev2))
     ui.write(b'%d:%s\n' % (r.rev(a), hex(a)))
 
 
 @command(b'debugantivirusrunning', [])
 def debugantivirusrunning(ui, repo):
     """attempt to trigger an antivirus scanner to see if one is active"""
     with repo.cachevfs.open('eicar-test-file.com', b'wb') as f:
         f.write(
             util.b85decode(
                 # This is a base85-armored version of the EICAR test file. See
                 # https://en.wikipedia.org/wiki/EICAR_test_file for details.
                 b'ST#=}P$fV?P+K%yP+C|uG$>GBDK|qyDK~v2MM*<JQY}+dK~6+LQba95P'
                 b'E<)&Nm5l)EmTEQR4qnHOhq9iNGnJx'
             )
         )
     # Give an AV engine time to scan the file.
     time.sleep(2)
     util.unlink(repo.cachevfs.join('eicar-test-file.com'))
 
 
 @command(b'debugapplystreamclonebundle', [], b'FILE')
 def debugapplystreamclonebundle(ui, repo, fname):
     """apply a stream clone bundle file"""
     f = hg.openpath(ui, fname)
     gen = exchange.readbundle(ui, f, fname)
     gen.apply(repo)
 
 
 @command(
     b'debugbuilddag',
     [
         (
             b'm',
             b'mergeable-file',
             None,
             _(b'add single file mergeable changes'),
         ),
         (
             b'o',
             b'overwritten-file',
             None,
             _(b'add single file all revs overwrite'),
         ),
         (b'n', b'new-file', None, _(b'add new file at each rev')),
     ],
     _(b'[OPTION]... [TEXT]'),
 )
 def debugbuilddag(
     ui,
     repo,
     text=None,
     mergeable_file=False,
     overwritten_file=False,
     new_file=False,
 ):
     """builds a repo with a given DAG from scratch in the current empty repo
 
     The description of the DAG is read from stdin if not given on the
     command line.
 
     Elements:
 
      - "+n" is a linear run of n nodes based on the current default parent
      - "." is a single node based on the current default parent
      - "$" resets the default parent to null (implied at the start);
            otherwise the default parent is always the last node created
      - "<p" sets the default parent to the backref p
      - "*p" is a fork at parent p, which is a backref
      - "*p1/p2" is a merge of parents p1 and p2, which are backrefs
      - "/p2" is a merge of the preceding node and p2
      - ":tag" defines a local tag for the preceding node
      - "@branch" sets the named branch for subsequent nodes
      - "#...\\n" is a comment up to the end of the line
 
     Whitespace between the above elements is ignored.
 
     A backref is either
 
      - a number n, which references the node curr-n, where curr is the current
        node, or
      - the name of a local tag you placed earlier using ":tag", or
      - empty to denote the default parent.
 
     All string valued-elements are either strictly alphanumeric, or must
     be enclosed in double quotes ("..."), with "\\" as escape character.
     """
 
     if text is None:
         ui.status(_(b"reading DAG from stdin\n"))
         text = ui.fin.read()
 
     cl = repo.changelog
     if len(cl) > 0:
         raise error.Abort(_(b'repository is not empty'))
 
     # determine number of revs in DAG
     total = 0
     for type, data in dagparser.parsedag(text):
         if type == b'n':
             total += 1
 
     if mergeable_file:
         linesperrev = 2
         # make a file with k lines per rev
         initialmergedlines = [
             b'%d' % i for i in pycompat.xrange(0, total * linesperrev)
         ]
         initialmergedlines.append(b"")
 
     tags = []
     progress = ui.makeprogress(
         _(b'building'), unit=_(b'revisions'), total=total
     )
     with progress, repo.wlock(), repo.lock(), repo.transaction(b"builddag"):
         at = -1
         atbranch = b'default'
         nodeids = []
         id = 0
         progress.update(id)
         for type, data in dagparser.parsedag(text):
             if type == b'n':
                 ui.note((b'node %s\n' % pycompat.bytestr(data)))
                 id, ps = data
 
                 files = []
                 filecontent = {}
 
                 p2 = None
                 if mergeable_file:
                     fn = b"mf"
                     p1 = repo[ps[0]]
                     if len(ps) > 1:
                         p2 = repo[ps[1]]
                         pa = p1.ancestor(p2)
                         base, local, other = [
                             x[fn].data() for x in (pa, p1, p2)
                         ]
                         m3 = simplemerge.Merge3Text(base, local, other)
                         ml = [l.strip() for l in m3.merge_lines()]
                         ml.append(b"")
                     elif at > 0:
                         ml = p1[fn].data().split(b"\n")
                     else:
                         ml = initialmergedlines
                     ml[id * linesperrev] += b" r%i" % id
                     mergedtext = b"\n".join(ml)
                     files.append(fn)
                     filecontent[fn] = mergedtext
 
                 if overwritten_file:
                     fn = b"of"
                     files.append(fn)
                     filecontent[fn] = b"r%i\n" % id
 
                 if new_file:
                     fn = b"nf%i" % id
                     files.append(fn)
                     filecontent[fn] = b"r%i\n" % id
                     if len(ps) > 1:
                         if not p2:
                             p2 = repo[ps[1]]
                         for fn in p2:
                             if fn.startswith(b"nf"):
                                 files.append(fn)
                                 filecontent[fn] = p2[fn].data()
 
                 def fctxfn(repo, cx, path):
                     if path in filecontent:
                         return context.memfilectx(
                             repo, cx, path, filecontent[path]
                         )
                     return None
 
                 if len(ps) == 0 or ps[0] < 0:
                     pars = [None, None]
                 elif len(ps) == 1:
                     pars = [nodeids[ps[0]], None]
                 else:
                     pars = [nodeids[p] for p in ps]
                 cx = context.memctx(
                     repo,
                     pars,
                     b"r%i" % id,
                     files,
                     fctxfn,
                     date=(id, 0),
                     user=b"debugbuilddag",
                     extra={b'branch': atbranch},
                 )
                 nodeid = repo.commitctx(cx)
                 nodeids.append(nodeid)
                 at = id
             elif type == b'l':
                 id, name = data
                 ui.note((b'tag %s\n' % name))
                 tags.append(b"%s %s\n" % (hex(repo.changelog.node(id)), name))
             elif type == b'a':
                 ui.note((b'branch %s\n' % data))
                 atbranch = data
             progress.update(id)
 
         if tags:
             repo.vfs.write(b"localtags", b"".join(tags))
 
 
 def _debugchangegroup(ui, gen, all=None, indent=0, **opts):
     indent_string = b' ' * indent
     if all:
         ui.writenoi18n(
             b"%sformat: id, p1, p2, cset, delta base, len(delta)\n"
             % indent_string
         )
 
         def showchunks(named):
             ui.write(b"\n%s%s\n" % (indent_string, named))
             for deltadata in gen.deltaiter():
-                node, p1, p2, cs, deltabase, delta, flags = deltadata
+                node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
                 ui.write(
                     b"%s%s %s %s %s %s %d\n"
                     % (
                         indent_string,
                         hex(node),
                         hex(p1),
                         hex(p2),
                         hex(cs),
                         hex(deltabase),
                         len(delta),
                     )
                 )
 
         gen.changelogheader()
         showchunks(b"changelog")
         gen.manifestheader()
         showchunks(b"manifest")
         for chunkdata in iter(gen.filelogheader, {}):
             fname = chunkdata[b'filename']
             showchunks(fname)
     else:
         if isinstance(gen, bundle2.unbundle20):
             raise error.Abort(_(b'use debugbundle2 for this file'))
         gen.changelogheader()
         for deltadata in gen.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
             ui.write(b"%s%s\n" % (indent_string, hex(node)))
 
 
 def _debugobsmarkers(ui, part, indent=0, **opts):
     """display version and markers contained in 'data'"""
     opts = pycompat.byteskwargs(opts)
     data = part.read()
     indent_string = b' ' * indent
     try:
         version, markers = obsolete._readmarkers(data)
     except error.UnknownVersion as exc:
         msg = b"%sunsupported version: %s (%d bytes)\n"
         msg %= indent_string, exc.version, len(data)
         ui.write(msg)
     else:
         msg = b"%sversion: %d (%d bytes)\n"
         msg %= indent_string, version, len(data)
         ui.write(msg)
         fm = ui.formatter(b'debugobsolete', opts)
         for rawmarker in sorted(markers):
             m = obsutil.marker(None, rawmarker)
             fm.startitem()
             fm.plain(indent_string)
             cmdutil.showmarker(fm, m)
         fm.end()
 
 
 def _debugphaseheads(ui, data, indent=0):
     """display version and markers contained in 'data'"""
     indent_string = b' ' * indent
     headsbyphase = phases.binarydecode(data)
     for phase in phases.allphases:
         for head in headsbyphase[phase]:
             ui.write(indent_string)
             ui.write(b'%s %s\n' % (hex(head), phases.phasenames[phase]))
 
 
 def _quasirepr(thing):
     if isinstance(thing, (dict, util.sortdict, collections.OrderedDict)):
         return b'{%s}' % (
             b', '.join(b'%s: %s' % (k, thing[k]) for k in sorted(thing))
         )
     return pycompat.bytestr(repr(thing))
 
 
 def _debugbundle2(ui, gen, all=None, **opts):
     """lists the contents of a bundle2"""
     if not isinstance(gen, bundle2.unbundle20):
         raise error.Abort(_(b'not a bundle2 file'))
     ui.write((b'Stream params: %s\n' % _quasirepr(gen.params)))
     parttypes = opts.get('part_type', [])
     for part in gen.iterparts():
         if parttypes and part.type not in parttypes:
             continue
         msg = b'%s -- %s (mandatory: %r)\n'
         ui.write((msg % (part.type, _quasirepr(part.params), part.mandatory)))
         if part.type == b'changegroup':
             version = part.params.get(b'version', b'01')
             cg = changegroup.getunbundler(version, part, b'UN')
             if not ui.quiet:
                 _debugchangegroup(ui, cg, all=all, indent=4, **opts)
         if part.type == b'obsmarkers':
             if not ui.quiet:
                 _debugobsmarkers(ui, part, indent=4, **opts)
         if part.type == b'phase-heads':
             if not ui.quiet:
                 _debugphaseheads(ui, part, indent=4)
 
 
 @command(
     b'debugbundle',
     [
         (b'a', b'all', None, _(b'show all details')),
         (b'', b'part-type', [], _(b'show only the named part type')),
         (b'', b'spec', None, _(b'print the bundlespec of the bundle')),
     ],
     _(b'FILE'),
     norepo=True,
 )
 def debugbundle(ui, bundlepath, all=None, spec=None, **opts):
     """lists the contents of a bundle"""
     with hg.openpath(ui, bundlepath) as f:
         if spec:
             spec = exchange.getbundlespec(ui, f)
             ui.write(b'%s\n' % spec)
             return
 
         gen = exchange.readbundle(ui, f, bundlepath)
         if isinstance(gen, bundle2.unbundle20):
             return _debugbundle2(ui, gen, all=all, **opts)
         _debugchangegroup(ui, gen, all=all, **opts)
 
 
 @command(b'debugcapabilities', [], _(b'PATH'), norepo=True)
 def debugcapabilities(ui, path, **opts):
     """lists the capabilities of a remote peer"""
     opts = pycompat.byteskwargs(opts)
     peer = hg.peer(ui, opts, path)
     try:
         caps = peer.capabilities()
         ui.writenoi18n(b'Main capabilities:\n')
         for c in sorted(caps):
             ui.write(b'  %s\n' % c)
         b2caps = bundle2.bundle2caps(peer)
         if b2caps:
             ui.writenoi18n(b'Bundle2 capabilities:\n')
             for key, values in sorted(pycompat.iteritems(b2caps)):
                 ui.write(b'  %s\n' % key)
                 for v in values:
                     ui.write(b'    %s\n' % v)
     finally:
         peer.close()
 
 
 @command(
     b'debugchangedfiles',
     [
         (
             b'',
             b'compute',
             False,
             b"compute information instead of reading it from storage",
         ),
     ],
     b'REV',
 )
 def debugchangedfiles(ui, repo, rev, **opts):
     """list the stored files changes for a revision"""
     ctx = scmutil.revsingle(repo, rev, None)
     files = None
 
     if opts['compute']:
         files = metadata.compute_all_files_changes(ctx)
     else:
         sd = repo.changelog.sidedata(ctx.rev())
         files_block = sd.get(sidedata.SD_FILES)
         if files_block is not None:
             files = metadata.decode_files_sidedata(sd)
     if files is not None:
         for f in sorted(files.touched):
             if f in files.added:
                 action = b"added"
             elif f in files.removed:
                 action = b"removed"
             elif f in files.merged:
                 action = b"merged"
             elif f in files.salvaged:
                 action = b"salvaged"
             else:
                 action = b"touched"
 
             copy_parent = b""
             copy_source = b""
             if f in files.copied_from_p1:
                 copy_parent = b"p1"
                 copy_source = files.copied_from_p1[f]
             elif f in files.copied_from_p2:
                 copy_parent = b"p2"
                 copy_source = files.copied_from_p2[f]
 
             data = (action, copy_parent, f, copy_source)
             template = b"%-8s %2s: %s, %s;\n"
             ui.write(template % data)
 
 
 @command(b'debugcheckstate', [], b'')
 def debugcheckstate(ui, repo):
     """validate the correctness of the current dirstate"""
     parent1, parent2 = repo.dirstate.parents()
     m1 = repo[parent1].manifest()
     m2 = repo[parent2].manifest()
     errors = 0
     for f in repo.dirstate:
         state = repo.dirstate[f]
         if state in b"nr" and f not in m1:
             ui.warn(_(b"%s in state %s, but not in manifest1\n") % (f, state))
             errors += 1
         if state in b"a" and f in m1:
             ui.warn(_(b"%s in state %s, but also in manifest1\n") % (f, state))
             errors += 1
         if state in b"m" and f not in m1 and f not in m2:
             ui.warn(
                 _(b"%s in state %s, but not in either manifest\n") % (f, state)
             )
             errors += 1
     for f in m1:
         state = repo.dirstate[f]
         if state not in b"nrm":
             ui.warn(_(b"%s in manifest1, but listed as state %s") % (f, state))
             errors += 1
     if errors:
         errstr = _(b".hg/dirstate inconsistent with current parent's manifest")
         raise error.Abort(errstr)
 
 
 @command(
     b'debugcolor',
     [(b'', b'style', None, _(b'show all configured styles'))],
     b'hg debugcolor',
 )
 def debugcolor(ui, repo, **opts):
     """show available color, effects or style"""
     ui.writenoi18n(b'color mode: %s\n' % stringutil.pprint(ui._colormode))
     if opts.get('style'):
         return _debugdisplaystyle(ui)
     else:
         return _debugdisplaycolor(ui)
 
 
 def _debugdisplaycolor(ui):
     ui = ui.copy()
     ui._styles.clear()
     for effect in color._activeeffects(ui).keys():
         ui._styles[effect] = effect
     if ui._terminfoparams:
         for k, v in ui.configitems(b'color'):
             if k.startswith(b'color.'):
                 ui._styles[k] = k[6:]
             elif k.startswith(b'terminfo.'):
                 ui._styles[k] = k[9:]
     ui.write(_(b'available colors:\n'))
     # sort label with a '_' after the other to group '_background' entry.
     items = sorted(ui._styles.items(), key=lambda i: (b'_' in i[0], i[0], i[1]))
     for colorname, label in items:
         ui.write(b'%s\n' % colorname, label=label)
 
 
 def _debugdisplaystyle(ui):
     ui.write(_(b'available style:\n'))
     if not ui._styles:
         return
     width = max(len(s) for s in ui._styles)
     for label, effects in sorted(ui._styles.items()):
         ui.write(b'%s' % label, label=label)
         if effects:
             # 50
             ui.write(b': ')
             ui.write(b' ' * (max(0, width - len(label))))
             ui.write(b', '.join(ui.label(e, e) for e in effects.split()))
         ui.write(b'\n')
 
 
 @command(b'debugcreatestreamclonebundle', [], b'FILE')
 def debugcreatestreamclonebundle(ui, repo, fname):
     """create a stream clone bundle file
 
     Stream bundles are special bundles that are essentially archives of
     revlog files. They are commonly used for cloning very quickly.
     """
     # TODO we may want to turn this into an abort when this functionality
     # is moved into `hg bundle`.
     if phases.hassecret(repo):
         ui.warn(
             _(
                 b'(warning: stream clone bundle will contain secret '
                 b'revisions)\n'
             )
         )
 
     requirements, gen = streamclone.generatebundlev1(repo)
     changegroup.writechunks(ui, gen, fname)
 
     ui.write(_(b'bundle requirements: %s\n') % b', '.join(sorted(requirements)))
 
 
 @command(
     b'debugdag',
     [
         (b't', b'tags', None, _(b'use tags as labels')),
         (b'b', b'branches', None, _(b'annotate with branch names')),
         (b'', b'dots', None, _(b'use dots for runs')),
         (b's', b'spaces', None, _(b'separate elements by spaces')),
     ],
     _(b'[OPTION]... [FILE [REV]...]'),
     optionalrepo=True,
 )
 def debugdag(ui, repo, file_=None, *revs, **opts):
     """format the changelog or an index DAG as a concise textual description
 
     If you pass a revlog index, the revlog's DAG is emitted. If you list
     revision numbers, they get labeled in the output as rN.
 
     Otherwise, the changelog DAG of the current repo is emitted.
     """
     spaces = opts.get('spaces')
     dots = opts.get('dots')
     if file_:
         rlog = revlog.revlog(vfsmod.vfs(encoding.getcwd(), audit=False), file_)
         revs = {int(r) for r in revs}
 
         def events():
             for r in rlog:
                 yield b'n', (r, list(p for p in rlog.parentrevs(r) if p != -1))
                 if r in revs:
                     yield b'l', (r, b"r%i" % r)
 
     elif repo:
         cl = repo.changelog
         tags = opts.get('tags')
         branches = opts.get('branches')
         if tags:
             labels = {}
             for l, n in repo.tags().items():
                 labels.setdefault(cl.rev(n), []).append(l)
 
         def events():
             b = b"default"
             for r in cl:
                 if branches:
                     newb = cl.read(cl.node(r))[5][b'branch']
                     if newb != b:
                         yield b'a', newb
                         b = newb
                 yield b'n', (r, list(p for p in cl.parentrevs(r) if p != -1))
                 if tags:
                     ls = labels.get(r)
                     if ls:
                         for l in ls:
                             yield b'l', (r, l)
 
     else:
         raise error.Abort(_(b'need repo for changelog dag'))
 
     for line in dagparser.dagtextlines(
         events(),
         addspaces=spaces,
         wraplabels=True,
         wrapannotations=True,
         wrapnonlinear=dots,
         usedots=dots,
         maxlinewidth=70,
     ):
         ui.write(line)
         ui.write(b"\n")
 
 
 @command(b'debugdata', cmdutil.debugrevlogopts, _(b'-c|-m|FILE REV'))
 def debugdata(ui, repo, file_, rev=None, **opts):
     """dump the contents of a data file revision"""
     opts = pycompat.byteskwargs(opts)
     if opts.get(b'changelog') or opts.get(b'manifest') or opts.get(b'dir'):
         if rev is not None:
             raise error.CommandError(b'debugdata', _(b'invalid arguments'))
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'debugdata', _(b'invalid arguments'))
     r = cmdutil.openstorage(repo, b'debugdata', file_, opts)
     try:
         ui.write(r.rawdata(r.lookup(rev)))
     except KeyError:
         raise error.Abort(_(b'invalid revision identifier %s') % rev)
 
 
 @command(
     b'debugdate',
     [(b'e', b'extended', None, _(b'try extended date formats'))],
     _(b'[-e] DATE [RANGE]'),
     norepo=True,
     optionalrepo=True,
 )
 def debugdate(ui, date, range=None, **opts):
     """parse and display a date"""
     if opts["extended"]:
         d = dateutil.parsedate(date, dateutil.extendeddateformats)
     else:
         d = dateutil.parsedate(date)
     ui.writenoi18n(b"internal: %d %d\n" % d)
     ui.writenoi18n(b"standard: %s\n" % dateutil.datestr(d))
     if range:
         m = dateutil.matchdate(range)
         ui.writenoi18n(b"match: %s\n" % m(d[0]))
 
 
 @command(
     b'debugdeltachain',
     cmdutil.debugrevlogopts + cmdutil.formatteropts,
     _(b'-c|-m|FILE'),
     optionalrepo=True,
 )
 def debugdeltachain(ui, repo, file_=None, **opts):
     """dump information about delta chains in a revlog
 
     Output can be templatized. Available template keywords are:
 
     :``rev``:       revision number
     :``chainid``:   delta chain identifier (numbered by unique base)
     :``chainlen``:  delta chain length to this revision
     :``prevrev``:   previous revision in delta chain
     :``deltatype``: role of delta / how it was computed
     :``compsize``:  compressed size of revision
     :``uncompsize``: uncompressed size of revision
     :``chainsize``: total size of compressed revisions in chain
     :``chainratio``: total chain size divided by uncompressed revision size
                     (new delta chains typically start at ratio 2.00)
     :``lindist``:   linear distance from base revision in delta chain to end
                     of this revision
     :``extradist``: total size of revisions not part of this delta chain from
                     base of delta chain to end of this revision; a measurement
                     of how much extra data we need to read/seek across to read
                     the delta chain for this revision
     :``extraratio``: extradist divided by chainsize; another representation of
                     how much unrelated data is needed to load this delta chain
 
     If the repository is configured to use the sparse read, additional keywords
     are available:
 
     :``readsize``:     total size of data read from the disk for a revision
                        (sum of the sizes of all the blocks)
     :``largestblock``: size of the largest block of data read from the disk
     :``readdensity``:  density of useful bytes in the data read from the disk
     :``srchunks``:  in how many data hunks the whole revision would be read
 
     The sparse read can be enabled with experimental.sparse-read = True
     """
     opts = pycompat.byteskwargs(opts)
     r = cmdutil.openrevlog(repo, b'debugdeltachain', file_, opts)
     index = r.index
     start = r.start
     length = r.length
     generaldelta = r.version & revlog.FLAG_GENERALDELTA
     withsparseread = getattr(r, '_withsparseread', False)
 
     def revinfo(rev):
         e = index[rev]
         compsize = e[1]
         uncompsize = e[2]
         chainsize = 0
 
         if generaldelta:
             if e[3] == e[5]:
                 deltatype = b'p1'
             elif e[3] == e[6]:
                 deltatype = b'p2'
             elif e[3] == rev - 1:
                 deltatype = b'prev'
             elif e[3] == rev:
                 deltatype = b'base'
             else:
                 deltatype = b'other'
         else:
             if e[3] == rev:
                 deltatype = b'base'
             else:
                 deltatype = b'prev'
 
         chain = r._deltachain(rev)[0]
         for iterrev in chain:
             e = index[iterrev]
             chainsize += e[1]
 
         return compsize, uncompsize, deltatype, chain, chainsize
 
     fm = ui.formatter(b'debugdeltachain', opts)
 
     fm.plain(
         b'    rev  chain# chainlen     prev   delta       '
         b'size    rawsize  chainsize     ratio   lindist extradist '
         b'extraratio'
     )
     if withsparseread:
         fm.plain(b'   readsize largestblk rddensity srchunks')
     fm.plain(b'\n')
 
     chainbases = {}
     for rev in r:
         comp, uncomp, deltatype, chain, chainsize = revinfo(rev)
         chainbase = chain[0]
         chainid = chainbases.setdefault(chainbase, len(chainbases) + 1)
         basestart = start(chainbase)
         revstart = start(rev)
         lineardist = revstart + comp - basestart
         extradist = lineardist - chainsize
         try:
             prevrev = chain[-2]
         except IndexError:
             prevrev = -1
 
         if uncomp != 0:
             chainratio = float(chainsize) / float(uncomp)
         else:
             chainratio = chainsize
 
         if chainsize != 0:
             extraratio = float(extradist) / float(chainsize)
         else:
             extraratio = extradist
 
         fm.startitem()
         fm.write(
             b'rev chainid chainlen prevrev deltatype compsize '
             b'uncompsize chainsize chainratio lindist extradist '
             b'extraratio',
             b'%7d %7d %8d %8d %7s %10d %10d %10d %9.5f %9d %9d %10.5f',
             rev,
             chainid,
             len(chain),
             prevrev,
             deltatype,
             comp,
             uncomp,
             chainsize,
             chainratio,
             lineardist,
             extradist,
             extraratio,
             rev=rev,
             chainid=chainid,
             chainlen=len(chain),
             prevrev=prevrev,
             deltatype=deltatype,
             compsize=comp,
             uncompsize=uncomp,
             chainsize=chainsize,
             chainratio=chainratio,
             lindist=lineardist,
             extradist=extradist,
             extraratio=extraratio,
         )
         if withsparseread:
             readsize = 0
             largestblock = 0
             srchunks = 0
 
             for revschunk in deltautil.slicechunk(r, chain):
                 srchunks += 1
                 blkend = start(revschunk[-1]) + length(revschunk[-1])
                 blksize = blkend - start(revschunk[0])
 
                 readsize += blksize
                 if largestblock < blksize:
                     largestblock = blksize
 
             if readsize:
                 readdensity = float(chainsize) / float(readsize)
             else:
                 readdensity = 1
 
             fm.write(
                 b'readsize largestblock readdensity srchunks',
                 b' %10d %10d %9.5f %8d',
                 readsize,
                 largestblock,
                 readdensity,
                 srchunks,
                 readsize=readsize,
                 largestblock=largestblock,
                 readdensity=readdensity,
                 srchunks=srchunks,
             )
 
         fm.plain(b'\n')
 
     fm.end()
 
 
 @command(
     b'debugdirstate|debugstate',
     [
         (
             b'',
             b'nodates',
             None,
             _(b'do not display the saved mtime (DEPRECATED)'),
         ),
         (b'', b'dates', True, _(b'display the saved mtime')),
         (b'', b'datesort', None, _(b'sort by saved mtime')),
     ],
     _(b'[OPTION]...'),
 )
 def debugstate(ui, repo, **opts):
     """show the contents of the current dirstate"""
 
     nodates = not opts['dates']
     if opts.get('nodates') is not None:
         nodates = True
     datesort = opts.get('datesort')
 
     if datesort:
         keyfunc = lambda x: (x[1][3], x[0])  # sort by mtime, then by filename
     else:
         keyfunc = None  # sort by filename
     for file_, ent in sorted(pycompat.iteritems(repo.dirstate), key=keyfunc):
         if ent[3] == -1:
             timestr = b'unset               '
         elif nodates:
             timestr = b'set                 '
         else:
             timestr = time.strftime(
                 "%Y-%m-%d %H:%M:%S ", time.localtime(ent[3])
             )
             timestr = encoding.strtolocal(timestr)
         if ent[1] & 0o20000:
             mode = b'lnk'
         else:
             mode = b'%3o' % (ent[1] & 0o777 & ~util.umask)
         ui.write(b"%c %s %10d %s%s\n" % (ent[0], mode, ent[2], timestr, file_))
     for f in repo.dirstate.copies():
         ui.write(_(b"copy: %s -> %s\n") % (repo.dirstate.copied(f), f))
 
 
 @command(
     b'debugdiscovery',
     [
         (b'', b'old', None, _(b'use old-style discovery')),
         (
             b'',
             b'nonheads',
             None,
             _(b'use old-style discovery with non-heads included'),
         ),
         (b'', b'rev', [], b'restrict discovery to this set of revs'),
         (b'', b'seed', b'12323', b'specify the random seed use for discovery'),
         (
             b'',
             b'local-as-revs',
             "",
             'treat local has having these revisions only',
         ),
         (
             b'',
             b'remote-as-revs',
             "",
             'use local as remote, with only these these revisions',
         ),
     ]
     + cmdutil.remoteopts,
     _(b'[--rev REV] [OTHER]'),
 )
 def debugdiscovery(ui, repo, remoteurl=b"default", **opts):
     """runs the changeset discovery protocol in isolation
 
     The local peer can be "replaced" by a subset of the local repository by
     using the `--local-as-revs` flag. Int he same way, usual `remote` peer can
     be "replaced" by a subset of the local repository using the
     `--local-as-revs` flag. This is useful to efficiently debug pathological
     discovery situation.
     """
     opts = pycompat.byteskwargs(opts)
     unfi = repo.unfiltered()
 
     # setup potential extra filtering
     local_revs = opts[b"local_as_revs"]
     remote_revs = opts[b"remote_as_revs"]
 
     # make sure tests are repeatable
     random.seed(int(opts[b'seed']))
 
     if not remote_revs:
 
         remoteurl, branches = hg.parseurl(ui.expandpath(remoteurl))
         remote = hg.peer(repo, opts, remoteurl)
         ui.status(_(b'comparing with %s\n') % util.hidepassword(remoteurl))
     else:
         branches = (None, [])
         remote_filtered_revs = scmutil.revrange(
             unfi, [b"not (::(%s))" % remote_revs]
         )
         remote_filtered_revs = frozenset(remote_filtered_revs)
 
         def remote_func(x):
             return remote_filtered_revs
 
         repoview.filtertable[b'debug-discovery-remote-filter'] = remote_func
 
         remote = repo.peer()
         remote._repo = remote._repo.filtered(b'debug-discovery-remote-filter')
 
     if local_revs:
         local_filtered_revs = scmutil.revrange(
             unfi, [b"not (::(%s))" % local_revs]
         )
         local_filtered_revs = frozenset(local_filtered_revs)
 
         def local_func(x):
             return local_filtered_revs
 
         repoview.filtertable[b'debug-discovery-local-filter'] = local_func
         repo = repo.filtered(b'debug-discovery-local-filter')
 
     data = {}
     if opts.get(b'old'):
 
         def doit(pushedrevs, remoteheads, remote=remote):
             if not util.safehasattr(remote, b'branches'):
                 # enable in-client legacy support
                 remote = localrepo.locallegacypeer(remote.local())
             common, _in, hds = treediscovery.findcommonincoming(
                 repo, remote, force=True, audit=data
             )
             common = set(common)
             if not opts.get(b'nonheads'):
                 ui.writenoi18n(
                     b"unpruned common: %s\n"
                     % b" ".join(sorted(short(n) for n in common))
                 )
 
                 clnode = repo.changelog.node
                 common = repo.revs(b'heads(::%ln)', common)
                 common = {clnode(r) for r in common}
             return common, hds
 
     else:
 
         def doit(pushedrevs, remoteheads, remote=remote):
             nodes = None
             if pushedrevs:
                 revs = scmutil.revrange(repo, pushedrevs)
                 nodes = [repo[r].node() for r in revs]
             common, any, hds = setdiscovery.findcommonheads(
                 ui, repo, remote, ancestorsof=nodes, audit=data
             )
             return common, hds
 
     remoterevs, _checkout = hg.addbranchrevs(repo, remote, branches, revs=None)
     localrevs = opts[b'rev']
     with util.timedcm('debug-discovery') as t:
         common, hds = doit(localrevs, remoterevs)
 
     # compute all statistics
     heads_common = set(common)
     heads_remote = set(hds)
     heads_local = set(repo.heads())
     # note: they cannot be a local or remote head that is in common and not
     # itself a head of common.
     heads_common_local = heads_common & heads_local
     heads_common_remote = heads_common & heads_remote
     heads_common_both = heads_common & heads_remote & heads_local
 
     all = repo.revs(b'all()')
     common = repo.revs(b'::%ln', common)
     roots_common = repo.revs(b'roots(::%ld)', common)
     missing = repo.revs(b'not ::%ld', common)
     heads_missing = repo.revs(b'heads(%ld)', missing)
     roots_missing = repo.revs(b'roots(%ld)', missing)
     assert len(common) + len(missing) == len(all)
 
     initial_undecided = repo.revs(
         b'not (::%ln or %ln::)', heads_common_remote, heads_common_local
     )
     heads_initial_undecided = repo.revs(b'heads(%ld)', initial_undecided)
     roots_initial_undecided = repo.revs(b'roots(%ld)', initial_undecided)
     common_initial_undecided = initial_undecided & common
     missing_initial_undecided = initial_undecided & missing
 
     data[b'elapsed'] = t.elapsed
     data[b'nb-common-heads'] = len(heads_common)
     data[b'nb-common-heads-local'] = len(heads_common_local)
     data[b'nb-common-heads-remote'] = len(heads_common_remote)
     data[b'nb-common-heads-both'] = len(heads_common_both)
     data[b'nb-common-roots'] = len(roots_common)
     data[b'nb-head-local'] = len(heads_local)
     data[b'nb-head-local-missing'] = len(heads_local) - len(heads_common_local)
     data[b'nb-head-remote'] = len(heads_remote)
     data[b'nb-head-remote-unknown'] = len(heads_remote) - len(
         heads_common_remote
     )
     data[b'nb-revs'] = len(all)
     data[b'nb-revs-common'] = len(common)
     data[b'nb-revs-missing'] = len(missing)
     data[b'nb-missing-heads'] = len(heads_missing)
     data[b'nb-missing-roots'] = len(roots_missing)
     data[b'nb-ini_und'] = len(initial_undecided)
     data[b'nb-ini_und-heads'] = len(heads_initial_undecided)
     data[b'nb-ini_und-roots'] = len(roots_initial_undecided)
     data[b'nb-ini_und-common'] = len(common_initial_undecided)
     data[b'nb-ini_und-missing'] = len(missing_initial_undecided)
 
     # display discovery summary
     ui.writenoi18n(b"elapsed time:  %(elapsed)f seconds\n" % data)
     ui.writenoi18n(b"round-trips:           %(total-roundtrips)9d\n" % data)
     ui.writenoi18n(b"heads summary:\n")
     ui.writenoi18n(b"  total common heads:  %(nb-common-heads)9d\n" % data)
     ui.writenoi18n(
         b"    also local heads:  %(nb-common-heads-local)9d\n" % data
     )
     ui.writenoi18n(
         b"    also remote heads: %(nb-common-heads-remote)9d\n" % data
     )
     ui.writenoi18n(b"    both:              %(nb-common-heads-both)9d\n" % data)
     ui.writenoi18n(b"  local heads:         %(nb-head-local)9d\n" % data)
     ui.writenoi18n(
         b"    common:            %(nb-common-heads-local)9d\n" % data
     )
     ui.writenoi18n(
         b"    missing:           %(nb-head-local-missing)9d\n" % data
     )
     ui.writenoi18n(b"  remote heads:        %(nb-head-remote)9d\n" % data)
     ui.writenoi18n(
         b"    common:            %(nb-common-heads-remote)9d\n" % data
     )
     ui.writenoi18n(
         b"    unknown:           %(nb-head-remote-unknown)9d\n" % data
     )
     ui.writenoi18n(b"local changesets:      %(nb-revs)9d\n" % data)
     ui.writenoi18n(b"  common:              %(nb-revs-common)9d\n" % data)
     ui.writenoi18n(b"    heads:             %(nb-common-heads)9d\n" % data)
     ui.writenoi18n(b"    roots:             %(nb-common-roots)9d\n" % data)
     ui.writenoi18n(b"  missing:             %(nb-revs-missing)9d\n" % data)
     ui.writenoi18n(b"    heads:             %(nb-missing-heads)9d\n" % data)
     ui.writenoi18n(b"    roots:             %(nb-missing-roots)9d\n" % data)
     ui.writenoi18n(b"  first undecided set: %(nb-ini_und)9d\n" % data)
     ui.writenoi18n(b"    heads:             %(nb-ini_und-heads)9d\n" % data)
     ui.writenoi18n(b"    roots:             %(nb-ini_und-roots)9d\n" % data)
     ui.writenoi18n(b"    common:            %(nb-ini_und-common)9d\n" % data)
     ui.writenoi18n(b"    missing:           %(nb-ini_und-missing)9d\n" % data)
 
     if ui.verbose:
         ui.writenoi18n(
             b"common heads: %s\n"
             % b" ".join(sorted(short(n) for n in heads_common))
         )
 
 
 _chunksize = 4 << 10
 
 
 @command(
     b'debugdownload',
     [
         (b'o', b'output', b'', _(b'path')),
     ],
     optionalrepo=True,
 )
 def debugdownload(ui, repo, url, output=None, **opts):
     """download a resource using Mercurial logic and config"""
     fh = urlmod.open(ui, url, output)
 
     dest = ui
     if output:
         dest = open(output, b"wb", _chunksize)
     try:
         data = fh.read(_chunksize)
         while data:
             dest.write(data)
             data = fh.read(_chunksize)
     finally:
         if output:
             dest.close()
 
 
 @command(b'debugextensions', cmdutil.formatteropts, [], optionalrepo=True)
 def debugextensions(ui, repo, **opts):
     '''show information about active extensions'''
     opts = pycompat.byteskwargs(opts)
     exts = extensions.extensions(ui)
     hgver = util.version()
     fm = ui.formatter(b'debugextensions', opts)
     for extname, extmod in sorted(exts, key=operator.itemgetter(0)):
         isinternal = extensions.ismoduleinternal(extmod)
         extsource = None
 
         if util.safehasattr(extmod, '__file__'):
             extsource = pycompat.fsencode(extmod.__file__)
         elif getattr(sys, 'oxidized', False):
             extsource = pycompat.sysexecutable
         if isinternal:
             exttestedwith = []  # never expose magic string to users
         else:
             exttestedwith = getattr(extmod, 'testedwith', b'').split()
         extbuglink = getattr(extmod, 'buglink', None)
 
         fm.startitem()
 
         if ui.quiet or ui.verbose:
             fm.write(b'name', b'%s\n', extname)
         else:
             fm.write(b'name', b'%s', extname)
             if isinternal or hgver in exttestedwith:
                 fm.plain(b'\n')
             elif not exttestedwith:
                 fm.plain(_(b' (untested!)\n'))
             else:
                 lasttestedversion = exttestedwith[-1]
                 fm.plain(b' (%s!)\n' % lasttestedversion)
 
         fm.condwrite(
             ui.verbose and extsource,
             b'source',
             _(b'  location: %s\n'),
             extsource or b"",
         )
 
         if ui.verbose:
             fm.plain(_(b'  bundled: %s\n') % [b'no', b'yes'][isinternal])
         fm.data(bundled=isinternal)
 
         fm.condwrite(
             ui.verbose and exttestedwith,
             b'testedwith',
             _(b'  tested with: %s\n'),
             fm.formatlist(exttestedwith, name=b'ver'),
         )
 
         fm.condwrite(
             ui.verbose and extbuglink,
             b'buglink',
             _(b'  bug reporting: %s\n'),
             extbuglink or b"",
         )
 
     fm.end()
 
 
 @command(
     b'debugfileset',
     [
         (
             b'r',
             b'rev',
             b'',
             _(b'apply the filespec on this revision'),
             _(b'REV'),
         ),
         (
             b'',
             b'all-files',
             False,
             _(b'test files from all revisions and working directory'),
         ),
         (
             b's',
             b'show-matcher',
             None,
             _(b'print internal representation of matcher'),
         ),
         (
             b'p',
             b'show-stage',
             [],
             _(b'print parsed tree at the given stage'),
             _(b'NAME'),
         ),
     ],
     _(b'[-r REV] [--all-files] [OPTION]... FILESPEC'),
 )
 def debugfileset(ui, repo, expr, **opts):
     '''parse and apply a fileset specification'''
     from . import fileset
 
     fileset.symbols  # force import of fileset so we have predicates to optimize
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get(b'rev'), None)
 
     stages = [
         (b'parsed', pycompat.identity),
         (b'analyzed', filesetlang.analyze),
         (b'optimized', filesetlang.optimize),
     ]
     stagenames = {n for n, f in stages}
 
     showalways = set()
     if ui.verbose and not opts[b'show_stage']:
         # show parsed tree by --verbose (deprecated)
         showalways.add(b'parsed')
     if opts[b'show_stage'] == [b'all']:
         showalways.update(stagenames)
     else:
         for n in opts[b'show_stage']:
             if n not in stagenames:
                 raise error.Abort(_(b'invalid stage name: %s') % n)
         showalways.update(opts[b'show_stage'])
 
     tree = filesetlang.parse(expr)
     for n, f in stages:
         tree = f(tree)
         if n in showalways:
             if opts[b'show_stage'] or n != b'parsed':
                 ui.write(b"* %s:\n" % n)
             ui.write(filesetlang.prettyformat(tree), b"\n")
 
     files = set()
     if opts[b'all_files']:
         for r in repo:
             c = repo[r]
             files.update(c.files())
             files.update(c.substate)
     if opts[b'all_files'] or ctx.rev() is None:
         wctx = repo[None]
         files.update(
             repo.dirstate.walk(
                 scmutil.matchall(repo),
                 subrepos=list(wctx.substate),
                 unknown=True,
                 ignored=True,
             )
         )
         files.update(wctx.substate)
     else:
         files.update(ctx.files())
         files.update(ctx.substate)
 
     m = ctx.matchfileset(repo.getcwd(), expr)
     if opts[b'show_matcher'] or (opts[b'show_matcher'] is None and ui.verbose):
         ui.writenoi18n(b'* matcher:\n', stringutil.prettyrepr(m), b'\n')
     for f in sorted(files):
         if not m(f):
             continue
         ui.write(b"%s\n" % f)
 
 
 @command(b'debugformat', [] + cmdutil.formatteropts)
 def debugformat(ui, repo, **opts):
     """display format information about the current repository
 
     Use --verbose to get extra information about current config value and
     Mercurial default."""
     opts = pycompat.byteskwargs(opts)
     maxvariantlength = max(len(fv.name) for fv in upgrade.allformatvariant)
     maxvariantlength = max(len(b'format-variant'), maxvariantlength)
 
     def makeformatname(name):
         return b'%s:' + (b' ' * (maxvariantlength - len(name)))
 
     fm = ui.formatter(b'debugformat', opts)
     if fm.isplain():
 
         def formatvalue(value):
             if util.safehasattr(value, b'startswith'):
                 return value
             if value:
                 return b'yes'
             else:
                 return b'no'
 
     else:
         formatvalue = pycompat.identity
 
     fm.plain(b'format-variant')
     fm.plain(b' ' * (maxvariantlength - len(b'format-variant')))
     fm.plain(b' repo')
     if ui.verbose:
         fm.plain(b' config default')
     fm.plain(b'\n')
     for fv in upgrade.allformatvariant:
         fm.startitem()
         repovalue = fv.fromrepo(repo)
         configvalue = fv.fromconfig(repo)
 
         if repovalue != configvalue:
             namelabel = b'formatvariant.name.mismatchconfig'
             repolabel = b'formatvariant.repo.mismatchconfig'
         elif repovalue != fv.default:
             namelabel = b'formatvariant.name.mismatchdefault'
             repolabel = b'formatvariant.repo.mismatchdefault'
         else:
             namelabel = b'formatvariant.name.uptodate'
             repolabel = b'formatvariant.repo.uptodate'
 
         fm.write(b'name', makeformatname(fv.name), fv.name, label=namelabel)
         fm.write(b'repo', b' %3s', formatvalue(repovalue), label=repolabel)
         if fv.default != configvalue:
             configlabel = b'formatvariant.config.special'
         else:
             configlabel = b'formatvariant.config.default'
         fm.condwrite(
             ui.verbose,
             b'config',
             b' %6s',
             formatvalue(configvalue),
             label=configlabel,
         )
         fm.condwrite(
             ui.verbose,
             b'default',
             b' %7s',
             formatvalue(fv.default),
             label=b'formatvariant.default',
         )
         fm.plain(b'\n')
     fm.end()
 
 
 @command(b'debugfsinfo', [], _(b'[PATH]'), norepo=True)
 def debugfsinfo(ui, path=b"."):
     """show information detected about current filesystem"""
     ui.writenoi18n(b'path: %s\n' % path)
     ui.writenoi18n(
         b'mounted on: %s\n' % (util.getfsmountpoint(path) or b'(unknown)')
     )
     ui.writenoi18n(b'exec: %s\n' % (util.checkexec(path) and b'yes' or b'no'))
     ui.writenoi18n(b'fstype: %s\n' % (util.getfstype(path) or b'(unknown)'))
     ui.writenoi18n(
         b'symlink: %s\n' % (util.checklink(path) and b'yes' or b'no')
     )
     ui.writenoi18n(
         b'hardlink: %s\n' % (util.checknlink(path) and b'yes' or b'no')
     )
     casesensitive = b'(unknown)'
     try:
         with pycompat.namedtempfile(prefix=b'.debugfsinfo', dir=path) as f:
             casesensitive = util.fscasesensitive(f.name) and b'yes' or b'no'
     except OSError:
         pass
     ui.writenoi18n(b'case-sensitive: %s\n' % casesensitive)
 
 
 @command(
     b'debuggetbundle',
     [
         (b'H', b'head', [], _(b'id of head node'), _(b'ID')),
         (b'C', b'common', [], _(b'id of common node'), _(b'ID')),
         (
             b't',
             b'type',
             b'bzip2',
             _(b'bundle compression type to use'),
             _(b'TYPE'),
         ),
     ],
     _(b'REPO FILE [-H|-C ID]...'),
     norepo=True,
 )
 def debuggetbundle(ui, repopath, bundlepath, head=None, common=None, **opts):
     """retrieves a bundle from a repo
 
     Every ID must be a full-length hex node id string. Saves the bundle to the
     given file.
     """
     opts = pycompat.byteskwargs(opts)
     repo = hg.peer(ui, opts, repopath)
     if not repo.capable(b'getbundle'):
         raise error.Abort(b"getbundle() not supported by target repository")
     args = {}
     if common:
         args['common'] = [bin(s) for s in common]
     if head:
         args['heads'] = [bin(s) for s in head]
     # TODO: get desired bundlecaps from command line.
     args['bundlecaps'] = None
     bundle = repo.getbundle(b'debug', **args)
 
     bundletype = opts.get(b'type', b'bzip2').lower()
     btypes = {
         b'none': b'HG10UN',
         b'bzip2': b'HG10BZ',
         b'gzip': b'HG10GZ',
         b'bundle2': b'HG20',
     }
     bundletype = btypes.get(bundletype)
     if bundletype not in bundle2.bundletypes:
         raise error.Abort(_(b'unknown bundle type specified with --type'))
     bundle2.writebundle(ui, bundle, bundlepath, bundletype)
 
 
 @command(b'debugignore', [], b'[FILE]')
 def debugignore(ui, repo, *files, **opts):
     """display the combined ignore pattern and information about ignored files
 
     With no argument display the combined ignore pattern.
 
     Given space separated file names, shows if the given file is ignored and
     if so, show the ignore rule (file and line number) that matched it.
     """
     ignore = repo.dirstate._ignore
     if not files:
         # Show all the patterns
         ui.write(b"%s\n" % pycompat.byterepr(ignore))
     else:
         m = scmutil.match(repo[None], pats=files)
         uipathfn = scmutil.getuipathfn(repo, legacyrelativevalue=True)
         for f in m.files():
             nf = util.normpath(f)
             ignored = None
             ignoredata = None
             if nf != b'.':
                 if ignore(nf):
                     ignored = nf
                     ignoredata = repo.dirstate._ignorefileandline(nf)
                 else:
                     for p in pathutil.finddirs(nf):
                         if ignore(p):
                             ignored = p
                             ignoredata = repo.dirstate._ignorefileandline(p)
                             break
             if ignored:
                 if ignored == nf:
                     ui.write(_(b"%s is ignored\n") % uipathfn(f))
                 else:
                     ui.write(
                         _(
                             b"%s is ignored because of "
                             b"containing directory %s\n"
                         )
                         % (uipathfn(f), ignored)
                     )
                 ignorefile, lineno, line = ignoredata
                 ui.write(
                     _(b"(ignore rule in %s, line %d: '%s')\n")
                     % (ignorefile, lineno, line)
                 )
             else:
                 ui.write(_(b"%s is not ignored\n") % uipathfn(f))
 
 
 @command(
     b'debugindex',
     cmdutil.debugrevlogopts + cmdutil.formatteropts,
     _(b'-c|-m|FILE'),
 )
 def debugindex(ui, repo, file_=None, **opts):
     """dump index data for a storage primitive"""
     opts = pycompat.byteskwargs(opts)
     store = cmdutil.openstorage(repo, b'debugindex', file_, opts)
 
     if ui.debugflag:
         shortfn = hex
     else:
         shortfn = short
 
     idlen = 12
     for i in store:
         idlen = len(shortfn(store.node(i)))
         break
 
     fm = ui.formatter(b'debugindex', opts)
     fm.plain(
         b'   rev linkrev %s %s p2\n'
         % (b'nodeid'.ljust(idlen), b'p1'.ljust(idlen))
     )
 
     for rev in store:
         node = store.node(rev)
         parents = store.parents(node)
 
         fm.startitem()
         fm.write(b'rev', b'%6d ', rev)
         fm.write(b'linkrev', b'%7d ', store.linkrev(rev))
         fm.write(b'node', b'%s ', shortfn(node))
         fm.write(b'p1', b'%s ', shortfn(parents[0]))
         fm.write(b'p2', b'%s', shortfn(parents[1]))
         fm.plain(b'\n')
 
     fm.end()
 
 
 @command(
     b'debugindexdot',
     cmdutil.debugrevlogopts,
     _(b'-c|-m|FILE'),
     optionalrepo=True,
 )
 def debugindexdot(ui, repo, file_=None, **opts):
     """dump an index DAG as a graphviz dot file"""
     opts = pycompat.byteskwargs(opts)
     r = cmdutil.openstorage(repo, b'debugindexdot', file_, opts)
     ui.writenoi18n(b"digraph G {\n")
     for i in r:
         node = r.node(i)
         pp = r.parents(node)
         ui.write(b"\t%d -> %d\n" % (r.rev(pp[0]), i))
         if pp[1] != nullid:
             ui.write(b"\t%d -> %d\n" % (r.rev(pp[1]), i))
     ui.write(b"}\n")
 
 
 @command(b'debugindexstats', [])
 def debugindexstats(ui, repo):
     """show stats related to the changelog index"""
     repo.changelog.shortest(nullid, 1)
     index = repo.changelog.index
     if not util.safehasattr(index, b'stats'):
         raise error.Abort(_(b'debugindexstats only works with native code'))
     for k, v in sorted(index.stats().items()):
         ui.write(b'%s: %d\n' % (k, v))
 
 
 @command(b'debuginstall', [] + cmdutil.formatteropts, b'', norepo=True)
 def debuginstall(ui, **opts):
     """test Mercurial installation
 
     Returns 0 on success.
     """
     opts = pycompat.byteskwargs(opts)
 
     problems = 0
 
     fm = ui.formatter(b'debuginstall', opts)
     fm.startitem()
 
     # encoding might be unknown or wrong. don't translate these messages.
     fm.write(b'encoding', b"checking encoding (%s)...\n", encoding.encoding)
     err = None
     try:
         codecs.lookup(pycompat.sysstr(encoding.encoding))
     except LookupError as inst:
         err = stringutil.forcebytestr(inst)
         problems += 1
     fm.condwrite(
         err,
         b'encodingerror',
         b" %s\n (check that your locale is properly set)\n",
         err,
     )
 
     # Python
     pythonlib = None
     if util.safehasattr(os, '__file__'):
         pythonlib = os.path.dirname(pycompat.fsencode(os.__file__))
     elif getattr(sys, 'oxidized', False):
         pythonlib = pycompat.sysexecutable
 
     fm.write(
         b'pythonexe',
         _(b"checking Python executable (%s)\n"),
         pycompat.sysexecutable or _(b"unknown"),
     )
     fm.write(
         b'pythonimplementation',
         _(b"checking Python implementation (%s)\n"),
         pycompat.sysbytes(platform.python_implementation()),
     )
     fm.write(
         b'pythonver',
         _(b"checking Python version (%s)\n"),
         (b"%d.%d.%d" % sys.version_info[:3]),
     )
     fm.write(
         b'pythonlib',
         _(b"checking Python lib (%s)...\n"),
         pythonlib or _(b"unknown"),
     )
 
     try:
         from . import rustext
 
         rustext.__doc__  # trigger lazy import
     except ImportError:
         rustext = None
 
     security = set(sslutil.supportedprotocols)
     if sslutil.hassni:
         security.add(b'sni')
 
     fm.write(
         b'pythonsecurity',
         _(b"checking Python security support (%s)\n"),
         fm.formatlist(sorted(security), name=b'protocol', fmt=b'%s', sep=b','),
     )
 
     # These are warnings, not errors. So don't increment problem count. This
     # may change in the future.
     if b'tls1.2' not in security:
         fm.plain(
             _(
                 b'  TLS 1.2 not supported by Python install; '
                 b'network connections lack modern security\n'
             )
         )
     if b'sni' not in security:
         fm.plain(
             _(
                 b'  SNI not supported by Python install; may have '
                 b'connectivity issues with some servers\n'
             )
         )
 
     fm.plain(
         _(
             b"checking Rust extensions (%s)\n"
             % (b'missing' if rustext is None else b'installed')
         ),
     )
 
     # TODO print CA cert info
 
     # hg version
     hgver = util.version()
     fm.write(
         b'hgver', _(b"checking Mercurial version (%s)\n"), hgver.split(b'+')[0]
     )
     fm.write(
         b'hgverextra',
         _(b"checking Mercurial custom build (%s)\n"),
         b'+'.join(hgver.split(b'+')[1:]),
     )
 
     # compiled modules
     hgmodules = None
     if util.safehasattr(sys.modules[__name__], '__file__'):
         hgmodules = os.path.dirname(pycompat.fsencode(__file__))
     elif getattr(sys, 'oxidized', False):
         hgmodules = pycompat.sysexecutable
 
     fm.write(
         b'hgmodulepolicy', _(b"checking module policy (%s)\n"), policy.policy
     )
     fm.write(
         b'hgmodules',
         _(b"checking installed modules (%s)...\n"),
         hgmodules or _(b"unknown"),
     )
 
     rustandc = policy.policy in (b'rust+c', b'rust+c-allow')
     rustext = rustandc  # for now, that's the only case
     cext = policy.policy in (b'c', b'allow') or rustandc
     nopure = cext or rustext
     if nopure:
         err = None
         try:
             if cext:
                 from .cext import (  # pytype: disable=import-error
                     base85,
                     bdiff,
                     mpatch,
                     osutil,
                 )
 
                 # quiet pyflakes
                 dir(bdiff), dir(mpatch), dir(base85), dir(osutil)
             if rustext:
                 from .rustext import (  # pytype: disable=import-error
                     ancestor,
                     dirstate,
                 )
 
                 dir(ancestor), dir(dirstate)  # quiet pyflakes
         except Exception as inst:
             err = stringutil.forcebytestr(inst)
             problems += 1
         fm.condwrite(err, b'extensionserror', b" %s\n", err)
 
     compengines = util.compengines._engines.values()
     fm.write(
         b'compengines',
         _(b'checking registered compression engines (%s)\n'),
         fm.formatlist(
             sorted(e.name() for e in compengines),
             name=b'compengine',
             fmt=b'%s',
             sep=b', ',
         ),
     )
     fm.write(
         b'compenginesavail',
         _(b'checking available compression engines (%s)\n'),
         fm.formatlist(
             sorted(e.name() for e in compengines if e.available()),
             name=b'compengine',
             fmt=b'%s',
             sep=b', ',
         ),
     )
     wirecompengines = compression.compengines.supportedwireengines(
         compression.SERVERROLE
     )
     fm.write(
         b'compenginesserver',
         _(
             b'checking available compression engines '
             b'for wire protocol (%s)\n'
         ),
         fm.formatlist(
             [e.name() for e in wirecompengines if e.wireprotosupport()],
             name=b'compengine',
             fmt=b'%s',
             sep=b', ',
         ),
     )
     re2 = b'missing'
     if util._re2:
         re2 = b'available'
     fm.plain(_(b'checking "re2" regexp engine (%s)\n') % re2)
     fm.data(re2=bool(util._re2))
 
     # templates
     p = templater.templatedir()
     fm.write(b'templatedirs', b'checking templates (%s)...\n', p or b'')
     fm.condwrite(not p, b'', _(b" no template directories found\n"))
     if p:
         (m, fp) = templater.try_open_template(b"map-cmdline.default")
         if m:
             # template found, check if it is working
             err = None
             try:
                 templater.templater.frommapfile(m)
             except Exception as inst:
                 err = stringutil.forcebytestr(inst)
                 p = None
             fm.condwrite(err, b'defaulttemplateerror', b" %s\n", err)
         else:
             p = None
         fm.condwrite(
             p, b'defaulttemplate', _(b"checking default template (%s)\n"), m
         )
         fm.condwrite(
             not m,
             b'defaulttemplatenotfound',
             _(b" template '%s' not found\n"),
             b"default",
         )
     if not p:
         problems += 1
     fm.condwrite(
         not p, b'', _(b" (templates seem to have been installed incorrectly)\n")
     )
 
     # editor
     editor = ui.geteditor()
     editor = util.expandpath(editor)
     editorbin = procutil.shellsplit(editor)[0]
     fm.write(b'editor', _(b"checking commit editor... (%s)\n"), editorbin)
     cmdpath = procutil.findexe(editorbin)
     fm.condwrite(
         not cmdpath and editor == b'vi',
         b'vinotfound',
         _(
             b" No commit editor set and can't find %s in PATH\n"
             b" (specify a commit editor in your configuration"
             b" file)\n"
         ),
         not cmdpath and editor == b'vi' and editorbin,
     )
     fm.condwrite(
         not cmdpath and editor != b'vi',
         b'editornotfound',
         _(
             b" Can't find editor '%s' in PATH\n"
             b" (specify a commit editor in your configuration"
             b" file)\n"
         ),
         not cmdpath and editorbin,
     )
     if not cmdpath and editor != b'vi':
         problems += 1
 
     # check username
     username = None
     err = None
     try:
         username = ui.username()
     except error.Abort as e:
         err = e.message
         problems += 1
 
     fm.condwrite(
         username, b'username', _(b"checking username (%s)\n"), username
     )
     fm.condwrite(
         err,
         b'usernameerror',
         _(
             b"checking username...\n %s\n"
             b" (specify a username in your configuration file)\n"
         ),
         err,
     )
 
     for name, mod in extensions.extensions():
         handler = getattr(mod, 'debuginstall', None)
         if handler is not None:
             problems += handler(ui, fm)
 
     fm.condwrite(not problems, b'', _(b"no problems detected\n"))
     if not problems:
         fm.data(problems=problems)
     fm.condwrite(
         problems,
         b'problems',
         _(b"%d problems detected, please check your install!\n"),
         problems,
     )
     fm.end()
 
     return problems
 
 
 @command(b'debugknown', [], _(b'REPO ID...'), norepo=True)
 def debugknown(ui, repopath, *ids, **opts):
     """test whether node ids are known to a repo
 
     Every ID must be a full-length hex node id string. Returns a list of 0s
     and 1s indicating unknown/known.
     """
     opts = pycompat.byteskwargs(opts)
     repo = hg.peer(ui, opts, repopath)
     if not repo.capable(b'known'):
         raise error.Abort(b"known() not supported by target repository")
     flags = repo.known([bin(s) for s in ids])
     ui.write(b"%s\n" % (b"".join([f and b"1" or b"0" for f in flags])))
 
 
 @command(b'debuglabelcomplete', [], _(b'LABEL...'))
 def debuglabelcomplete(ui, repo, *args):
     '''backwards compatibility with old bash completion scripts (DEPRECATED)'''
     debugnamecomplete(ui, repo, *args)
 
 
 @command(
     b'debuglocks',
     [
         (b'L', b'force-free-lock', None, _(b'free the store lock (DANGEROUS)')),
         (
             b'W',
             b'force-free-wlock',
             None,
             _(b'free the working state lock (DANGEROUS)'),
         ),
         (b's', b'set-lock', None, _(b'set the store lock until stopped')),
         (
             b'S',
             b'set-wlock',
             None,
             _(b'set the working state lock until stopped'),
         ),
     ],
     _(b'[OPTION]...'),
 )
 def debuglocks(ui, repo, **opts):
     """show or modify state of locks
 
     By default, this command will show which locks are held. This
     includes the user and process holding the lock, the amount of time
     the lock has been held, and the machine name where the process is
     running if it's not local.
 
     Locks protect the integrity of Mercurial's data, so should be
     treated with care. System crashes or other interruptions may cause
     locks to not be properly released, though Mercurial will usually
     detect and remove such stale locks automatically.
 
     However, detecting stale locks may not always be possible (for
     instance, on a shared filesystem). Removing locks may also be
     blocked by filesystem permissions.
 
     Setting a lock will prevent other commands from changing the data.
     The command will wait until an interruption (SIGINT, SIGTERM, ...) occurs.
     The set locks are removed when the command exits.
 
     Returns 0 if no locks are held.
 
     """
 
     if opts.get('force_free_lock'):
         repo.svfs.unlink(b'lock')
     if opts.get('force_free_wlock'):
         repo.vfs.unlink(b'wlock')
     if opts.get('force_free_lock') or opts.get('force_free_wlock'):
         return 0
 
     locks = []
     try:
         if opts.get('set_wlock'):
             try:
                 locks.append(repo.wlock(False))
             except error.LockHeld:
                 raise error.Abort(_(b'wlock is already held'))
         if opts.get('set_lock'):
             try:
                 locks.append(repo.lock(False))
             except error.LockHeld:
                 raise error.Abort(_(b'lock is already held'))
         if len(locks):
             ui.promptchoice(_(b"ready to release the lock (y)? $$ &Yes"))
             return 0
     finally:
         release(*locks)
 
     now = time.time()
     held = 0
 
     def report(vfs, name, method):
         # this causes stale locks to get reaped for more accurate reporting
         try:
             l = method(False)
         except error.LockHeld:
             l = None
 
         if l:
             l.release()
         else:
             try:
                 st = vfs.lstat(name)
                 age = now - st[stat.ST_MTIME]
                 user = util.username(st.st_uid)
                 locker = vfs.readlock(name)
                 if b":" in locker:
                     host, pid = locker.split(b':')
                     if host == socket.gethostname():
                         locker = b'user %s, process %s' % (user or b'None', pid)
                     else:
                         locker = b'user %s, process %s, host %s' % (
                             user or b'None',
                             pid,
                             host,
                         )
                 ui.writenoi18n(b"%-6s %s (%ds)\n" % (name + b":", locker, age))
                 return 1
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
 
         ui.writenoi18n(b"%-6s free\n" % (name + b":"))
         return 0
 
     held += report(repo.svfs, b"lock", repo.lock)
     held += report(repo.vfs, b"wlock", repo.wlock)
 
     return held
 
 
 @command(
     b'debugmanifestfulltextcache',
     [
         (b'', b'clear', False, _(b'clear the cache')),
         (
             b'a',
             b'add',
             [],
             _(b'add the given manifest nodes to the cache'),
             _(b'NODE'),
         ),
     ],
     b'',
 )
 def debugmanifestfulltextcache(ui, repo, add=(), **opts):
     """show, clear or amend the contents of the manifest fulltext cache"""
 
     def getcache():
         r = repo.manifestlog.getstorage(b'')
         try:
             return r._fulltextcache
         except AttributeError:
             msg = _(
                 b"Current revlog implementation doesn't appear to have a "
                 b"manifest fulltext cache\n"
             )
             raise error.Abort(msg)
 
     if opts.get('clear'):
         with repo.wlock():
             cache = getcache()
             cache.clear(clear_persisted_data=True)
             return
 
     if add:
         with repo.wlock():
             m = repo.manifestlog
             store = m.getstorage(b'')
             for n in add:
                 try:
                     manifest = m[store.lookup(n)]
                 except error.LookupError as e:
                     raise error.Abort(e, hint=b"Check your manifest node id")
                 manifest.read()  # stores revisision in cache too
             return
 
     cache = getcache()
     if not len(cache):
         ui.write(_(b'cache empty\n'))
     else:
         ui.write(
             _(
                 b'cache contains %d manifest entries, in order of most to '
                 b'least recent:\n'
             )
             % (len(cache),)
         )
         totalsize = 0
         for nodeid in cache:
             # Use cache.get to not update the LRU order
             data = cache.peek(nodeid)
             size = len(data)
             totalsize += size + 24  # 20 bytes nodeid, 4 bytes size
             ui.write(
                 _(b'id: %s, size %s\n') % (hex(nodeid), util.bytecount(size))
             )
         ondisk = cache._opener.stat(b'manifestfulltextcache').st_size
         ui.write(
             _(b'total cache data size %s, on-disk %s\n')
             % (util.bytecount(totalsize), util.bytecount(ondisk))
         )
 
 
 @command(b'debugmergestate', [] + cmdutil.templateopts, b'')
 def debugmergestate(ui, repo, *args, **opts):
     """print merge state
 
     Use --verbose to print out information about whether v1 or v2 merge state
     was chosen."""
 
     if ui.verbose:
         ms = mergestatemod.mergestate(repo)
 
         # sort so that reasonable information is on top
         v1records = ms._readrecordsv1()
         v2records = ms._readrecordsv2()
 
         if not v1records and not v2records:
             pass
         elif not v2records:
             ui.writenoi18n(b'no version 2 merge state\n')
         elif ms._v1v2match(v1records, v2records):
             ui.writenoi18n(b'v1 and v2 states match: using v2\n')
         else:
             ui.writenoi18n(b'v1 and v2 states mismatch: using v1\n')
 
     opts = pycompat.byteskwargs(opts)
     if not opts[b'template']:
         opts[b'template'] = (
             b'{if(commits, "", "no merge state found\n")}'
             b'{commits % "{name}{if(label, " ({label})")}: {node}\n"}'
             b'{files % "file: {path} (state \\"{state}\\")\n'
             b'{if(local_path, "'
             b'  local path: {local_path} (hash {local_key}, flags \\"{local_flags}\\")\n'
             b'  ancestor path: {ancestor_path} (node {ancestor_node})\n'
             b'  other path: {other_path} (node {other_node})\n'
             b'")}'
             b'{if(rename_side, "'
             b'  rename side: {rename_side}\n'
             b'  renamed path: {renamed_path}\n'
             b'")}'
             b'{extras % "  extra: {key} = {value}\n"}'
             b'"}'
             b'{extras % "extra: {file} ({key} = {value})\n"}'
         )
 
     ms = mergestatemod.mergestate.read(repo)
 
     fm = ui.formatter(b'debugmergestate', opts)
     fm.startitem()
 
     fm_commits = fm.nested(b'commits')
     if ms.active():
         for name, node, label_index in (
             (b'local', ms.local, 0),
             (b'other', ms.other, 1),
         ):
             fm_commits.startitem()
             fm_commits.data(name=name)
             fm_commits.data(node=hex(node))
             if ms._labels and len(ms._labels) > label_index:
                 fm_commits.data(label=ms._labels[label_index])
     fm_commits.end()
 
     fm_files = fm.nested(b'files')
     if ms.active():
         for f in ms:
             fm_files.startitem()
             fm_files.data(path=f)
             state = ms._state[f]
             fm_files.data(state=state[0])
             if state[0] in (
                 mergestatemod.MERGE_RECORD_UNRESOLVED,
                 mergestatemod.MERGE_RECORD_RESOLVED,
             ):
                 fm_files.data(local_key=state[1])
                 fm_files.data(local_path=state[2])
                 fm_files.data(ancestor_path=state[3])
                 fm_files.data(ancestor_node=state[4])
                 fm_files.data(other_path=state[5])
                 fm_files.data(other_node=state[6])
                 fm_files.data(local_flags=state[7])
             elif state[0] in (
                 mergestatemod.MERGE_RECORD_UNRESOLVED_PATH,
                 mergestatemod.MERGE_RECORD_RESOLVED_PATH,
             ):
                 fm_files.data(renamed_path=state[1])
                 fm_files.data(rename_side=state[2])
             fm_extras = fm_files.nested(b'extras')
             for k, v in sorted(ms.extras(f).items()):
                 fm_extras.startitem()
                 fm_extras.data(key=k)
                 fm_extras.data(value=v)
             fm_extras.end()
 
     fm_files.end()
 
     fm_extras = fm.nested(b'extras')
     for f, d in sorted(pycompat.iteritems(ms.allextras())):
         if f in ms:
             # If file is in mergestate, we have already processed it's extras
             continue
         for k, v in pycompat.iteritems(d):
             fm_extras.startitem()
             fm_extras.data(file=f)
             fm_extras.data(key=k)
             fm_extras.data(value=v)
     fm_extras.end()
 
     fm.end()
 
 
 @command(b'debugnamecomplete', [], _(b'NAME...'))
 def debugnamecomplete(ui, repo, *args):
     '''complete "names" - tags, open branch names, bookmark names'''
 
     names = set()
     # since we previously only listed open branches, we will handle that
     # specially (after this for loop)
     for name, ns in pycompat.iteritems(repo.names):
         if name != b'branches':
             names.update(ns.listnames(repo))
     names.update(
         tag
         for (tag, heads, tip, closed) in repo.branchmap().iterbranches()
         if not closed
     )
     completions = set()
     if not args:
         args = [b'']
     for a in args:
         completions.update(n for n in names if n.startswith(a))
     ui.write(b'\n'.join(sorted(completions)))
     ui.write(b'\n')
 
 
 @command(
     b'debugnodemap',
     [
         (
             b'',
             b'dump-new',
             False,
             _(b'write a (new) persistent binary nodemap on stdout'),
         ),
         (b'', b'dump-disk', False, _(b'dump on-disk data on stdout')),
         (
             b'',
             b'check',
             False,
             _(b'check that the data on disk data are correct.'),
         ),
         (
             b'',
             b'metadata',
             False,
             _(b'display the on disk meta data for the nodemap'),
         ),
     ],
 )
 def debugnodemap(ui, repo, **opts):
     """write and inspect on disk nodemap"""
     if opts['dump_new']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
         if util.safehasattr(cl.index, "nodemap_data_all"):
             data = cl.index.nodemap_data_all()
         else:
             data = nodemap.persistent_data(cl.index)
         ui.write(data)
     elif opts['dump_disk']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
             ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
             return nodemap.check_data(ui, cl.index, data)
     elif opts['metadata']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
             ui.write((b"uid: %s\n") % docket.uid)
             ui.write((b"tip-rev: %d\n") % docket.tip_rev)
             ui.write((b"tip-node: %s\n") % hex(docket.tip_node))
             ui.write((b"data-length: %d\n") % docket.data_length)
             ui.write((b"data-unused: %d\n") % docket.data_unused)
             unused_perc = docket.data_unused * 100.0 / docket.data_length
             ui.write((b"data-unused: %2.3f%%\n") % unused_perc)
 
 
 @command(
     b'debugobsolete',
     [
         (b'', b'flags', 0, _(b'markers flag')),
         (
             b'',
             b'record-parents',
             False,
             _(b'record parent information for the precursor'),
         ),
         (b'r', b'rev', [], _(b'display markers relevant to REV')),
         (
             b'',
             b'exclusive',
             False,
             _(b'restrict display to markers only relevant to REV'),
         ),
         (b'', b'index', False, _(b'display index of the marker')),
         (b'', b'delete', [], _(b'delete markers specified by indices')),
     ]
     + cmdutil.commitopts2
     + cmdutil.formatteropts,
     _(b'[OBSOLETED [REPLACEMENT ...]]'),
 )
 def debugobsolete(ui, repo, precursor=None, *successors, **opts):
     """create arbitrary obsolete marker
 
     With no arguments, displays the list of obsolescence markers."""
 
     opts = pycompat.byteskwargs(opts)
 
     def parsenodeid(s):
         try:
             # We do not use revsingle/revrange functions here to accept
             # arbitrary node identifiers, possibly not present in the
             # local repository.
             n = bin(s)
             if len(n) != len(nullid):
                 raise TypeError()
             return n
         except TypeError:
             raise error.InputError(
                 b'changeset references must be full hexadecimal '
                 b'node identifiers'
             )
 
     if opts.get(b'delete'):
         indices = []
         for v in opts.get(b'delete'):
             try:
                 indices.append(int(v))
             except ValueError:
                 raise error.InputError(
                     _(b'invalid index value: %r') % v,
                     hint=_(b'use integers for indices'),
                 )
 
         if repo.currenttransaction():
             raise error.Abort(
                 _(b'cannot delete obsmarkers in the middle of transaction.')
             )
 
         with repo.lock():
             n = repair.deleteobsmarkers(repo.obsstore, indices)
             ui.write(_(b'deleted %i obsolescence markers\n') % n)
 
         return
 
     if precursor is not None:
         if opts[b'rev']:
             raise error.InputError(
                 b'cannot select revision when creating marker'
             )
         metadata = {}
         metadata[b'user'] = encoding.fromlocal(opts[b'user'] or ui.username())
         succs = tuple(parsenodeid(succ) for succ in successors)
         l = repo.lock()
         try:
             tr = repo.transaction(b'debugobsolete')
             try:
                 date = opts.get(b'date')
                 if date:
                     date = dateutil.parsedate(date)
                 else:
                     date = None
                 prec = parsenodeid(precursor)
                 parents = None
                 if opts[b'record_parents']:
                     if prec not in repo.unfiltered():
                         raise error.Abort(
                             b'cannot used --record-parents on '
                             b'unknown changesets'
                         )
                     parents = repo.unfiltered()[prec].parents()
                     parents = tuple(p.node() for p in parents)
                 repo.obsstore.create(
                     tr,
                     prec,
                     succs,
                     opts[b'flags'],
                     parents=parents,
                     date=date,
                     metadata=metadata,
                     ui=ui,
                 )
                 tr.close()
             except ValueError as exc:
                 raise error.Abort(
                     _(b'bad obsmarker input: %s') % pycompat.bytestr(exc)
                 )
             finally:
                 tr.release()
         finally:
             l.release()
     else:
         if opts[b'rev']:
             revs = scmutil.revrange(repo, opts[b'rev'])
             nodes = [repo[r].node() for r in revs]
             markers = list(
                 obsutil.getmarkers(
                     repo, nodes=nodes, exclusive=opts[b'exclusive']
                 )
             )
             markers.sort(key=lambda x: x._data)
         else:
             markers = obsutil.getmarkers(repo)
 
         markerstoiter = markers
         isrelevant = lambda m: True
         if opts.get(b'rev') and opts.get(b'index'):
             markerstoiter = obsutil.getmarkers(repo)
             markerset = set(markers)
             isrelevant = lambda m: m in markerset
 
         fm = ui.formatter(b'debugobsolete', opts)
         for i, m in enumerate(markerstoiter):
             if not isrelevant(m):
                 # marker can be irrelevant when we're iterating over a set
                 # of markers (markerstoiter) which is bigger than the set
                 # of markers we want to display (markers)
                 # this can happen if both --index and --rev options are
                 # provided and thus we need to iterate over all of the markers
                 # to get the correct indices, but only display the ones that
                 # are relevant to --rev value
                 continue
             fm.startitem()
             ind = i if opts.get(b'index') else None
             cmdutil.showmarker(fm, m, index=ind)
         fm.end()
 
 
 @command(
     b'debugp1copies',
     [(b'r', b'rev', b'', _(b'revision to debug'), _(b'REV'))],
     _(b'[-r REV]'),
 )
 def debugp1copies(ui, repo, **opts):
     """dump copy information compared to p1"""
 
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get(b'rev'), default=None)
     for dst, src in ctx.p1copies().items():
         ui.write(b'%s -> %s\n' % (src, dst))
 
 
 @command(
     b'debugp2copies',
     [(b'r', b'rev', b'', _(b'revision to debug'), _(b'REV'))],
     _(b'[-r REV]'),
 )
 def debugp1copies(ui, repo, **opts):
     """dump copy information compared to p2"""
 
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get(b'rev'), default=None)
     for dst, src in ctx.p2copies().items():
         ui.write(b'%s -> %s\n' % (src, dst))
 
 
 @command(
     b'debugpathcomplete',
     [
         (b'f', b'full', None, _(b'complete an entire path')),
         (b'n', b'normal', None, _(b'show only normal files')),
         (b'a', b'added', None, _(b'show only added files')),
         (b'r', b'removed', None, _(b'show only removed files')),
     ],
     _(b'FILESPEC...'),
 )
 def debugpathcomplete(ui, repo, *specs, **opts):
     """complete part or all of a tracked path
 
     This command supports shells that offer path name completion. It
     currently completes only files already known to the dirstate.
 
     Completion extends only to the next path segment unless
     --full is specified, in which case entire paths are used."""
 
     def complete(path, acceptable):
         dirstate = repo.dirstate
         spec = os.path.normpath(os.path.join(encoding.getcwd(), path))
         rootdir = repo.root + pycompat.ossep
         if spec != repo.root and not spec.startswith(rootdir):
             return [], []
         if os.path.isdir(spec):
             spec += b'/'
         spec = spec[len(rootdir) :]
         fixpaths = pycompat.ossep != b'/'
         if fixpaths:
             spec = spec.replace(pycompat.ossep, b'/')
         speclen = len(spec)
         fullpaths = opts['full']
         files, dirs = set(), set()
         adddir, addfile = dirs.add, files.add
         for f, st in pycompat.iteritems(dirstate):
             if f.startswith(spec) and st[0] in acceptable:
                 if fixpaths:
                     f = f.replace(b'/', pycompat.ossep)
                 if fullpaths:
                     addfile(f)
                     continue
                 s = f.find(pycompat.ossep, speclen)
                 if s >= 0:
                     adddir(f[:s])
                 else:
                     addfile(f)
         return files, dirs
 
     acceptable = b''
     if opts['normal']:
         acceptable += b'nm'
     if opts['added']:
         acceptable += b'a'
     if opts['removed']:
         acceptable += b'r'
     cwd = repo.getcwd()
     if not specs:
         specs = [b'.']
 
     files, dirs = set(), set()
     for spec in specs:
         f, d = complete(spec, acceptable or b'nmar')
         files.update(f)
         dirs.update(d)
     files.update(dirs)
     ui.write(b'\n'.join(repo.pathto(p, cwd) for p in sorted(files)))
     ui.write(b'\n')
 
 
 @command(
     b'debugpathcopies',
     cmdutil.walkopts,
     b'hg debugpathcopies REV1 REV2 [FILE]',
     inferrepo=True,
 )
 def debugpathcopies(ui, repo, rev1, rev2, *pats, **opts):
     """show copies between two revisions"""
     ctx1 = scmutil.revsingle(repo, rev1)
     ctx2 = scmutil.revsingle(repo, rev2)
     m = scmutil.match(ctx1, pats, opts)
     for dst, src in sorted(copies.pathcopies(ctx1, ctx2, m).items()):
         ui.write(b'%s -> %s\n' % (src, dst))
 
 
 @command(b'debugpeer', [], _(b'PATH'), norepo=True)
 def debugpeer(ui, path):
     """establish a connection to a peer repository"""
     # Always enable peer request logging. Requires --debug to display
     # though.
     overrides = {
         (b'devel', b'debug.peer-request'): True,
     }
 
     with ui.configoverride(overrides):
         peer = hg.peer(ui, {}, path)
 
         try:
             local = peer.local() is not None
             canpush = peer.canpush()
 
             ui.write(_(b'url: %s\n') % peer.url())
             ui.write(_(b'local: %s\n') % (_(b'yes') if local else _(b'no')))
             ui.write(
                 _(b'pushable: %s\n') % (_(b'yes') if canpush else _(b'no'))
             )
         finally:
             peer.close()
 
 
 @command(
     b'debugpickmergetool',
     [
         (b'r', b'rev', b'', _(b'check for files in this revision'), _(b'REV')),
         (b'', b'changedelete', None, _(b'emulate merging change and delete')),
     ]
     + cmdutil.walkopts
     + cmdutil.mergetoolopts,
     _(b'[PATTERN]...'),
     inferrepo=True,
 )
 def debugpickmergetool(ui, repo, *pats, **opts):
     """examine which merge tool is chosen for specified file
 
     As described in :hg:`help merge-tools`, Mercurial examines
     configurations below in this order to decide which merge tool is
     chosen for specified file.
 
     1. ``--tool`` option
     2. ``HGMERGE`` environment variable
     3. configurations in ``merge-patterns`` section
     4. configuration of ``ui.merge``
     5. configurations in ``merge-tools`` section
     6. ``hgmerge`` tool (for historical reason only)
     7. default tool for fallback (``:merge`` or ``:prompt``)
 
     This command writes out examination result in the style below::
 
         FILE = MERGETOOL
 
     By default, all files known in the first parent context of the
     working directory are examined. Use file patterns and/or -I/-X
     options to limit target files. -r/--rev is also useful to examine
     files in another context without actual updating to it.
 
     With --debug, this command shows warning messages while matching
     against ``merge-patterns`` and so on, too. It is recommended to
     use this option with explicit file patterns and/or -I/-X options,
     because this option increases amount of output per file according
     to configurations in hgrc.
 
     With -v/--verbose, this command shows configurations below at
     first (only if specified).
 
     - ``--tool`` option
     - ``HGMERGE`` environment variable
     - configuration of ``ui.merge``
 
     If merge tool is chosen before matching against
     ``merge-patterns``, this command can't show any helpful
     information, even with --debug. In such case, information above is
     useful to know why a merge tool is chosen.
     """
     opts = pycompat.byteskwargs(opts)
     overrides = {}
     if opts[b'tool']:
         overrides[(b'ui', b'forcemerge')] = opts[b'tool']
         ui.notenoi18n(b'with --tool %r\n' % (pycompat.bytestr(opts[b'tool'])))
 
     with ui.configoverride(overrides, b'debugmergepatterns'):
         hgmerge = encoding.environ.get(b"HGMERGE")
         if hgmerge is not None:
             ui.notenoi18n(b'with HGMERGE=%r\n' % (pycompat.bytestr(hgmerge)))
         uimerge = ui.config(b"ui", b"merge")
         if uimerge:
             ui.notenoi18n(b'with ui.merge=%r\n' % (pycompat.bytestr(uimerge)))
 
         ctx = scmutil.revsingle(repo, opts.get(b'rev'))
         m = scmutil.match(ctx, pats, opts)
         changedelete = opts[b'changedelete']
         for path in ctx.walk(m):
             fctx = ctx[path]
             try:
                 if not ui.debugflag:
                     ui.pushbuffer(error=True)
                 tool, toolpath = filemerge._picktool(
                     repo,
                     ui,
                     path,
                     fctx.isbinary(),
                     b'l' in fctx.flags(),
                     changedelete,
                 )
             finally:
                 if not ui.debugflag:
                     ui.popbuffer()
             ui.write(b'%s = %s\n' % (path, tool))
 
 
 @command(b'debugpushkey', [], _(b'REPO NAMESPACE [KEY OLD NEW]'), norepo=True)
 def debugpushkey(ui, repopath, namespace, *keyinfo, **opts):
     """access the pushkey key/value protocol
 
     With two args, list the keys in the given namespace.
 
     With five args, set a key to new if it currently is set to old.
     Reports success or failure.
     """
 
     target = hg.peer(ui, {}, repopath)
     try:
         if keyinfo:
             key, old, new = keyinfo
             with target.commandexecutor() as e:
                 r = e.callcommand(
                     b'pushkey',
                     {
                         b'namespace': namespace,
                         b'key': key,
                         b'old': old,
                         b'new': new,
                     },
                 ).result()
 
             ui.status(pycompat.bytestr(r) + b'\n')
             return not r
         else:
             for k, v in sorted(pycompat.iteritems(target.listkeys(namespace))):
                 ui.write(
                     b"%s\t%s\n"
                     % (stringutil.escapestr(k), stringutil.escapestr(v))
                 )
     finally:
         target.close()
 
 
 @command(b'debugpvec', [], _(b'A B'))
 def debugpvec(ui, repo, a, b=None):
     ca = scmutil.revsingle(repo, a)
     cb = scmutil.revsingle(repo, b)
     pa = pvec.ctxpvec(ca)
     pb = pvec.ctxpvec(cb)
     if pa == pb:
         rel = b"="
     elif pa > pb:
         rel = b">"
     elif pa < pb:
         rel = b"<"
     elif pa | pb:
         rel = b"|"
     ui.write(_(b"a: %s\n") % pa)
     ui.write(_(b"b: %s\n") % pb)
     ui.write(_(b"depth(a): %d depth(b): %d\n") % (pa._depth, pb._depth))
     ui.write(
         _(b"delta: %d hdist: %d distance: %d relation: %s\n")
         % (
             abs(pa._depth - pb._depth),
             pvec._hamming(pa._vec, pb._vec),
             pa.distance(pb),
             rel,
         )
     )
 
 
 @command(
     b'debugrebuilddirstate|debugrebuildstate',
     [
         (b'r', b'rev', b'', _(b'revision to rebuild to'), _(b'REV')),
         (
             b'',
             b'minimal',
             None,
             _(
                 b'only rebuild files that are inconsistent with '
                 b'the working copy parent'
             ),
         ),
     ],
     _(b'[-r REV]'),
 )
 def debugrebuilddirstate(ui, repo, rev, **opts):
     """rebuild the dirstate as it would look like for the given revision
 
     If no revision is specified the first current parent will be used.
 
     The dirstate will be set to the files of the given revision.
     The actual working directory content or existing dirstate
     information such as adds or removes is not considered.
 
     ``minimal`` will only rebuild the dirstate status for files that claim to be
     tracked but are not in the parent manifest, or that exist in the parent
     manifest but are not in the dirstate. It will not change adds, removes, or
     modified files that are in the working copy parent.
 
     One use of this command is to make the next :hg:`status` invocation
     check the actual file content.
     """
     ctx = scmutil.revsingle(repo, rev)
     with repo.wlock():
         dirstate = repo.dirstate
         changedfiles = None
         # See command doc for what minimal does.
         if opts.get('minimal'):
             manifestfiles = set(ctx.manifest().keys())
             dirstatefiles = set(dirstate)
             manifestonly = manifestfiles - dirstatefiles
             dsonly = dirstatefiles - manifestfiles
             dsnotadded = {f for f in dsonly if dirstate[f] != b'a'}
             changedfiles = manifestonly | dsnotadded
 
         dirstate.rebuild(ctx.node(), ctx.manifest(), changedfiles)
 
 
 @command(b'debugrebuildfncache', [], b'')
 def debugrebuildfncache(ui, repo):
     """rebuild the fncache file"""
     repair.rebuildfncache(ui, repo)
 
 
 @command(
     b'debugrename',
     [(b'r', b'rev', b'', _(b'revision to debug'), _(b'REV'))],
     _(b'[-r REV] [FILE]...'),
 )
 def debugrename(ui, repo, *pats, **opts):
     """dump rename information"""
 
     opts = pycompat.byteskwargs(opts)
     ctx = scmutil.revsingle(repo, opts.get(b'rev'))
     m = scmutil.match(ctx, pats, opts)
     for abs in ctx.walk(m):
         fctx = ctx[abs]
         o = fctx.filelog().renamed(fctx.filenode())
         rel = repo.pathto(abs)
         if o:
             ui.write(_(b"%s renamed from %s:%s\n") % (rel, o[0], hex(o[1])))
         else:
             ui.write(_(b"%s not renamed\n") % rel)
 
 
 @command(b'debugrequires|debugrequirements', [], b'')
 def debugrequirements(ui, repo):
     """ print the current repo requirements """
     for r in sorted(repo.requirements):
         ui.write(b"%s\n" % r)
 
 
 @command(
     b'debugrevlog',
     cmdutil.debugrevlogopts + [(b'd', b'dump', False, _(b'dump index data'))],
     _(b'-c|-m|FILE'),
     optionalrepo=True,
 )
 def debugrevlog(ui, repo, file_=None, **opts):
     """show data and statistics about a revlog"""
     opts = pycompat.byteskwargs(opts)
     r = cmdutil.openrevlog(repo, b'debugrevlog', file_, opts)
 
     if opts.get(b"dump"):
         numrevs = len(r)
         ui.write(
             (
                 b"# rev p1rev p2rev start   end deltastart base   p1   p2"
                 b" rawsize totalsize compression heads chainlen\n"
             )
         )
         ts = 0
         heads = set()
 
         for rev in pycompat.xrange(numrevs):
             dbase = r.deltaparent(rev)
             if dbase == -1:
                 dbase = rev
             cbase = r.chainbase(rev)
             clen = r.chainlen(rev)
             p1, p2 = r.parentrevs(rev)
             rs = r.rawsize(rev)
             ts = ts + rs
             heads -= set(r.parentrevs(rev))
             heads.add(rev)
             try:
                 compression = ts / r.end(rev)
             except ZeroDivisionError:
                 compression = 0
             ui.write(
                 b"%5d %5d %5d %5d %5d %10d %4d %4d %4d %7d %9d "
                 b"%11d %5d %8d\n"
                 % (
                     rev,
                     p1,
                     p2,
                     r.start(rev),
                     r.end(rev),
                     r.start(dbase),
                     r.start(cbase),
                     r.start(p1),
                     r.start(p2),
                     rs,
                     ts,
                     compression,
                     len(heads),
                     clen,
                 )
             )
         return 0
 
     v = r.version
     format = v & 0xFFFF
     flags = []
     gdelta = False
     if v & revlog.FLAG_INLINE_DATA:
         flags.append(b'inline')
     if v & revlog.FLAG_GENERALDELTA:
         gdelta = True
         flags.append(b'generaldelta')
     if not flags:
         flags = [b'(none)']
 
     ### tracks merge vs single parent
     nummerges = 0
 
     ### tracks ways the "delta" are build
     # nodelta
     numempty = 0
     numemptytext = 0
     numemptydelta = 0
     # full file content
     numfull = 0
     # intermediate snapshot against a prior snapshot
     numsemi = 0
     # snapshot count per depth
     numsnapdepth = collections.defaultdict(lambda: 0)
     # delta against previous revision
     numprev = 0
     # delta against first or second parent (not prev)
     nump1 = 0
     nump2 = 0
     # delta against neither prev nor parents
     numother = 0
     # delta against prev that are also first or second parent
     # (details of `numprev`)
     nump1prev = 0
     nump2prev = 0
 
     # data about delta chain of each revs
     chainlengths = []
     chainbases = []
     chainspans = []
 
     # data about each revision
     datasize = [None, 0, 0]
     fullsize = [None, 0, 0]
     semisize = [None, 0, 0]
     # snapshot count per depth
     snapsizedepth = collections.defaultdict(lambda: [None, 0, 0])
     deltasize = [None, 0, 0]
     chunktypecounts = {}
     chunktypesizes = {}
 
     def addsize(size, l):
         if l[0] is None or size < l[0]:
             l[0] = size
         if size > l[1]:
             l[1] = size
         l[2] += size
 
     numrevs = len(r)
     for rev in pycompat.xrange(numrevs):
         p1, p2 = r.parentrevs(rev)
         delta = r.deltaparent(rev)
         if format > 0:
             addsize(r.rawsize(rev), datasize)
         if p2 != nullrev:
             nummerges += 1
         size = r.length(rev)
         if delta == nullrev:
             chainlengths.append(0)
             chainbases.append(r.start(rev))
             chainspans.append(size)
             if size == 0:
                 numempty += 1
                 numemptytext += 1
             else:
                 numfull += 1
                 numsnapdepth[0] += 1
                 addsize(size, fullsize)
                 addsize(size, snapsizedepth[0])
         else:
             chainlengths.append(chainlengths[delta] + 1)
             baseaddr = chainbases[delta]
             revaddr = r.start(rev)
             chainbases.append(baseaddr)
             chainspans.append((revaddr - baseaddr) + size)
             if size == 0:
                 numempty += 1
                 numemptydelta += 1
             elif r.issnapshot(rev):
                 addsize(size, semisize)
                 numsemi += 1
                 depth = r.snapshotdepth(rev)
                 numsnapdepth[depth] += 1
                 addsize(size, snapsizedepth[depth])
             else:
                 addsize(size, deltasize)
                 if delta == rev - 1:
                     numprev += 1
                     if delta == p1:
                         nump1prev += 1
                     elif delta == p2:
                         nump2prev += 1
                 elif delta == p1:
                     nump1 += 1
                 elif delta == p2:
                     nump2 += 1
                 elif delta != nullrev:
                     numother += 1
 
         # Obtain data on the raw chunks in the revlog.
         if util.safehasattr(r, b'_getsegmentforrevs'):
             segment = r._getsegmentforrevs(rev, rev)[1]
         else:
             segment = r._revlog._getsegmentforrevs(rev, rev)[1]
         if segment:
             chunktype = bytes(segment[0:1])
         else:
             chunktype = b'empty'
 
         if chunktype not in chunktypecounts:
             chunktypecounts[chunktype] = 0
             chunktypesizes[chunktype] = 0
 
         chunktypecounts[chunktype] += 1
         chunktypesizes[chunktype] += size
 
     # Adjust size min value for empty cases
     for size in (datasize, fullsize, semisize, deltasize):
         if size[0] is None:
             size[0] = 0
 
     numdeltas = numrevs - numfull - numempty - numsemi
     numoprev = numprev - nump1prev - nump2prev
     totalrawsize = datasize[2]
     datasize[2] /= numrevs
     fulltotal = fullsize[2]
     if numfull == 0:
         fullsize[2] = 0
     else:
         fullsize[2] /= numfull
     semitotal = semisize[2]
     snaptotal = {}
     if numsemi > 0:
         semisize[2] /= numsemi
     for depth in snapsizedepth:
         snaptotal[depth] = snapsizedepth[depth][2]
         snapsizedepth[depth][2] /= numsnapdepth[depth]
 
     deltatotal = deltasize[2]
     if numdeltas > 0:
         deltasize[2] /= numdeltas
     totalsize = fulltotal + semitotal + deltatotal
     avgchainlen = sum(chainlengths) / numrevs
     maxchainlen = max(chainlengths)
     maxchainspan = max(chainspans)
     compratio = 1
     if totalsize:
         compratio = totalrawsize / totalsize
 
     basedfmtstr = b'%%%dd\n'
     basepcfmtstr = b'%%%dd %s(%%5.2f%%%%)\n'
 
     def dfmtstr(max):
         return basedfmtstr % len(str(max))
 
     def pcfmtstr(max, padding=0):
         return basepcfmtstr % (len(str(max)), b' ' * padding)
 
     def pcfmt(value, total):
         if total:
             return (value, 100 * float(value) / total)
         else:
             return value, 100.0
 
     ui.writenoi18n(b'format : %d\n' % format)
     ui.writenoi18n(b'flags  : %s\n' % b', '.join(flags))
 
     ui.write(b'\n')
     fmt = pcfmtstr(totalsize)
     fmt2 = dfmtstr(totalsize)
     ui.writenoi18n(b'revisions     : ' + fmt2 % numrevs)
     ui.writenoi18n(b'    merges    : ' + fmt % pcfmt(nummerges, numrevs))
     ui.writenoi18n(
         b'    normal    : ' + fmt % pcfmt(numrevs - nummerges, numrevs)
     )
     ui.writenoi18n(b'revisions     : ' + fmt2 % numrevs)
     ui.writenoi18n(b'    empty     : ' + fmt % pcfmt(numempty, numrevs))
     ui.writenoi18n(
         b'                   text  : '
         + fmt % pcfmt(numemptytext, numemptytext + numemptydelta)
     )
     ui.writenoi18n(
         b'                   delta : '
         + fmt % pcfmt(numemptydelta, numemptytext + numemptydelta)
     )
     ui.writenoi18n(
         b'    snapshot  : ' + fmt % pcfmt(numfull + numsemi, numrevs)
     )
     for depth in sorted(numsnapdepth):
         ui.write(
             (b'      lvl-%-3d :       ' % depth)
             + fmt % pcfmt(numsnapdepth[depth], numrevs)
         )
     ui.writenoi18n(b'    deltas    : ' + fmt % pcfmt(numdeltas, numrevs))
     ui.writenoi18n(b'revision size : ' + fmt2 % totalsize)
     ui.writenoi18n(
         b'    snapshot  : ' + fmt % pcfmt(fulltotal + semitotal, totalsize)
     )
     for depth in sorted(numsnapdepth):
         ui.write(
             (b'      lvl-%-3d :       ' % depth)
             + fmt % pcfmt(snaptotal[depth], totalsize)
         )
     ui.writenoi18n(b'    deltas    : ' + fmt % pcfmt(deltatotal, totalsize))
 
     def fmtchunktype(chunktype):
         if chunktype == b'empty':
             return b'    %s     : ' % chunktype
         elif chunktype in pycompat.bytestr(string.ascii_letters):
             return b'    0x%s (%s)  : ' % (hex(chunktype), chunktype)
         else:
             return b'    0x%s      : ' % hex(chunktype)
 
     ui.write(b'\n')
     ui.writenoi18n(b'chunks        : ' + fmt2 % numrevs)
     for chunktype in sorted(chunktypecounts):
         ui.write(fmtchunktype(chunktype))
         ui.write(fmt % pcfmt(chunktypecounts[chunktype], numrevs))
     ui.writenoi18n(b'chunks size   : ' + fmt2 % totalsize)
     for chunktype in sorted(chunktypecounts):
         ui.write(fmtchunktype(chunktype))
         ui.write(fmt % pcfmt(chunktypesizes[chunktype], totalsize))
 
     ui.write(b'\n')
     fmt = dfmtstr(max(avgchainlen, maxchainlen, maxchainspan, compratio))
     ui.writenoi18n(b'avg chain length  : ' + fmt % avgchainlen)
     ui.writenoi18n(b'max chain length  : ' + fmt % maxchainlen)
     ui.writenoi18n(b'max chain reach   : ' + fmt % maxchainspan)
     ui.writenoi18n(b'compression ratio : ' + fmt % compratio)
 
     if format > 0:
         ui.write(b'\n')
         ui.writenoi18n(
             b'uncompressed data size (min/max/avg) : %d / %d / %d\n'
             % tuple(datasize)
         )
     ui.writenoi18n(
         b'full revision size (min/max/avg)     : %d / %d / %d\n'
         % tuple(fullsize)
     )
     ui.writenoi18n(
         b'inter-snapshot size (min/max/avg)    : %d / %d / %d\n'
         % tuple(semisize)
     )
     for depth in sorted(snapsizedepth):
         if depth == 0:
             continue
         ui.writenoi18n(
             b'    level-%-3d (min/max/avg)          : %d / %d / %d\n'
             % ((depth,) + tuple(snapsizedepth[depth]))
         )
     ui.writenoi18n(
         b'delta size (min/max/avg)             : %d / %d / %d\n'
         % tuple(deltasize)
     )
 
     if numdeltas > 0:
         ui.write(b'\n')
         fmt = pcfmtstr(numdeltas)
         fmt2 = pcfmtstr(numdeltas, 4)
         ui.writenoi18n(
             b'deltas against prev  : ' + fmt % pcfmt(numprev, numdeltas)
         )
         if numprev > 0:
             ui.writenoi18n(
                 b'    where prev = p1  : ' + fmt2 % pcfmt(nump1prev, numprev)
             )
             ui.writenoi18n(
                 b'    where prev = p2  : ' + fmt2 % pcfmt(nump2prev, numprev)
             )
             ui.writenoi18n(
                 b'    other            : ' + fmt2 % pcfmt(numoprev, numprev)
             )
         if gdelta:
             ui.writenoi18n(
                 b'deltas against p1    : ' + fmt % pcfmt(nump1, numdeltas)
             )
             ui.writenoi18n(
                 b'deltas against p2    : ' + fmt % pcfmt(nump2, numdeltas)
             )
             ui.writenoi18n(
                 b'deltas against other : ' + fmt % pcfmt(numother, numdeltas)
             )
 
 
 @command(
     b'debugrevlogindex',
     cmdutil.debugrevlogopts
     + [(b'f', b'format', 0, _(b'revlog format'), _(b'FORMAT'))],
     _(b'[-f FORMAT] -c|-m|FILE'),
     optionalrepo=True,
 )
 def debugrevlogindex(ui, repo, file_=None, **opts):
     """dump the contents of a revlog index"""
     opts = pycompat.byteskwargs(opts)
     r = cmdutil.openrevlog(repo, b'debugrevlogindex', file_, opts)
     format = opts.get(b'format', 0)
     if format not in (0, 1):
         raise error.Abort(_(b"unknown format %d") % format)
 
     if ui.debugflag:
         shortfn = hex
     else:
         shortfn = short
 
     # There might not be anything in r, so have a sane default
     idlen = 12
     for i in r:
         idlen = len(shortfn(r.node(i)))
         break
 
     if format == 0:
         if ui.verbose:
             ui.writenoi18n(
                 b"   rev    offset  length linkrev %s %s p2\n"
                 % (b"nodeid".ljust(idlen), b"p1".ljust(idlen))
             )
         else:
             ui.writenoi18n(
                 b"   rev linkrev %s %s p2\n"
                 % (b"nodeid".ljust(idlen), b"p1".ljust(idlen))
             )
     elif format == 1:
         if ui.verbose:
             ui.writenoi18n(
                 (
                     b"   rev flag   offset   length     size   link     p1"
                     b"     p2 %s\n"
                 )
                 % b"nodeid".rjust(idlen)
             )
         else:
             ui.writenoi18n(
                 b"   rev flag     size   link     p1     p2 %s\n"
                 % b"nodeid".rjust(idlen)
             )
 
     for i in r:
         node = r.node(i)
         if format == 0:
             try:
                 pp = r.parents(node)
             except Exception:
                 pp = [nullid, nullid]
             if ui.verbose:
                 ui.write(
                     b"% 6d % 9d % 7d % 7d %s %s %s\n"
                     % (
                         i,
                         r.start(i),
                         r.length(i),
                         r.linkrev(i),
                         shortfn(node),
                         shortfn(pp[0]),
                         shortfn(pp[1]),
                     )
                 )
             else:
                 ui.write(
                     b"% 6d % 7d %s %s %s\n"
                     % (
                         i,
                         r.linkrev(i),
                         shortfn(node),
                         shortfn(pp[0]),
                         shortfn(pp[1]),
                     )
                 )
         elif format == 1:
             pr = r.parentrevs(i)
             if ui.verbose:
                 ui.write(
                     b"% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d %s\n"
                     % (
                         i,
                         r.flags(i),
                         r.start(i),
                         r.length(i),
                         r.rawsize(i),
                         r.linkrev(i),
                         pr[0],
                         pr[1],
                         shortfn(node),
                     )
                 )
             else:
                 ui.write(
                     b"% 6d %04x % 8d % 6d % 6d % 6d %s\n"
                     % (
                         i,
                         r.flags(i),
                         r.rawsize(i),
                         r.linkrev(i),
                         pr[0],
                         pr[1],
                         shortfn(node),
                     )
                 )
 
 
 @command(
     b'debugrevspec',
     [
         (
             b'',
             b'optimize',
             None,
             _(b'print parsed tree after optimizing (DEPRECATED)'),
         ),
         (
             b'',
             b'show-revs',
             True,
             _(b'print list of result revisions (default)'),
         ),
         (
             b's',
             b'show-set',
             None,
             _(b'print internal representation of result set'),
         ),
         (
             b'p',
             b'show-stage',
             [],
             _(b'print parsed tree at the given stage'),
             _(b'NAME'),
         ),
         (b'', b'no-optimized', False, _(b'evaluate tree without optimization')),
         (b'', b'verify-optimized', False, _(b'verify optimized result')),
     ],
     b'REVSPEC',
 )
 def debugrevspec(ui, repo, expr, **opts):
     """parse and apply a revision specification
 
     Use -p/--show-stage option to print the parsed tree at the given stages.
     Use -p all to print tree at every stage.
 
     Use --no-show-revs option with -s or -p to print only the set
     representation or the parsed tree respectively.
 
     Use --verify-optimized to compare the optimized result with the unoptimized
     one. Returns 1 if the optimized result differs.
     """
     opts = pycompat.byteskwargs(opts)
     aliases = ui.configitems(b'revsetalias')
     stages = [
         (b'parsed', lambda tree: tree),
         (
             b'expanded',
             lambda tree: revsetlang.expandaliases(tree, aliases, ui.warn),
         ),
         (b'concatenated', revsetlang.foldconcat),
         (b'analyzed', revsetlang.analyze),
         (b'optimized', revsetlang.optimize),
     ]
     if opts[b'no_optimized']:
         stages = stages[:-1]
     if opts[b'verify_optimized'] and opts[b'no_optimized']:
         raise error.Abort(
             _(b'cannot use --verify-optimized with --no-optimized')
         )
     stagenames = {n for n, f in stages}
 
     showalways = set()
     showchanged = set()
     if ui.verbose and not opts[b'show_stage']:
         # show parsed tree by --verbose (deprecated)
         showalways.add(b'parsed')
         showchanged.update([b'expanded', b'concatenated'])
         if opts[b'optimize']:
             showalways.add(b'optimized')
     if opts[b'show_stage'] and opts[b'optimize']:
         raise error.Abort(_(b'cannot use --optimize with --show-stage'))
     if opts[b'show_stage'] == [b'all']:
         showalways.update(stagenames)
     else:
         for n in opts[b'show_stage']:
             if n not in stagenames:
                 raise error.Abort(_(b'invalid stage name: %s') % n)
         showalways.update(opts[b'show_stage'])
 
     treebystage = {}
     printedtree = None
     tree = revsetlang.parse(expr, lookup=revset.lookupfn(repo))
     for n, f in stages:
         treebystage[n] = tree = f(tree)
         if n in showalways or (n in showchanged and tree != printedtree):
             if opts[b'show_stage'] or n != b'parsed':
                 ui.write(b"* %s:\n" % n)
             ui.write(revsetlang.prettyformat(tree), b"\n")
             printedtree = tree
 
     if opts[b'verify_optimized']:
         arevs = revset.makematcher(treebystage[b'analyzed'])(repo)
         brevs = revset.makematcher(treebystage[b'optimized'])(repo)
         if opts[b'show_set'] or (opts[b'show_set'] is None and ui.verbose):
             ui.writenoi18n(
                 b"* analyzed set:\n", stringutil.prettyrepr(arevs), b"\n"
             )
             ui.writenoi18n(
                 b"* optimized set:\n", stringutil.prettyrepr(brevs), b"\n"
             )
         arevs = list(arevs)
         brevs = list(brevs)
         if arevs == brevs:
             return 0
         ui.writenoi18n(b'--- analyzed\n', label=b'diff.file_a')
         ui.writenoi18n(b'+++ optimized\n', label=b'diff.file_b')
         sm = difflib.SequenceMatcher(None, arevs, brevs)
         for tag, alo, ahi, blo, bhi in sm.get_opcodes():
             if tag in ('delete', 'replace'):
                 for c in arevs[alo:ahi]:
                     ui.write(b'-%d\n' % c, label=b'diff.deleted')
             if tag in ('insert', 'replace'):
                 for c in brevs[blo:bhi]:
                     ui.write(b'+%d\n' % c, label=b'diff.inserted')
             if tag == 'equal':
                 for c in arevs[alo:ahi]:
                     ui.write(b' %d\n' % c)
         return 1
 
     func = revset.makematcher(tree)
     revs = func(repo)
     if opts[b'show_set'] or (opts[b'show_set'] is None and ui.verbose):
         ui.writenoi18n(b"* set:\n", stringutil.prettyrepr(revs), b"\n")
     if not opts[b'show_revs']:
         return
     for c in revs:
         ui.write(b"%d\n" % c)
 
 
 @command(
     b'debugserve',
     [
         (
             b'',
             b'sshstdio',
             False,
             _(b'run an SSH server bound to process handles'),
         ),
         (b'', b'logiofd', b'', _(b'file descriptor to log server I/O to')),
         (b'', b'logiofile', b'', _(b'file to log server I/O to')),
     ],
     b'',
 )
 def debugserve(ui, repo, **opts):
     """run a server with advanced settings
 
     This command is similar to :hg:`serve`. It exists partially as a
     workaround to the fact that ``hg serve --stdio`` must have specific
     arguments for security reasons.
     """
     opts = pycompat.byteskwargs(opts)
 
     if not opts[b'sshstdio']:
         raise error.Abort(_(b'only --sshstdio is currently supported'))
 
     logfh = None
 
     if opts[b'logiofd'] and opts[b'logiofile']:
         raise error.Abort(_(b'cannot use both --logiofd and --logiofile'))
 
     if opts[b'logiofd']:
         # Ideally we would be line buffered. But line buffering in binary
         # mode isn't supported and emits a warning in Python 3.8+. Disabling
         # buffering could have performance impacts. But since this isn't
         # performance critical code, it should be fine.
         try:
             logfh = os.fdopen(int(opts[b'logiofd']), 'ab', 0)
         except OSError as e:
             if e.errno != errno.ESPIPE:
                 raise
             # can't seek a pipe, so `ab` mode fails on py3
             logfh = os.fdopen(int(opts[b'logiofd']), 'wb', 0)
     elif opts[b'logiofile']:
         logfh = open(opts[b'logiofile'], b'ab', 0)
 
     s = wireprotoserver.sshserver(ui, repo, logfh=logfh)
     s.serve_forever()
 
 
 @command(b'debugsetparents', [], _(b'REV1 [REV2]'))
 def debugsetparents(ui, repo, rev1, rev2=None):
     """manually set the parents of the current working directory (DANGEROUS)
 
     This command is not what you are looking for and should not be used. Using
     this command will most certainly results in slight corruption of the file
     level histories withing your repository. DO NOT USE THIS COMMAND.
 
     The command update the p1 and p2 field in the dirstate, and not touching
     anything else. This useful for writing repository conversion tools, but
     should be used with extreme care. For example, neither the working
     directory nor the dirstate is updated, so file status may be incorrect
     after running this command. Only used if you are one of the few people that
     deeply unstand both conversion tools and file level histories. If you are
     reading this help, you are not one of this people (most of them sailed west
     from Mithlond anyway.
 
     So one last time DO NOT USE THIS COMMAND.
 
     Returns 0 on success.
     """
 
     node1 = scmutil.revsingle(repo, rev1).node()
     node2 = scmutil.revsingle(repo, rev2, b'null').node()
 
     with repo.wlock():
         repo.setparents(node1, node2)
 
 
 @command(b'debugsidedata', cmdutil.debugrevlogopts, _(b'-c|-m|FILE REV'))
 def debugsidedata(ui, repo, file_, rev=None, **opts):
     """dump the side data for a cl/manifest/file revision
 
     Use --verbose to dump the sidedata content."""
     opts = pycompat.byteskwargs(opts)
     if opts.get(b'changelog') or opts.get(b'manifest') or opts.get(b'dir'):
         if rev is not None:
             raise error.CommandError(b'debugdata', _(b'invalid arguments'))
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'debugdata', _(b'invalid arguments'))
     r = cmdutil.openstorage(repo, b'debugdata', file_, opts)
     r = getattr(r, '_revlog', r)
     try:
         sidedata = r.sidedata(r.lookup(rev))
     except KeyError:
         raise error.Abort(_(b'invalid revision identifier %s') % rev)
     if sidedata:
         sidedata = list(sidedata.items())
         sidedata.sort()
         ui.writenoi18n(b'%d sidedata entries\n' % len(sidedata))
         for key, value in sidedata:
             ui.writenoi18n(b' entry-%04o size %d\n' % (key, len(value)))
             if ui.verbose:
                 ui.writenoi18n(b'  %s\n' % stringutil.pprint(value))
 
 
 @command(b'debugssl', [], b'[SOURCE]', optionalrepo=True)
 def debugssl(ui, repo, source=None, **opts):
     """test a secure connection to a server
 
     This builds the certificate chain for the server on Windows, installing the
     missing intermediates and trusted root via Windows Update if necessary.  It
     does nothing on other platforms.
 
     If SOURCE is omitted, the 'default' path will be used.  If a URL is given,
     that server is used. See :hg:`help urls` for more information.
 
     If the update succeeds, retry the original operation.  Otherwise, the cause
     of the SSL error is likely another issue.
     """
     if not pycompat.iswindows:
         raise error.Abort(
             _(b'certificate chain building is only possible on Windows')
         )
 
     if not source:
         if not repo:
             raise error.Abort(
                 _(
                     b"there is no Mercurial repository here, and no "
                     b"server specified"
                 )
             )
         source = b"default"
 
     source, branches = hg.parseurl(ui.expandpath(source))
     url = util.url(source)
 
     defaultport = {b'https': 443, b'ssh': 22}
     if url.scheme in defaultport:
         try:
             addr = (url.host, int(url.port or defaultport[url.scheme]))
         except ValueError:
             raise error.Abort(_(b"malformed port number in URL"))
     else:
         raise error.Abort(_(b"only https and ssh connections are supported"))
 
     from . import win32
 
     s = ssl.wrap_socket(
         socket.socket(),
         ssl_version=ssl.PROTOCOL_TLS,
         cert_reqs=ssl.CERT_NONE,
         ca_certs=None,
     )
 
     try:
         s.connect(addr)
         cert = s.getpeercert(True)
 
         ui.status(_(b'checking the certificate chain for %s\n') % url.host)
 
         complete = win32.checkcertificatechain(cert, build=False)
 
         if not complete:
             ui.status(_(b'certificate chain is incomplete, updating... '))
 
             if not win32.checkcertificatechain(cert):
                 ui.status(_(b'failed.\n'))
             else:
                 ui.status(_(b'done.\n'))
         else:
             ui.status(_(b'full certificate chain is available\n'))
     finally:
         s.close()
 
 
 @command(
     b"debugbackupbundle",
     [
         (
             b"",
             b"recover",
             b"",
             b"brings the specified changeset back into the repository",
         )
     ]
     + cmdutil.logopts,
     _(b"hg debugbackupbundle [--recover HASH]"),
 )
 def debugbackupbundle(ui, repo, *pats, **opts):
     """lists the changesets available in backup bundles
 
     Without any arguments, this command prints a list of the changesets in each
     backup bundle.
 
     --recover takes a changeset hash and unbundles the first bundle that
     contains that hash, which puts that changeset back in your repository.
 
     --verbose will print the entire commit message and the bundle path for that
     backup.
     """
     backups = list(
         filter(
             os.path.isfile, glob.glob(repo.vfs.join(b"strip-backup") + b"/*.hg")
         )
     )
     backups.sort(key=lambda x: os.path.getmtime(x), reverse=True)
 
     opts = pycompat.byteskwargs(opts)
     opts[b"bundle"] = b""
     opts[b"force"] = None
     limit = logcmdutil.getlimit(opts)
 
     def display(other, chlist, displayer):
         if opts.get(b"newest_first"):
             chlist.reverse()
         count = 0
         for n in chlist:
             if limit is not None and count >= limit:
                 break
             parents = [True for p in other.changelog.parents(n) if p != nullid]
             if opts.get(b"no_merges") and len(parents) == 2:
                 continue
             count += 1
             displayer.show(other[n])
 
     recovernode = opts.get(b"recover")
     if recovernode:
         if scmutil.isrevsymbol(repo, recovernode):
             ui.warn(_(b"%s already exists in the repo\n") % recovernode)
             return
     elif backups:
         msg = _(
             b"Recover changesets using: hg debugbackupbundle --recover "
             b"<changeset hash>\n\nAvailable backup changesets:"
         )
         ui.status(msg, label=b"status.removed")
     else:
         ui.status(_(b"no backup changesets found\n"))
         return
 
     for backup in backups:
         # Much of this is copied from the hg incoming logic
         source = ui.expandpath(os.path.relpath(backup, encoding.getcwd()))
         source, branches = hg.parseurl(source, opts.get(b"branch"))
         try:
             other = hg.peer(repo, opts, source)
         except error.LookupError as ex:
             msg = _(b"\nwarning: unable to open bundle %s") % source
             hint = _(b"\n(missing parent rev %s)\n") % short(ex.name)
             ui.warn(msg, hint=hint)
             continue
         revs, checkout = hg.addbranchrevs(
             repo, other, branches, opts.get(b"rev")
         )
 
         if revs:
             revs = [other.lookup(rev) for rev in revs]
 
         quiet = ui.quiet
         try:
             ui.quiet = True
             other, chlist, cleanupfn = bundlerepo.getremotechanges(
                 ui, repo, other, revs, opts[b"bundle"], opts[b"force"]
             )
         except error.LookupError:
             continue
         finally:
             ui.quiet = quiet
 
         try:
             if not chlist:
                 continue
             if recovernode:
                 with repo.lock(), repo.transaction(b"unbundle") as tr:
                     if scmutil.isrevsymbol(other, recovernode):
                         ui.status(_(b"Unbundling %s\n") % (recovernode))
                         f = hg.openpath(ui, source)
                         gen = exchange.readbundle(ui, f, source)
                         if isinstance(gen, bundle2.unbundle20):
                             bundle2.applybundle(
                                 repo,
                                 gen,
                                 tr,
                                 source=b"unbundle",
                                 url=b"bundle:" + source,
                             )
                         else:
                             gen.apply(repo, b"unbundle", b"bundle:" + source)
                         break
             else:
                 backupdate = encoding.strtolocal(
                     time.strftime(
                         "%a %H:%M, %Y-%m-%d",
                         time.localtime(os.path.getmtime(source)),
                     )
                 )
                 ui.status(b"\n%s\n" % (backupdate.ljust(50)))
                 if ui.verbose:
                     ui.status(b"%s%s\n" % (b"bundle:".ljust(13), source))
                 else:
                     opts[
                         b"template"
                     ] = b"{label('status.modified', node|short)} {desc|firstline}\n"
                 displayer = logcmdutil.changesetdisplayer(
                     ui, other, opts, False
                 )
                 display(other, chlist, displayer)
                 displayer.close()
         finally:
             cleanupfn()
 
 
 @command(
     b'debugsub',
     [(b'r', b'rev', b'', _(b'revision to check'), _(b'REV'))],
     _(b'[-r REV] [REV]'),
 )
 def debugsub(ui, repo, rev=None):
     ctx = scmutil.revsingle(repo, rev, None)
     for k, v in sorted(ctx.substate.items()):
         ui.writenoi18n(b'path %s\n' % k)
         ui.writenoi18n(b' source   %s\n' % v[0])
         ui.writenoi18n(b' revision %s\n' % v[1])
 
 
 @command(b'debugshell', optionalrepo=True)
 def debugshell(ui, repo):
     """run an interactive Python interpreter
 
     The local namespace is provided with a reference to the ui and
     the repo instance (if available).
     """
     import code
 
     imported_objects = {
         'ui': ui,
         'repo': repo,
     }
 
     code.interact(local=imported_objects)
 
 
 @command(
     b'debugsuccessorssets',
     [(b'', b'closest', False, _(b'return closest successors sets only'))],
     _(b'[REV]'),
 )
 def debugsuccessorssets(ui, repo, *revs, **opts):
     """show set of successors for revision
 
     A successors set of changeset A is a consistent group of revisions that
     succeed A. It contains non-obsolete changesets only unless closests
     successors set is set.
 
     In most cases a changeset A has a single successors set containing a single
     successor (changeset A replaced by A').
 
     A changeset that is made obsolete with no successors are called "pruned".
     Such changesets have no successors sets at all.
 
     A changeset that has been "split" will have a successors set containing
     more than one successor.
 
     A changeset that has been rewritten in multiple different ways is called
     "divergent". Such changesets have multiple successor sets (each of which
     may also be split, i.e. have multiple successors).
 
     Results are displayed as follows::
 
         <rev1>
             <successors-1A>
         <rev2>
             <successors-2A>
             <successors-2B1> <successors-2B2> <successors-2B3>
 
     Here rev2 has two possible (i.e. divergent) successors sets. The first
     holds one element, whereas the second holds three (i.e. the changeset has
     been split).
     """
     # passed to successorssets caching computation from one call to another
     cache = {}
     ctx2str = bytes
     node2str = short
     for rev in scmutil.revrange(repo, revs):
         ctx = repo[rev]
         ui.write(b'%s\n' % ctx2str(ctx))
         for succsset in obsutil.successorssets(
             repo, ctx.node(), closest=opts['closest'], cache=cache
         ):
             if succsset:
                 ui.write(b'    ')
                 ui.write(node2str(succsset[0]))
                 for node in succsset[1:]:
                     ui.write(b' ')
                     ui.write(node2str(node))
             ui.write(b'\n')
 
 
 @command(b'debugtagscache', [])
 def debugtagscache(ui, repo):
     """display the contents of .hg/cache/hgtagsfnodes1"""
     cache = tagsmod.hgtagsfnodescache(repo.unfiltered())
     flog = repo.file(b'.hgtags')
     for r in repo:
         node = repo[r].node()
         tagsnode = cache.getfnode(node, computemissing=False)
         if tagsnode:
             tagsnodedisplay = hex(tagsnode)
             if not flog.hasnode(tagsnode):
                 tagsnodedisplay += b' (unknown node)'
         elif tagsnode is None:
             tagsnodedisplay = b'missing'
         else:
             tagsnodedisplay = b'invalid'
 
         ui.write(b'%d %s %s\n' % (r, hex(node), tagsnodedisplay))
 
 
 @command(
     b'debugtemplate',
     [
         (b'r', b'rev', [], _(b'apply template on changesets'), _(b'REV')),
         (b'D', b'define', [], _(b'define template keyword'), _(b'KEY=VALUE')),
     ],
     _(b'[-r REV]... [-D KEY=VALUE]... TEMPLATE'),
     optionalrepo=True,
 )
 def debugtemplate(ui, repo, tmpl, **opts):
     """parse and apply a template
 
     If -r/--rev is given, the template is processed as a log template and
     applied to the given changesets. Otherwise, it is processed as a generic
     template.
 
     Use --verbose to print the parsed tree.
     """
     revs = None
     if opts['rev']:
         if repo is None:
             raise error.RepoError(
                 _(b'there is no Mercurial repository here (.hg not found)')
             )
         revs = scmutil.revrange(repo, opts['rev'])
 
     props = {}
     for d in opts['define']:
         try:
             k, v = (e.strip() for e in d.split(b'=', 1))
             if not k or k == b'ui':
                 raise ValueError
             props[k] = v
         except ValueError:
             raise error.Abort(_(b'malformed keyword definition: %s') % d)
 
     if ui.verbose:
         aliases = ui.configitems(b'templatealias')
         tree = templater.parse(tmpl)
         ui.note(templater.prettyformat(tree), b'\n')
         newtree = templater.expandaliases(tree, aliases)
         if newtree != tree:
             ui.notenoi18n(
                 b"* expanded:\n", templater.prettyformat(newtree), b'\n'
             )
 
     if revs is None:
         tres = formatter.templateresources(ui, repo)
         t = formatter.maketemplater(ui, tmpl, resources=tres)
         if ui.verbose:
             kwds, funcs = t.symbolsuseddefault()
             ui.writenoi18n(b"* keywords: %s\n" % b', '.join(sorted(kwds)))
             ui.writenoi18n(b"* functions: %s\n" % b', '.join(sorted(funcs)))
         ui.write(t.renderdefault(props))
     else:
         displayer = logcmdutil.maketemplater(ui, repo, tmpl)
         if ui.verbose:
             kwds, funcs = displayer.t.symbolsuseddefault()
             ui.writenoi18n(b"* keywords: %s\n" % b', '.join(sorted(kwds)))
             ui.writenoi18n(b"* functions: %s\n" % b', '.join(sorted(funcs)))
         for r in revs:
             displayer.show(repo[r], **pycompat.strkwargs(props))
         displayer.close()
 
 
 @command(
     b'debuguigetpass',
     [
         (b'p', b'prompt', b'', _(b'prompt text'), _(b'TEXT')),
     ],
     _(b'[-p TEXT]'),
     norepo=True,
 )
 def debuguigetpass(ui, prompt=b''):
     """show prompt to type password"""
     r = ui.getpass(prompt)
     if r is None:
         r = b"<default response>"
     ui.writenoi18n(b'response: %s\n' % r)
 
 
 @command(
     b'debuguiprompt',
     [
         (b'p', b'prompt', b'', _(b'prompt text'), _(b'TEXT')),
     ],
     _(b'[-p TEXT]'),
     norepo=True,
 )
 def debuguiprompt(ui, prompt=b''):
     """show plain prompt"""
     r = ui.prompt(prompt)
     ui.writenoi18n(b'response: %s\n' % r)
 
 
 @command(b'debugupdatecaches', [])
 def debugupdatecaches(ui, repo, *pats, **opts):
     """warm all known caches in the repository"""
     with repo.wlock(), repo.lock():
         repo.updatecaches(full=True)
 
 
 @command(
     b'debugupgraderepo',
     [
         (
             b'o',
             b'optimize',
             [],
             _(b'extra optimization to perform'),
             _(b'NAME'),
         ),
         (b'', b'run', False, _(b'performs an upgrade')),
         (b'', b'backup', True, _(b'keep the old repository content around')),
         (b'', b'changelog', None, _(b'select the changelog for upgrade')),
         (b'', b'manifest', None, _(b'select the manifest for upgrade')),
         (b'', b'filelogs', None, _(b'select all filelogs for upgrade')),
     ],
 )
 def debugupgraderepo(ui, repo, run=False, optimize=None, backup=True, **opts):
     """upgrade a repository to use different features
 
     If no arguments are specified, the repository is evaluated for upgrade
     and a list of problems and potential optimizations is printed.
 
     With ``--run``, a repository upgrade is performed. Behavior of the upgrade
     can be influenced via additional arguments. More details will be provided
     by the command output when run without ``--run``.
 
     During the upgrade, the repository will be locked and no writes will be
     allowed.
 
     At the end of the upgrade, the repository may not be readable while new
     repository data is swapped in. This window will be as long as it takes to
     rename some directories inside the ``.hg`` directory. On most machines, this
     should complete almost instantaneously and the chances of a consumer being
     unable to access the repository should be low.
 
     By default, all revlog will be upgraded. You can restrict this using flag
     such as `--manifest`:
 
       * `--manifest`: only optimize the manifest
       * `--no-manifest`: optimize all revlog but the manifest
       * `--changelog`: optimize the changelog only
       * `--no-changelog --no-manifest`: optimize filelogs only
       * `--filelogs`: optimize the filelogs only
       * `--no-changelog --no-manifest --no-filelogs`: skip all revlog optimizations
     """
     return upgrade.upgraderepo(
         ui, repo, run=run, optimize=set(optimize), backup=backup, **opts
     )
 
 
 @command(
     b'debugwalk', cmdutil.walkopts, _(b'[OPTION]... [FILE]...'), inferrepo=True
 )
 def debugwalk(ui, repo, *pats, **opts):
     """show how files match on given patterns"""
     opts = pycompat.byteskwargs(opts)
     m = scmutil.match(repo[None], pats, opts)
     if ui.verbose:
         ui.writenoi18n(b'* matcher:\n', stringutil.prettyrepr(m), b'\n')
     items = list(repo[None].walk(m))
     if not items:
         return
     f = lambda fn: fn
     if ui.configbool(b'ui', b'slash') and pycompat.ossep != b'/':
         f = lambda fn: util.normpath(fn)
     fmt = b'f  %%-%ds  %%-%ds  %%s' % (
         max([len(abs) for abs in items]),
         max([len(repo.pathto(abs)) for abs in items]),
     )
     for abs in items:
         line = fmt % (
             abs,
             f(repo.pathto(abs)),
             m.exact(abs) and b'exact' or b'',
         )
         ui.write(b"%s\n" % line.rstrip())
 
 
 @command(b'debugwhyunstable', [], _(b'REV'))
 def debugwhyunstable(ui, repo, rev):
     """explain instabilities of a changeset"""
     for entry in obsutil.whyunstable(repo, scmutil.revsingle(repo, rev)):
         dnodes = b''
         if entry.get(b'divergentnodes'):
             dnodes = (
                 b' '.join(
                     b'%s (%s)' % (ctx.hex(), ctx.phasestr())
                     for ctx in entry[b'divergentnodes']
                 )
                 + b' '
             )
         ui.write(
             b'%s: %s%s %s\n'
             % (entry[b'instability'], dnodes, entry[b'reason'], entry[b'node'])
         )
 
 
 @command(
     b'debugwireargs',
     [
         (b'', b'three', b'', b'three'),
         (b'', b'four', b'', b'four'),
         (b'', b'five', b'', b'five'),
     ]
     + cmdutil.remoteopts,
     _(b'REPO [OPTIONS]... [ONE [TWO]]'),
     norepo=True,
 )
 def debugwireargs(ui, repopath, *vals, **opts):
     opts = pycompat.byteskwargs(opts)
     repo = hg.peer(ui, opts, repopath)
     try:
         for opt in cmdutil.remoteopts:
             del opts[opt[1]]
         args = {}
         for k, v in pycompat.iteritems(opts):
             if v:
                 args[k] = v
         args = pycompat.strkwargs(args)
         # run twice to check that we don't mess up the stream for the next command
         res1 = repo.debugwireargs(*vals, **args)
         res2 = repo.debugwireargs(*vals, **args)
         ui.write(b"%s\n" % res1)
         if res1 != res2:
             ui.warn(b"%s\n" % res2)
     finally:
         repo.close()
 
 
 def _parsewirelangblocks(fh):
     activeaction = None
     blocklines = []
     lastindent = 0
 
     for line in fh:
         line = line.rstrip()
         if not line:
             continue
 
         if line.startswith(b'#'):
             continue
 
         if not line.startswith(b' '):
             # New block. Flush previous one.
             if activeaction:
                 yield activeaction, blocklines
 
             activeaction = line
             blocklines = []
             lastindent = 0
             continue
 
         # Else we start with an indent.
 
         if not activeaction:
             raise error.Abort(_(b'indented line outside of block'))
 
         indent = len(line) - len(line.lstrip())
 
         # If this line is indented more than the last line, concatenate it.
         if indent > lastindent and blocklines:
             blocklines[-1] += line.lstrip()
         else:
             blocklines.append(line)
             lastindent = indent
 
     # Flush last block.
     if activeaction:
         yield activeaction, blocklines
 
 
 @command(
     b'debugwireproto',
     [
         (b'', b'localssh', False, _(b'start an SSH server for this repo')),
         (b'', b'peer', b'', _(b'construct a specific version of the peer')),
         (
             b'',
             b'noreadstderr',
             False,
             _(b'do not read from stderr of the remote'),
         ),
         (
             b'',
             b'nologhandshake',
             False,
             _(b'do not log I/O related to the peer handshake'),
         ),
     ]
     + cmdutil.remoteopts,
     _(b'[PATH]'),
     optionalrepo=True,
 )
 def debugwireproto(ui, repo, path=None, **opts):
     """send wire protocol commands to a server
 
     This command can be used to issue wire protocol commands to remote
     peers and to debug the raw data being exchanged.
 
     ``--localssh`` will start an SSH server against the current repository
     and connect to that. By default, the connection will perform a handshake
     and establish an appropriate peer instance.
 
     ``--peer`` can be used to bypass the handshake protocol and construct a
     peer instance using the specified class type. Valid values are ``raw``,
     ``http2``, ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending
     raw data payloads and don't support higher-level command actions.
 
     ``--noreadstderr`` can be used to disable automatic reading from stderr
     of the peer (for SSH connections only). Disabling automatic reading of
     stderr is useful for making output more deterministic.
 
     Commands are issued via a mini language which is specified via stdin.
     The language consists of individual actions to perform. An action is
     defined by a block. A block is defined as a line with no leading
     space followed by 0 or more lines with leading space. Blocks are
     effectively a high-level command with additional metadata.
 
     Lines beginning with ``#`` are ignored.
 
     The following sections denote available actions.
 
     raw
     ---
 
     Send raw data to the server.
 
     The block payload contains the raw data to send as one atomic send
     operation. The data may not actually be delivered in a single system
     call: it depends on the abilities of the transport being used.
 
     Each line in the block is de-indented and concatenated. Then, that
     value is evaluated as a Python b'' literal. This allows the use of
     backslash escaping, etc.
 
     raw+
     ----
 
     Behaves like ``raw`` except flushes output afterwards.
 
     command <X>
     -----------
 
     Send a request to run a named command, whose name follows the ``command``
     string.
 
     Arguments to the command are defined as lines in this block. The format of
     each line is ``<key> <value>``. e.g.::
 
        command listkeys
            namespace bookmarks
 
     If the value begins with ``eval:``, it will be interpreted as a Python
     literal expression. Otherwise values are interpreted as Python b'' literals.
     This allows sending complex types and encoding special byte sequences via
     backslash escaping.
 
     The following arguments have special meaning:
 
     ``PUSHFILE``
         When defined, the *push* mechanism of the peer will be used instead
         of the static request-response mechanism and the content of the
         file specified in the value of this argument will be sent as the
         command payload.
 
         This can be used to submit a local bundle file to the remote.
 
     batchbegin
     ----------
 
     Instruct the peer to begin a batched send.
 
     All ``command`` blocks are queued for execution until the next
     ``batchsubmit`` block.
 
     batchsubmit
     -----------
 
     Submit previously queued ``command`` blocks as a batch request.
 
     This action MUST be paired with a ``batchbegin`` action.
 
     httprequest <method> <path>
     ---------------------------
 
     (HTTP peer only)
 
     Send an HTTP request to the peer.
 
     The HTTP request line follows the ``httprequest`` action. e.g. ``GET /foo``.
 
     Arguments of the form ``<key>: <value>`` are interpreted as HTTP request
     headers to add to the request. e.g. ``Accept: foo``.
 
     The following arguments are special:
 
     ``BODYFILE``
         The content of the file defined as the value to this argument will be
         transferred verbatim as the HTTP request body.
 
     ``frame <type> <flags> <payload>``
         Send a unified protocol frame as part of the request body.
 
         All frames will be collected and sent as the body to the HTTP
         request.
 
     close
     -----
 
     Close the connection to the server.
 
     flush
     -----
 
     Flush data written to the server.
 
     readavailable
     -------------
 
     Close the write end of the connection and read all available data from
     the server.
 
     If the connection to the server encompasses multiple pipes, we poll both
     pipes and read available data.
 
     readline
     --------
 
     Read a line of output from the server. If there are multiple output
     pipes, reads only the main pipe.
 
     ereadline
     ---------
 
     Like ``readline``, but read from the stderr pipe, if available.
 
     read <X>
     --------
 
     ``read()`` N bytes from the server's main output pipe.
 
     eread <X>
     ---------
 
     ``read()`` N bytes from the server's stderr pipe, if available.
 
     Specifying Unified Frame-Based Protocol Frames
     ----------------------------------------------
 
     It is possible to emit a *Unified Frame-Based Protocol* by using special
     syntax.
 
     A frame is composed as a type, flags, and payload. These can be parsed
     from a string of the form:
 
        <request-id> <stream-id> <stream-flags> <type> <flags> <payload>
 
     ``request-id`` and ``stream-id`` are integers defining the request and
     stream identifiers.
 
     ``type`` can be an integer value for the frame type or the string name
     of the type. The strings are defined in ``wireprotoframing.py``. e.g.
     ``command-name``.
 
     ``stream-flags`` and ``flags`` are a ``|`` delimited list of flag
     components. Each component (and there can be just one) can be an integer
     or a flag name for stream flags or frame flags, respectively. Values are
     resolved to integers and then bitwise OR'd together.
 
     ``payload`` represents the raw frame payload. If it begins with
     ``cbor:``, the following string is evaluated as Python code and the
     resulting object is fed into a CBOR encoder. Otherwise it is interpreted
     as a Python byte string literal.
     """
     opts = pycompat.byteskwargs(opts)
 
     if opts[b'localssh'] and not repo:
         raise error.Abort(_(b'--localssh requires a repository'))
 
     if opts[b'peer'] and opts[b'peer'] not in (
         b'raw',
         b'http2',
         b'ssh1',
         b'ssh2',
     ):
         raise error.Abort(
             _(b'invalid value for --peer'),
             hint=_(b'valid values are "raw", "ssh1", and "ssh2"'),
         )
 
     if path and opts[b'localssh']:
         raise error.Abort(_(b'cannot specify --localssh with an explicit path'))
 
     if ui.interactive():
         ui.write(_(b'(waiting for commands on stdin)\n'))
 
     blocks = list(_parsewirelangblocks(ui.fin))
 
     proc = None
     stdin = None
     stdout = None
     stderr = None
     opener = None
 
     if opts[b'localssh']:
         # We start the SSH server in its own process so there is process
         # separation. This prevents a whole class of potential bugs around
         # shared state from interfering with server operation.
         args = procutil.hgcmd() + [
             b'-R',
             repo.root,
             b'debugserve',
             b'--sshstdio',
         ]
         proc = subprocess.Popen(
             pycompat.rapply(procutil.tonativestr, args),
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             bufsize=0,
         )
 
         stdin = proc.stdin
         stdout = proc.stdout
         stderr = proc.stderr
 
         # We turn the pipes into observers so we can log I/O.
         if ui.verbose or opts[b'peer'] == b'raw':
             stdin = util.makeloggingfileobject(
                 ui, proc.stdin, b'i', logdata=True
             )
             stdout = util.makeloggingfileobject(
                 ui, proc.stdout, b'o', logdata=True
             )
             stderr = util.makeloggingfileobject(
                 ui, proc.stderr, b'e', logdata=True
             )
 
         # --localssh also implies the peer connection settings.
 
         url = b'ssh://localserver'
         autoreadstderr = not opts[b'noreadstderr']
 
         if opts[b'peer'] == b'ssh1':
             ui.write(_(b'creating ssh peer for wire protocol version 1\n'))
             peer = sshpeer.sshv1peer(
                 ui,
                 url,
                 proc,
                 stdin,
                 stdout,
                 stderr,
                 None,
                 autoreadstderr=autoreadstderr,
             )
         elif opts[b'peer'] == b'ssh2':
             ui.write(_(b'creating ssh peer for wire protocol version 2\n'))
             peer = sshpeer.sshv2peer(
                 ui,
                 url,
                 proc,
                 stdin,
                 stdout,
                 stderr,
                 None,
                 autoreadstderr=autoreadstderr,
             )
         elif opts[b'peer'] == b'raw':
             ui.write(_(b'using raw connection to peer\n'))
             peer = None
         else:
             ui.write(_(b'creating ssh peer from handshake results\n'))
             peer = sshpeer.makepeer(
                 ui,
                 url,
                 proc,
                 stdin,
                 stdout,
                 stderr,
                 autoreadstderr=autoreadstderr,
             )
 
     elif path:
         # We bypass hg.peer() so we can proxy the sockets.
         # TODO consider not doing this because we skip
         # ``hg.wirepeersetupfuncs`` and potentially other useful functionality.
         u = util.url(path)
         if u.scheme != b'http':
             raise error.Abort(_(b'only http:// paths are currently supported'))
 
         url, authinfo = u.authinfo()
         openerargs = {
             'useragent': b'Mercurial debugwireproto',
         }
 
         # Turn pipes/sockets into observers so we can log I/O.
         if ui.verbose:
             openerargs.update(
                 {
                     'loggingfh': ui,
                     'loggingname': b's',
                     'loggingopts': {
                         'logdata': True,
                         'logdataapis': False,
                     },
                 }
             )
 
         if ui.debugflag:
             openerargs['loggingopts']['logdataapis'] = True
 
         # Don't send default headers when in raw mode. This allows us to
         # bypass most of the behavior of our URL handling code so we can
         # have near complete control over what's sent on the wire.
         if opts[b'peer'] == b'raw':
             openerargs['sendaccept'] = False
 
         opener = urlmod.opener(ui, authinfo, **openerargs)
 
         if opts[b'peer'] == b'http2':
             ui.write(_(b'creating http peer for wire protocol version 2\n'))
             # We go through makepeer() because we need an API descriptor for
             # the peer instance to be useful.
             with ui.configoverride(
                 {(b'experimental', b'httppeer.advertise-v2'): True}
             ):
                 if opts[b'nologhandshake']:
                     ui.pushbuffer()
 
                 peer = httppeer.makepeer(ui, path, opener=opener)
 
                 if opts[b'nologhandshake']:
                     ui.popbuffer()
 
             if not isinstance(peer, httppeer.httpv2peer):
                 raise error.Abort(
                     _(
                         b'could not instantiate HTTP peer for '
                         b'wire protocol version 2'
                     ),
                     hint=_(
                         b'the server may not have the feature '
                         b'enabled or is not allowing this '
                         b'client version'
                     ),
                 )
 
         elif opts[b'peer'] == b'raw':
             ui.write(_(b'using raw connection to peer\n'))
             peer = None
         elif opts[b'peer']:
             raise error.Abort(
                 _(b'--peer %s not supported with HTTP peers') % opts[b'peer']
             )
         else:
             peer = httppeer.makepeer(ui, path, opener=opener)
 
         # We /could/ populate stdin/stdout with sock.makefile()...
     else:
         raise error.Abort(_(b'unsupported connection configuration'))
 
     batchedcommands = None
 
     # Now perform actions based on the parsed wire language instructions.
     for action, lines in blocks:
         if action in (b'raw', b'raw+'):
             if not stdin:
                 raise error.Abort(_(b'cannot call raw/raw+ on this peer'))
 
             # Concatenate the data together.
             data = b''.join(l.lstrip() for l in lines)
             data = stringutil.unescapestr(data)
             stdin.write(data)
 
             if action == b'raw+':
                 stdin.flush()
         elif action == b'flush':
             if not stdin:
                 raise error.Abort(_(b'cannot call flush on this peer'))
             stdin.flush()
         elif action.startswith(b'command'):
             if not peer:
                 raise error.Abort(
                     _(
                         b'cannot send commands unless peer instance '
                         b'is available'
                     )
                 )
 
             command = action.split(b' ', 1)[1]
 
             args = {}
             for line in lines:
                 # We need to allow empty values.
                 fields = line.lstrip().split(b' ', 1)
                 if len(fields) == 1:
                     key = fields[0]
                     value = b''
                 else:
                     key, value = fields
 
                 if value.startswith(b'eval:'):
                     value = stringutil.evalpythonliteral(value[5:])
                 else:
                     value = stringutil.unescapestr(value)
 
                 args[key] = value
 
             if batchedcommands is not None:
                 batchedcommands.append((command, args))
                 continue
 
             ui.status(_(b'sending %s command\n') % command)
 
             if b'PUSHFILE' in args:
                 with open(args[b'PUSHFILE'], 'rb') as fh:
                     del args[b'PUSHFILE']
                     res, output = peer._callpush(
                         command, fh, **pycompat.strkwargs(args)
                     )
                     ui.status(_(b'result: %s\n') % stringutil.escapestr(res))
                     ui.status(
                         _(b'remote output: %s\n') % stringutil.escapestr(output)
                     )
             else:
                 with peer.commandexecutor() as e:
                     res = e.callcommand(command, args).result()
 
                 if isinstance(res, wireprotov2peer.commandresponse):
                     val = res.objects()
                     ui.status(
                         _(b'response: %s\n')
                         % stringutil.pprint(val, bprefix=True, indent=2)
                     )
                 else:
                     ui.status(
                         _(b'response: %s\n')
                         % stringutil.pprint(res, bprefix=True, indent=2)
                     )
 
         elif action == b'batchbegin':
             if batchedcommands is not None:
                 raise error.Abort(_(b'nested batchbegin not allowed'))
 
             batchedcommands = []
         elif action == b'batchsubmit':
             # There is a batching API we could go through. But it would be
             # difficult to normalize requests into function calls. It is easier
             # to bypass this layer and normalize to commands + args.
             ui.status(
                 _(b'sending batch with %d sub-commands\n')
                 % len(batchedcommands)
             )
             assert peer is not None
             for i, chunk in enumerate(peer._submitbatch(batchedcommands)):
                 ui.status(
                     _(b'response #%d: %s\n') % (i, stringutil.escapestr(chunk))
                 )
 
             batchedcommands = None
 
         elif action.startswith(b'httprequest '):
             if not opener:
                 raise error.Abort(
                     _(b'cannot use httprequest without an HTTP peer')
                 )
 
             request = action.split(b' ', 2)
             if len(request) != 3:
                 raise error.Abort(
                     _(
                         b'invalid httprequest: expected format is '
                         b'"httprequest <method> <path>'
                     )
                 )
 
             method, httppath = request[1:]
             headers = {}
             body = None
             frames = []
             for line in lines:
                 line = line.lstrip()
                 m = re.match(b'^([a-zA-Z0-9_-]+): (.*)$', line)
                 if m:
                     # Headers need to use native strings.
                     key = pycompat.strurl(m.group(1))
                     value = pycompat.strurl(m.group(2))
                     headers[key] = value
                     continue
 
                 if line.startswith(b'BODYFILE '):
                     with open(line.split(b' ', 1), b'rb') as fh:
                         body = fh.read()
                 elif line.startswith(b'frame '):
                     frame = wireprotoframing.makeframefromhumanstring(
                         line[len(b'frame ') :]
                     )
 
                     frames.append(frame)
                 else:
                     raise error.Abort(
                         _(b'unknown argument to httprequest: %s') % line
                     )
 
             url = path + httppath
 
             if frames:
                 body = b''.join(bytes(f) for f in frames)
 
             req = urlmod.urlreq.request(pycompat.strurl(url), body, headers)
 
             # urllib.Request insists on using has_data() as a proxy for
             # determining the request method. Override that to use our
             # explicitly requested method.
             req.get_method = lambda: pycompat.sysstr(method)
 
             try:
                 res = opener.open(req)
                 body = res.read()
             except util.urlerr.urlerror as e:
                 # read() method must be called, but only exists in Python 2
                 getattr(e, 'read', lambda: None)()
                 continue
 
             ct = res.headers.get('Content-Type')
             if ct == 'application/mercurial-cbor':
                 ui.write(
                     _(b'cbor> %s\n')
                     % stringutil.pprint(
                         cborutil.decodeall(body), bprefix=True, indent=2
                     )
                 )
 
         elif action == b'close':
             assert peer is not None
             peer.close()
         elif action == b'readavailable':
             if not stdout or not stderr:
                 raise error.Abort(
                     _(b'readavailable not available on this peer')
                 )
 
             stdin.close()
             stdout.read()
             stderr.read()
 
         elif action == b'readline':
             if not stdout:
                 raise error.Abort(_(b'readline not available on this peer'))
             stdout.readline()
         elif action == b'ereadline':
             if not stderr:
                 raise error.Abort(_(b'ereadline not available on this peer'))
             stderr.readline()
         elif action.startswith(b'read '):
             count = int(action.split(b' ', 1)[1])
             if not stdout:
                 raise error.Abort(_(b'read not available on this peer'))
             stdout.read(count)
         elif action.startswith(b'eread '):
             count = int(action.split(b' ', 1)[1])
             if not stderr:
                 raise error.Abort(_(b'eread not available on this peer'))
             stderr.read(count)
         else:
             raise error.Abort(_(b'unknown action: %s') % action)
 
     if batchedcommands is not None:
         raise error.Abort(_(b'unclosed "batchbegin" request'))
 
     if peer:
         peer.close()
 
     if proc:
         proc.kill()

mercurial/exchange.py

@@ -1,2756 +1,2775 b''
 # exchange.py - utility to exchange data between repos.
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import weakref
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
 )
 from . import (
     bookmarks as bookmod,
     bundle2,
     bundlecaches,
     changegroup,
     discovery,
     error,
     exchangev2,
     lock as lockmod,
     logexchange,
     narrowspec,
     obsolete,
     obsutil,
     phases,
     pushkey,
     pycompat,
     requirements,
     scmutil,
     streamclone,
     url as urlmod,
     util,
     wireprototypes,
 )
 from .utils import (
     hashutil,
     stringutil,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 _NARROWACL_SECTION = b'narrowacl'
 
 
 def readbundle(ui, fh, fname, vfs=None):
     header = changegroup.readexactly(fh, 4)
 
     alg = None
     if not fname:
         fname = b"stream"
         if not header.startswith(b'HG') and header.startswith(b'\0'):
             fh = changegroup.headerlessfixup(fh, header)
             header = b"HG10"
             alg = b'UN'
     elif vfs:
         fname = vfs.join(fname)
 
     magic, version = header[0:2], header[2:4]
 
     if magic != b'HG':
         raise error.Abort(_(b'%s: not a Mercurial bundle') % fname)
     if version == b'10':
         if alg is None:
             alg = changegroup.readexactly(fh, 2)
         return changegroup.cg1unpacker(fh, alg)
     elif version.startswith(b'2'):
         return bundle2.getunbundler(ui, fh, magicstring=magic + version)
     elif version == b'S1':
         return streamclone.streamcloneapplier(fh)
     else:
         raise error.Abort(
             _(b'%s: unknown bundle version %s') % (fname, version)
         )
 
 
 def getbundlespec(ui, fh):
     """Infer the bundlespec from a bundle file handle.
 
     The input file handle is seeked and the original seek position is not
     restored.
     """
 
     def speccompression(alg):
         try:
             return util.compengines.forbundletype(alg).bundletype()[0]
         except KeyError:
             return None
 
     b = readbundle(ui, fh, None)
     if isinstance(b, changegroup.cg1unpacker):
         alg = b._type
         if alg == b'_truncatedBZ':
             alg = b'BZ'
         comp = speccompression(alg)
         if not comp:
             raise error.Abort(_(b'unknown compression algorithm: %s') % alg)
         return b'%s-v1' % comp
     elif isinstance(b, bundle2.unbundle20):
         if b'Compression' in b.params:
             comp = speccompression(b.params[b'Compression'])
             if not comp:
                 raise error.Abort(
                     _(b'unknown compression algorithm: %s') % comp
                 )
         else:
             comp = b'none'
 
         version = None
         for part in b.iterparts():
             if part.type == b'changegroup':
                 version = part.params[b'version']
                 if version in (b'01', b'02'):
                     version = b'v2'
                 else:
                     raise error.Abort(
                         _(
                             b'changegroup version %s does not have '
                             b'a known bundlespec'
                         )
                         % version,
                         hint=_(b'try upgrading your Mercurial client'),
                     )
             elif part.type == b'stream2' and version is None:
                 # A stream2 part requires to be part of a v2 bundle
                 requirements = urlreq.unquote(part.params[b'requirements'])
                 splitted = requirements.split()
                 params = bundle2._formatrequirementsparams(splitted)
                 return b'none-v2;stream=v2;%s' % params
 
         if not version:
             raise error.Abort(
                 _(b'could not identify changegroup version in bundle')
             )
 
         return b'%s-%s' % (comp, version)
     elif isinstance(b, streamclone.streamcloneapplier):
         requirements = streamclone.readbundle1header(fh)[2]
         formatted = bundle2._formatrequirementsparams(requirements)
         return b'none-packed1;%s' % formatted
     else:
         raise error.Abort(_(b'unknown bundle type: %s') % b)
 
 
 def _computeoutgoing(repo, heads, common):
     """Computes which revs are outgoing given a set of common
     and a set of heads.
 
     This is a separate function so extensions can have access to
     the logic.
 
     Returns a discovery.outgoing object.
     """
     cl = repo.changelog
     if common:
         hasnode = cl.hasnode
         common = [n for n in common if hasnode(n)]
     else:
         common = [nullid]
     if not heads:
         heads = cl.heads()
     return discovery.outgoing(repo, common, heads)
 
 
 def _checkpublish(pushop):
     repo = pushop.repo
     ui = repo.ui
     behavior = ui.config(b'experimental', b'auto-publish')
     if pushop.publish or behavior not in (b'warn', b'confirm', b'abort'):
         return
     remotephases = listkeys(pushop.remote, b'phases')
     if not remotephases.get(b'publishing', False):
         return
 
     if pushop.revs is None:
         published = repo.filtered(b'served').revs(b'not public()')
     else:
         published = repo.revs(b'::%ln - public()', pushop.revs)
     if published:
         if behavior == b'warn':
             ui.warn(
                 _(b'%i changesets about to be published\n') % len(published)
             )
         elif behavior == b'confirm':
             if ui.promptchoice(
                 _(b'push and publish %i changesets (yn)?$$ &Yes $$ &No')
                 % len(published)
             ):
                 raise error.CanceledError(_(b'user quit'))
         elif behavior == b'abort':
             msg = _(b'push would publish %i changesets') % len(published)
             hint = _(
                 b"use --publish or adjust 'experimental.auto-publish'"
                 b" config"
             )
             raise error.Abort(msg, hint=hint)
 
 
 def _forcebundle1(op):
     """return true if a pull/push must use bundle1
 
     This function is used to allow testing of the older bundle version"""
     ui = op.repo.ui
     # The goal is this config is to allow developer to choose the bundle
     # version used during exchanged. This is especially handy during test.
     # Value is a list of bundle version to be picked from, highest version
     # should be used.
     #
     # developer config: devel.legacy.exchange
     exchange = ui.configlist(b'devel', b'legacy.exchange')
     forcebundle1 = b'bundle2' not in exchange and b'bundle1' in exchange
     return forcebundle1 or not op.remote.capable(b'bundle2')
 
 
 class pushoperation(object):
     """A object that represent a single push operation
 
     Its purpose is to carry push related state and very common operations.
 
     A new pushoperation should be created at the beginning of each push and
     discarded afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         force=False,
         revs=None,
         newbranch=False,
         bookmarks=(),
         publish=False,
         pushvars=None,
     ):
         # repo we push from
         self.repo = repo
         self.ui = repo.ui
         # repo we push to
         self.remote = remote
         # force option provided
         self.force = force
         # revs to be pushed (None is "all")
         self.revs = revs
         # bookmark explicitly pushed
         self.bookmarks = bookmarks
         # allow push of new branch
         self.newbranch = newbranch
         # step already performed
         # (used to check what steps have been already performed through bundle2)
         self.stepsdone = set()
         # Integer version of the changegroup push result
         # - None means nothing to push
         # - 0 means HTTP error
         # - 1 means we pushed and remote head count is unchanged *or*
         #   we have outgoing changesets but refused to push
         # - other values as described by addchangegroup()
         self.cgresult = None
         # Boolean value for the bookmark push
         self.bkresult = None
         # discover.outgoing object (contains common and outgoing data)
         self.outgoing = None
         # all remote topological heads before the push
         self.remoteheads = None
         # Details of the remote branch pre and post push
         #
         # mapping: {'branch': ([remoteheads],
         #                      [newheads],
         #                      [unsyncedheads],
         #                      [discardedheads])}
         # - branch: the branch name
         # - remoteheads: the list of remote heads known locally
         #                None if the branch is new
         # - newheads: the new remote heads (known locally) with outgoing pushed
         # - unsyncedheads: the list of remote heads unknown locally.
         # - discardedheads: the list of remote heads made obsolete by the push
         self.pushbranchmap = None
         # testable as a boolean indicating if any nodes are missing locally.
         self.incoming = None
         # summary of the remote phase situation
         self.remotephases = None
         # phases changes that must be pushed along side the changesets
         self.outdatedphases = None
         # phases changes that must be pushed if changeset push fails
         self.fallbackoutdatedphases = None
         # outgoing obsmarkers
         self.outobsmarkers = set()
         # outgoing bookmarks, list of (bm, oldnode | '', newnode | '')
         self.outbookmarks = []
         # transaction manager
         self.trmanager = None
         # map { pushkey partid -> callback handling failure}
         # used to handle exception from mandatory pushkey part failure
         self.pkfailcb = {}
         # an iterable of pushvars or None
         self.pushvars = pushvars
         # publish pushed changesets
         self.publish = publish
 
     @util.propertycache
     def futureheads(self):
         """future remote heads if the changeset push succeeds"""
         return self.outgoing.ancestorsof
 
     @util.propertycache
     def fallbackheads(self):
         """future remote heads if the changeset push fails"""
         if self.revs is None:
             # not target to push, all common are relevant
             return self.outgoing.commonheads
         unfi = self.repo.unfiltered()
         # I want cheads = heads(::ancestorsof and ::commonheads)
         # (ancestorsof is revs with secret changeset filtered out)
         #
         # This can be expressed as:
         #     cheads = ( (ancestorsof and ::commonheads)
         #              + (commonheads and ::ancestorsof))"
         #              )
         #
         # while trying to push we already computed the following:
         #     common = (::commonheads)
         #     missing = ((commonheads::ancestorsof) - commonheads)
         #
         # We can pick:
         # * ancestorsof part of common (::commonheads)
         common = self.outgoing.common
         rev = self.repo.changelog.index.rev
         cheads = [node for node in self.revs if rev(node) in common]
         # and
         # * commonheads parents on missing
         revset = unfi.set(
             b'%ln and parents(roots(%ln))',
             self.outgoing.commonheads,
             self.outgoing.missing,
         )
         cheads.extend(c.node() for c in revset)
         return cheads
 
     @property
     def commonheads(self):
         """set of all common heads after changeset bundle push"""
         if self.cgresult:
             return self.futureheads
         else:
             return self.fallbackheads
 
 
 # mapping of message used when pushing bookmark
 bookmsgmap = {
     b'update': (
         _(b"updating bookmark %s\n"),
         _(b'updating bookmark %s failed\n'),
     ),
     b'export': (
         _(b"exporting bookmark %s\n"),
         _(b'exporting bookmark %s failed\n'),
     ),
     b'delete': (
         _(b"deleting remote bookmark %s\n"),
         _(b'deleting remote bookmark %s failed\n'),
     ),
 }
 
 
 def push(
     repo,
     remote,
     force=False,
     revs=None,
     newbranch=False,
     bookmarks=(),
     publish=False,
     opargs=None,
 ):
     """Push outgoing changesets (limited by revs) from a local
     repository to remote. Return an integer:
       - None means nothing to push
       - 0 means HTTP error
       - 1 means we pushed and remote head count is unchanged *or*
         we have outgoing changesets but refused to push
       - other values as described by addchangegroup()
     """
     if opargs is None:
         opargs = {}
     pushop = pushoperation(
         repo,
         remote,
         force,
         revs,
         newbranch,
         bookmarks,
         publish,
         **pycompat.strkwargs(opargs)
     )
     if pushop.remote.local():
         missing = (
             set(pushop.repo.requirements) - pushop.remote.local().supported
         )
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     if not pushop.remote.canpush():
         raise error.Abort(_(b"destination does not support push"))
 
     if not pushop.remote.capable(b'unbundle'):
         raise error.Abort(
             _(
                 b'cannot push: destination does not support the '
                 b'unbundle wire protocol command'
             )
         )
 
     # get lock as we might write phase data
     wlock = lock = None
     try:
         # bundle2 push may receive a reply bundle touching bookmarks
         # requiring the wlock. Take it now to ensure proper ordering.
         maypushback = pushop.ui.configbool(b'experimental', b'bundle2.pushback')
         if (
             (not _forcebundle1(pushop))
             and maypushback
             and not bookmod.bookmarksinstore(repo)
         ):
             wlock = pushop.repo.wlock()
         lock = pushop.repo.lock()
         pushop.trmanager = transactionmanager(
             pushop.repo, b'push-response', pushop.remote.url()
         )
     except error.LockUnavailable as err:
         # source repo cannot be locked.
         # We do not abort the push, but just disable the local phase
         # synchronisation.
         msg = b'cannot lock source repository: %s\n' % stringutil.forcebytestr(
             err
         )
         pushop.ui.debug(msg)
 
     with wlock or util.nullcontextmanager():
         with lock or util.nullcontextmanager():
             with pushop.trmanager or util.nullcontextmanager():
                 pushop.repo.checkpush(pushop)
                 _checkpublish(pushop)
                 _pushdiscovery(pushop)
                 if not pushop.force:
                     _checksubrepostate(pushop)
                 if not _forcebundle1(pushop):
                     _pushbundle2(pushop)
                 _pushchangeset(pushop)
                 _pushsyncphase(pushop)
                 _pushobsolete(pushop)
                 _pushbookmark(pushop)
 
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pushop
 
 
 # list of steps to perform discovery before push
 pushdiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pushdiscoverymapping = {}
 
 
 def pushdiscovery(stepname):
     """decorator for function performing discovery before push
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pushdiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pushdiscoverymapping
         pushdiscoverymapping[stepname] = func
         pushdiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pushdiscovery(pushop):
     """Run all discovery steps"""
     for stepname in pushdiscoveryorder:
         step = pushdiscoverymapping[stepname]
         step(pushop)
 
 
 def _checksubrepostate(pushop):
     """Ensure all outgoing referenced subrepo revisions are present locally"""
     for n in pushop.outgoing.missing:
         ctx = pushop.repo[n]
 
         if b'.hgsub' in ctx.manifest() and b'.hgsubstate' in ctx.files():
             for subpath in sorted(ctx.substate):
                 sub = ctx.sub(subpath)
                 sub.verify(onpush=True)
 
 
 @pushdiscovery(b'changeset')
 def _pushdiscoverychangeset(pushop):
     """discover the changeset that need to be pushed"""
     fci = discovery.findcommonincoming
     if pushop.revs:
         commoninc = fci(
             pushop.repo,
             pushop.remote,
             force=pushop.force,
             ancestorsof=pushop.revs,
         )
     else:
         commoninc = fci(pushop.repo, pushop.remote, force=pushop.force)
     common, inc, remoteheads = commoninc
     fco = discovery.findcommonoutgoing
     outgoing = fco(
         pushop.repo,
         pushop.remote,
         onlyheads=pushop.revs,
         commoninc=commoninc,
         force=pushop.force,
     )
     pushop.outgoing = outgoing
     pushop.remoteheads = remoteheads
     pushop.incoming = inc
 
 
 @pushdiscovery(b'phase')
 def _pushdiscoveryphase(pushop):
     """discover the phase that needs to be pushed
 
     (computed for both success and failure case for changesets push)"""
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     remotephases = listkeys(pushop.remote, b'phases')
 
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and not pushop.outgoing.missing  # no changesets to be pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset are to be pushed
         # - and remote is publishing
         # We may be in issue 3781 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         pushop.outdatedphases = []
         pushop.fallbackoutdatedphases = []
         return
 
     pushop.remotephases = phases.remotephasessummary(
         pushop.repo, pushop.fallbackheads, remotephases
     )
     droots = pushop.remotephases.draftroots
 
     extracond = b''
     if not pushop.remotephases.publishing:
         extracond = b' and public()'
     revset = b'heads((%%ln::%%ln) %s)' % extracond
     # Get the list of all revs draft on remote by public here.
     # XXX Beware that revset break if droots is not strictly
     # XXX root we may want to ensure it is but it is costly
     fallback = list(unfi.set(revset, droots, pushop.fallbackheads))
     if not pushop.remotephases.publishing and pushop.publish:
         future = list(
             unfi.set(
                 b'%ln and (not public() or %ln::)', pushop.futureheads, droots
             )
         )
     elif not outgoing.missing:
         future = fallback
     else:
         # adds changeset we are going to push as draft
         #
         # should not be necessary for publishing server, but because of an
         # issue fixed in xxxxx we have to do it anyway.
         fdroots = list(
             unfi.set(b'roots(%ln  + %ln::)', outgoing.missing, droots)
         )
         fdroots = [f.node() for f in fdroots]
         future = list(unfi.set(revset, fdroots, pushop.futureheads))
     pushop.outdatedphases = future
     pushop.fallbackoutdatedphases = fallback
 
 
 @pushdiscovery(b'obsmarker')
 def _pushdiscoveryobsmarkers(pushop):
     if not obsolete.isenabled(pushop.repo, obsolete.exchangeopt):
         return
 
     if not pushop.repo.obsstore:
         return
 
     if b'obsolete' not in listkeys(pushop.remote, b'namespaces'):
         return
 
     repo = pushop.repo
     # very naive computation, that can be quite expensive on big repo.
     # However: evolution is currently slow on them anyway.
     nodes = (c.node() for c in repo.set(b'::%ln', pushop.futureheads))
     pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes)
 
 
 @pushdiscovery(b'bookmarks')
 def _pushdiscoverybookmarks(pushop):
     ui = pushop.ui
     repo = pushop.repo.unfiltered()
     remote = pushop.remote
     ui.debug(b"checking for updated bookmarks\n")
     ancestors = ()
     if pushop.revs:
         revnums = pycompat.maplist(repo.changelog.rev, pushop.revs)
         ancestors = repo.changelog.ancestors(revnums, inclusive=True)
 
     remotebookmark = bookmod.unhexlifybookmarks(listkeys(remote, b'bookmarks'))
 
     explicit = {
         repo._bookmarks.expandname(bookmark) for bookmark in pushop.bookmarks
     }
 
     comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark)
     return _processcompared(pushop, ancestors, explicit, remotebookmark, comp)
 
 
 def _processcompared(pushop, pushed, explicit, remotebms, comp):
     """take decision on bookmarks to push to the remote repo
 
     Exists to help extensions alter this behavior.
     """
     addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp
 
     repo = pushop.repo
 
     for b, scid, dcid in advsrc:
         if b in explicit:
             explicit.remove(b)
         if not pushed or repo[scid].rev() in pushed:
             pushop.outbookmarks.append((b, dcid, scid))
     # search added bookmark
     for b, scid, dcid in addsrc:
         if b in explicit:
             explicit.remove(b)
             if bookmod.isdivergent(b):
                 pushop.ui.warn(_(b'cannot push divergent bookmark %s!\n') % b)
                 pushop.bkresult = 2
             else:
                 pushop.outbookmarks.append((b, b'', scid))
     # search for overwritten bookmark
     for b, scid, dcid in list(advdst) + list(diverge) + list(differ):
         if b in explicit:
             explicit.remove(b)
             pushop.outbookmarks.append((b, dcid, scid))
     # search for bookmark to delete
     for b, scid, dcid in adddst:
         if b in explicit:
             explicit.remove(b)
             # treat as "deleted locally"
             pushop.outbookmarks.append((b, dcid, b''))
     # identical bookmarks shouldn't get reported
     for b, scid, dcid in same:
         if b in explicit:
             explicit.remove(b)
 
     if explicit:
         explicit = sorted(explicit)
         # we should probably list all of them
         pushop.ui.warn(
             _(
                 b'bookmark %s does not exist on the local '
                 b'or remote repository!\n'
             )
             % explicit[0]
         )
         pushop.bkresult = 2
 
     pushop.outbookmarks.sort()
 
 
 def _pushcheckoutgoing(pushop):
     outgoing = pushop.outgoing
     unfi = pushop.repo.unfiltered()
     if not outgoing.missing:
         # nothing to push
         scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded)
         return False
     # something to push
     if not pushop.force:
         # if repo.obsstore == False --> no obsolete
         # then, save the iteration
         if unfi.obsstore:
             # this message are here for 80 char limit reason
             mso = _(b"push includes obsolete changeset: %s!")
             mspd = _(b"push includes phase-divergent changeset: %s!")
             mscd = _(b"push includes content-divergent changeset: %s!")
             mst = {
                 b"orphan": _(b"push includes orphan changeset: %s!"),
                 b"phase-divergent": mspd,
                 b"content-divergent": mscd,
             }
             # If we are to push if there is at least one
             # obsolete or unstable changeset in missing, at
             # least one of the missinghead will be obsolete or
             # unstable. So checking heads only is ok
             for node in outgoing.ancestorsof:
                 ctx = unfi[node]
                 if ctx.obsolete():
                     raise error.Abort(mso % ctx)
                 elif ctx.isunstable():
                     # TODO print more than one instability in the abort
                     # message
                     raise error.Abort(mst[ctx.instabilities()[0]] % ctx)
 
         discovery.checkheads(pushop)
     return True
 
 
 # List of names of steps to perform for an outgoing bundle2, order matters.
 b2partsgenorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 b2partsgenmapping = {}
 
 
 def b2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the b2partsgenmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in b2partsgenmapping
         b2partsgenmapping[stepname] = func
         if idx is None:
             b2partsgenorder.append(stepname)
         else:
             b2partsgenorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def _pushb2ctxcheckheads(pushop, bundler):
     """Generate race condition checking parts
 
     Exists as an independent function to aid extensions
     """
     # * 'force' do not check for push race,
     # * if we don't push anything, there are nothing to check.
     if not pushop.force and pushop.outgoing.ancestorsof:
         allowunrelated = b'related' in bundler.capabilities.get(
             b'checkheads', ()
         )
         emptyremote = pushop.pushbranchmap is None
         if not allowunrelated or emptyremote:
             bundler.newpart(b'check:heads', data=iter(pushop.remoteheads))
         else:
             affected = set()
             for branch, heads in pycompat.iteritems(pushop.pushbranchmap):
                 remoteheads, newheads, unsyncedheads, discardedheads = heads
                 if remoteheads is not None:
                     remote = set(remoteheads)
                     affected |= set(discardedheads) & remote
                     affected |= remote - set(newheads)
             if affected:
                 data = iter(sorted(affected))
                 bundler.newpart(b'check:updated-heads', data=data)
 
 
 def _pushing(pushop):
     """return True if we are pushing anything"""
     return bool(
         pushop.outgoing.missing
         or pushop.outdatedphases
         or pushop.outobsmarkers
         or pushop.outbookmarks
     )
 
 
 @b2partsgenerator(b'check-bookmarks')
 def _pushb2checkbookmarks(pushop, bundler):
     """insert bookmark move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasbookmarkcheck = b'bookmarks' in b2caps
     if not (pushop.outbookmarks and hasbookmarkcheck):
         return
     data = []
     for book, old, new in pushop.outbookmarks:
         data.append((book, old))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart(b'check:bookmarks', data=checkdata)
 
 
 @b2partsgenerator(b'check-phases')
 def _pushb2checkphases(pushop, bundler):
     """insert phase move checking"""
     if not _pushing(pushop) or pushop.force:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
     if pushop.remotephases is not None and hasphaseheads:
         # check that the remote phase has not changed
         checks = {p: [] for p in phases.allphases}
         checks[phases.public].extend(pushop.remotephases.publicheads)
         checks[phases.draft].extend(pushop.remotephases.draftroots)
         if any(pycompat.itervalues(checks)):
             for phase in checks:
                 checks[phase].sort()
             checkdata = phases.binaryencode(checks)
             bundler.newpart(b'check:phases', data=checkdata)
 
 
 @b2partsgenerator(b'changeset')
 def _pushb2ctx(pushop, bundler):
     """handle changegroup push through bundle2
 
     addchangegroup result is stored in the ``pushop.cgresult`` attribute.
     """
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     # Send known heads to the server for race detection.
     if not _pushcheckoutgoing(pushop):
         return
     pushop.repo.prepushoutgoinghooks(pushop)
 
     _pushb2ctxcheckheads(pushop, bundler)
 
     b2caps = bundle2.bundle2caps(pushop.remote)
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(pushop.repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
     cgstream = changegroup.makestream(
         pushop.repo, pushop.outgoing, version, b'push'
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         cgpart.addparam(b'version', version)
     if scmutil.istreemanifest(pushop.repo):
         cgpart.addparam(b'treemanifest', b'1')
     if b'exp-sidedata-flag' in pushop.repo.requirements:
         cgpart.addparam(b'exp-sidedata', b'1')
 
     def handlereply(op):
         """extract addchangegroup returns from server reply"""
         cgreplies = op.records.getreplies(cgpart.id)
         assert len(cgreplies[b'changegroup']) == 1
         pushop.cgresult = cgreplies[b'changegroup'][0][b'return']
 
     return handlereply
 
 
 @b2partsgenerator(b'phase')
 def _pushb2phases(pushop, bundler):
     """handle phase push through bundle2"""
     if b'phases' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
     ui = pushop.repo.ui
 
     legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
     haspushkey = b'pushkey' in b2caps
     hasphaseheads = b'heads' in b2caps.get(b'phases', ())
 
     if hasphaseheads and not legacyphase:
         return _pushb2phaseheads(pushop, bundler)
     elif haspushkey:
         return _pushb2phasespushkey(pushop, bundler)
 
 
 def _pushb2phaseheads(pushop, bundler):
     """push phase information through a bundle2 - binary part"""
     pushop.stepsdone.add(b'phases')
     if pushop.outdatedphases:
         updates = {p: [] for p in phases.allphases}
         updates[0].extend(h.node() for h in pushop.outdatedphases)
         phasedata = phases.binaryencode(updates)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 def _pushb2phasespushkey(pushop, bundler):
     """push phase information through a bundle2 - pushkey part"""
     pushop.stepsdone.add(b'phases')
     part2node = []
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, node in part2node:
             if partid == targetid:
                 raise error.Abort(_(b'updating %s to public failed') % node)
 
     enc = pushkey.encode
     for newremotehead in pushop.outdatedphases:
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'phases'))
         part.addparam(b'key', enc(newremotehead.hex()))
         part.addparam(b'old', enc(b'%d' % phases.draft))
         part.addparam(b'new', enc(b'%d' % phases.public))
         part2node.append((part.id, newremotehead))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         for partid, node in part2node:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             msg = None
             if not results:
                 msg = _(b'server ignored update of %s to public!\n') % node
             elif not int(results[0][b'return']):
                 msg = _(b'updating %s to public failed!\n') % node
             if msg is not None:
                 pushop.ui.warn(msg)
 
     return handlereply
 
 
 @b2partsgenerator(b'obsmarkers')
 def _pushb2obsmarkers(pushop, bundler):
     if b'obsmarkers' in pushop.stepsdone:
         return
     remoteversions = bundle2.obsmarkersversion(bundler.capabilities)
     if obsolete.commonversion(remoteversions) is None:
         return
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @b2partsgenerator(b'bookmarks')
 def _pushb2bookmarks(pushop, bundler):
     """handle bookmark push through bundle2"""
     if b'bookmarks' in pushop.stepsdone:
         return
     b2caps = bundle2.bundle2caps(pushop.remote)
 
     legacy = pushop.repo.ui.configlist(b'devel', b'legacy.exchange')
     legacybooks = b'bookmarks' in legacy
 
     if not legacybooks and b'bookmarks' in b2caps:
         return _pushb2bookmarkspart(pushop, bundler)
     elif b'pushkey' in b2caps:
         return _pushb2bookmarkspushkey(pushop, bundler)
 
 
 def _bmaction(old, new):
     """small utility for bookmark pushing"""
     if not old:
         return b'export'
     elif not new:
         return b'delete'
     return b'update'
 
 
 def _abortonsecretctx(pushop, node, b):
     """abort if a given bookmark points to a secret changeset"""
     if node and pushop.repo[node].phase() == phases.secret:
         raise error.Abort(
             _(b'cannot push bookmark %s as it points to a secret changeset') % b
         )
 
 
 def _pushb2bookmarkspart(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     if not pushop.outbookmarks:
         return
 
     allactions = []
     data = []
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         data.append((book, new))
         allactions.append((book, _bmaction(old, new)))
     checkdata = bookmod.binaryencode(data)
     bundler.newpart(b'bookmarks', data=checkdata)
 
     def handlereply(op):
         ui = pushop.ui
         # if success
         for book, action in allactions:
             ui.status(bookmsgmap[action][0] % book)
 
     return handlereply
 
 
 def _pushb2bookmarkspushkey(pushop, bundler):
     pushop.stepsdone.add(b'bookmarks')
     part2book = []
     enc = pushkey.encode
 
     def handlefailure(pushop, exc):
         targetid = int(exc.partid)
         for partid, book, action in part2book:
             if partid == targetid:
                 raise error.Abort(bookmsgmap[action][1].rstrip() % book)
         # we should not be called for part we did not generated
         assert False
 
     for book, old, new in pushop.outbookmarks:
         _abortonsecretctx(pushop, new, book)
         part = bundler.newpart(b'pushkey')
         part.addparam(b'namespace', enc(b'bookmarks'))
         part.addparam(b'key', enc(book))
         part.addparam(b'old', enc(hex(old)))
         part.addparam(b'new', enc(hex(new)))
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
         part2book.append((part.id, book, action))
         pushop.pkfailcb[part.id] = handlefailure
 
     def handlereply(op):
         ui = pushop.ui
         for partid, book, action in part2book:
             partrep = op.records.getreplies(partid)
             results = partrep[b'pushkey']
             assert len(results) <= 1
             if not results:
                 pushop.ui.warn(_(b'server ignored bookmark %s update\n') % book)
             else:
                 ret = int(results[0][b'return'])
                 if ret:
                     ui.status(bookmsgmap[action][0] % book)
                 else:
                     ui.warn(bookmsgmap[action][1] % book)
                     if pushop.bkresult is not None:
                         pushop.bkresult = 1
 
     return handlereply
 
 
 @b2partsgenerator(b'pushvars', idx=0)
 def _getbundlesendvars(pushop, bundler):
     '''send shellvars via bundle2'''
     pushvars = pushop.pushvars
     if pushvars:
         shellvars = {}
         for raw in pushvars:
             if b'=' not in raw:
                 msg = (
                     b"unable to parse variable '%s', should follow "
                     b"'KEY=VALUE' or 'KEY=' format"
                 )
                 raise error.Abort(msg % raw)
             k, v = raw.split(b'=', 1)
             shellvars[k] = v
 
         part = bundler.newpart(b'pushvars')
 
         for key, value in pycompat.iteritems(shellvars):
             part.addparam(key, value, mandatory=False)
 
 
 def _pushbundle2(pushop):
     """push data to the remote using bundle2
 
     The only currently supported type of data is changegroup but this will
     evolve in the future."""
     bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote))
     pushback = pushop.trmanager and pushop.ui.configbool(
         b'experimental', b'bundle2.pushback'
     )
 
     # create reply capability
     capsblob = bundle2.encodecaps(
         bundle2.getrepocaps(pushop.repo, allowpushback=pushback, role=b'client')
     )
     bundler.newpart(b'replycaps', data=capsblob)
     replyhandlers = []
     for partgenname in b2partsgenorder:
         partgen = b2partsgenmapping[partgenname]
         ret = partgen(pushop, bundler)
         if callable(ret):
             replyhandlers.append(ret)
     # do not push if nothing to push
     if bundler.nbparts <= 1:
         return
     stream = util.chunkbuffer(bundler.getchunks())
     try:
         try:
             with pushop.remote.commandexecutor() as e:
                 reply = e.callcommand(
                     b'unbundle',
                     {
                         b'bundle': stream,
                         b'heads': [b'force'],
                         b'url': pushop.remote.url(),
                     },
                 ).result()
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
         try:
             trgetter = None
             if pushback:
                 trgetter = pushop.trmanager.transaction
             op = bundle2.processbundle(pushop.repo, reply, trgetter)
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
         except bundle2.AbortFromPart as exc:
             pushop.ui.error(_(b'remote: %s\n') % exc)
             if exc.hint is not None:
                 pushop.ui.error(_(b'remote: %s\n') % (b'(%s)' % exc.hint))
             raise error.Abort(_(b'push failed on remote'))
     except error.PushkeyFailed as exc:
         partid = int(exc.partid)
         if partid not in pushop.pkfailcb:
             raise
         pushop.pkfailcb[partid](pushop, exc)
     for rephand in replyhandlers:
         rephand(op)
 
 
 def _pushchangeset(pushop):
     """Make the actual push of changeset bundle to remote repo"""
     if b'changesets' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'changesets')
     if not _pushcheckoutgoing(pushop):
         return
 
     # Should have verified this in push().
     assert pushop.remote.capable(b'unbundle')
 
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     # TODO: get bundlecaps from remote
     bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (
         outgoing.excluded or pushop.repo.changelog.filteredrevs
     ):
         # push everything,
         # use the fast path, no race possible on push
         cg = changegroup.makechangegroup(
             pushop.repo,
             outgoing,
             b'01',
             b'push',
             fastpath=True,
             bundlecaps=bundlecaps,
         )
     else:
         cg = changegroup.makechangegroup(
             pushop.repo, outgoing, b'01', b'push', bundlecaps=bundlecaps
         )
 
     # apply changegroup to remote
     # local repo finds heads on server, finds out what
     # revs it must push. once revs transferred, if server
     # finds it has different heads (someone else won
     # commit/push race), server aborts.
     if pushop.force:
         remoteheads = [b'force']
     else:
         remoteheads = pushop.remoteheads
     # ssh: return remote's addchangegroup()
     # http: return remote's addchangegroup() or 0 for error
     pushop.cgresult = pushop.remote.unbundle(cg, remoteheads, pushop.repo.url())
 
 
 def _pushsyncphase(pushop):
     """synchronise phase information locally and remotely"""
     cheads = pushop.commonheads
     # even when we don't push, exchanging phase data is useful
     remotephases = listkeys(pushop.remote, b'phases')
     if (
         pushop.ui.configbool(b'ui', b'_usedassubrepo')
         and remotephases  # server supports phases
         and pushop.cgresult is None  # nothing was pushed
         and remotephases.get(b'publishing', False)
     ):
         # When:
         # - this is a subrepo push
         # - and remote support phase
         # - and no changeset was pushed
         # - and remote is publishing
         # We may be in issue 3871 case!
         # We drop the possible phase synchronisation done by
         # courtesy to publish changesets possibly locally draft
         # on the remote.
         remotephases = {b'publishing': b'True'}
     if not remotephases:  # old server or public only reply from non-publishing
         _localphasemove(pushop, cheads)
         # don't push any phase data as there is nothing to push
     else:
         ana = phases.analyzeremotephases(pushop.repo, cheads, remotephases)
         pheads, droots = ana
         ### Apply remote phase on local
         if remotephases.get(b'publishing', False):
             _localphasemove(pushop, cheads)
         else:  # publish = False
             _localphasemove(pushop, pheads)
             _localphasemove(pushop, cheads, phases.draft)
         ### Apply local phase on remote
 
         if pushop.cgresult:
             if b'phases' in pushop.stepsdone:
                 # phases already pushed though bundle2
                 return
             outdated = pushop.outdatedphases
         else:
             outdated = pushop.fallbackoutdatedphases
 
         pushop.stepsdone.add(b'phases')
 
         # filter heads already turned public by the push
         outdated = [c for c in outdated if c.node() not in pheads]
         # fallback to independent pushkey command
         for newremotehead in outdated:
             with pushop.remote.commandexecutor() as e:
                 r = e.callcommand(
                     b'pushkey',
                     {
                         b'namespace': b'phases',
                         b'key': newremotehead.hex(),
                         b'old': b'%d' % phases.draft,
                         b'new': b'%d' % phases.public,
                     },
                 ).result()
 
             if not r:
                 pushop.ui.warn(
                     _(b'updating %s to public failed!\n') % newremotehead
                 )
 
 
 def _localphasemove(pushop, nodes, phase=phases.public):
     """move <nodes> to <phase> in the local source repo"""
     if pushop.trmanager:
         phases.advanceboundary(
             pushop.repo, pushop.trmanager.transaction(), phase, nodes
         )
     else:
         # repo is not locked, do not change any phases!
         # Informs the user that phases should have been moved when
         # applicable.
         actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()]
         phasestr = phases.phasenames[phase]
         if actualmoves:
             pushop.ui.status(
                 _(
                     b'cannot lock source repo, skipping '
                     b'local %s phase update\n'
                 )
                 % phasestr
             )
 
 
 def _pushobsolete(pushop):
     """utility function to push obsolete markers to a remote"""
     if b'obsmarkers' in pushop.stepsdone:
         return
     repo = pushop.repo
     remote = pushop.remote
     pushop.stepsdone.add(b'obsmarkers')
     if pushop.outobsmarkers:
         pushop.ui.debug(b'try to push obsolete markers to remote\n')
         rslts = []
         markers = obsutil.sortedmarkers(pushop.outobsmarkers)
         remotedata = obsolete._pushkeyescape(markers)
         for key in sorted(remotedata, reverse=True):
             # reverse sort to ensure we end with dump0
             data = remotedata[key]
             rslts.append(remote.pushkey(b'obsolete', key, b'', data))
         if [r for r in rslts if not r]:
             msg = _(b'failed to push some obsolete markers!\n')
             repo.ui.warn(msg)
 
 
 def _pushbookmark(pushop):
     """Update bookmark position on remote"""
     if pushop.cgresult == 0 or b'bookmarks' in pushop.stepsdone:
         return
     pushop.stepsdone.add(b'bookmarks')
     ui = pushop.ui
     remote = pushop.remote
 
     for b, old, new in pushop.outbookmarks:
         action = b'update'
         if not old:
             action = b'export'
         elif not new:
             action = b'delete'
 
         with remote.commandexecutor() as e:
             r = e.callcommand(
                 b'pushkey',
                 {
                     b'namespace': b'bookmarks',
                     b'key': b,
                     b'old': hex(old),
                     b'new': hex(new),
                 },
             ).result()
 
         if r:
             ui.status(bookmsgmap[action][0] % b)
         else:
             ui.warn(bookmsgmap[action][1] % b)
             # discovery can have set the value form invalid entry
             if pushop.bkresult is not None:
                 pushop.bkresult = 1
 
 
 class pulloperation(object):
     """A object that represent a single pull operation
 
     It purpose is to carry pull related state and very common operation.
 
     A new should be created at the beginning of each pull and discarded
     afterward.
     """
 
     def __init__(
         self,
         repo,
         remote,
         heads=None,
         force=False,
         bookmarks=(),
         remotebookmarks=None,
         streamclonerequested=None,
         includepats=None,
         excludepats=None,
         depth=None,
     ):
         # repo we pull into
         self.repo = repo
         # repo we pull from
         self.remote = remote
         # revision we try to pull (None is "all")
         self.heads = heads
         # bookmark pulled explicitly
         self.explicitbookmarks = [
             repo._bookmarks.expandname(bookmark) for bookmark in bookmarks
         ]
         # do we force pull?
         self.force = force
         # whether a streaming clone was requested
         self.streamclonerequested = streamclonerequested
         # transaction manager
         self.trmanager = None
         # set of common changeset between local and remote before pull
         self.common = None
         # set of pulled head
         self.rheads = None
         # list of missing changeset to fetch remotely
         self.fetch = None
         # remote bookmarks data
         self.remotebookmarks = remotebookmarks
         # result of changegroup pulling (used as return code by pull)
         self.cgresult = None
         # list of step already done
         self.stepsdone = set()
         # Whether we attempted a clone from pre-generated bundles.
         self.clonebundleattempted = False
         # Set of file patterns to include.
         self.includepats = includepats
         # Set of file patterns to exclude.
         self.excludepats = excludepats
         # Number of ancestor changesets to pull from each pulled head.
         self.depth = depth
 
     @util.propertycache
     def pulledsubset(self):
         """heads of the set of changeset target by the pull"""
         # compute target subset
         if self.heads is None:
             # We pulled every thing possible
             # sync on everything common
             c = set(self.common)
             ret = list(self.common)
             for n in self.rheads:
                 if n not in c:
                     ret.append(n)
             return ret
         else:
             # We pulled a specific subset
             # sync on this subset
             return self.heads
 
     @util.propertycache
     def canusebundle2(self):
         return not _forcebundle1(self)
 
     @util.propertycache
     def remotebundle2caps(self):
         return bundle2.bundle2caps(self.remote)
 
     def gettransaction(self):
         # deprecated; talk to trmanager directly
         return self.trmanager.transaction()
 
 
 class transactionmanager(util.transactional):
     """An object to manage the life cycle of a transaction
 
     It creates the transaction on demand and calls the appropriate hooks when
     closing the transaction."""
 
     def __init__(self, repo, source, url):
         self.repo = repo
         self.source = source
         self.url = url
         self._tr = None
 
     def transaction(self):
         """Return an open transaction object, constructing if necessary"""
         if not self._tr:
             trname = b'%s\n%s' % (self.source, util.hidepassword(self.url))
             self._tr = self.repo.transaction(trname)
             self._tr.hookargs[b'source'] = self.source
             self._tr.hookargs[b'url'] = self.url
         return self._tr
 
     def close(self):
         """close transaction if created"""
         if self._tr is not None:
             self._tr.close()
 
     def release(self):
         """release transaction if created"""
         if self._tr is not None:
             self._tr.release()
 
 
 def listkeys(remote, namespace):
     with remote.commandexecutor() as e:
         return e.callcommand(b'listkeys', {b'namespace': namespace}).result()
 
 
 def _fullpullbundle2(repo, pullop):
     # The server may send a partial reply, i.e. when inlining
     # pre-computed bundles. In that case, update the common
     # set based on the results and pull another bundle.
     #
     # There are two indicators that the process is finished:
     # - no changeset has been added, or
     # - all remote heads are known locally.
     # The head check must use the unfiltered view as obsoletion
     # markers can hide heads.
     unfi = repo.unfiltered()
     unficl = unfi.changelog
 
     def headsofdiff(h1, h2):
         """Returns heads(h1 % h2)"""
         res = unfi.set(b'heads(%ln %% %ln)', h1, h2)
         return {ctx.node() for ctx in res}
 
     def headsofunion(h1, h2):
         """Returns heads((h1 + h2) - null)"""
         res = unfi.set(b'heads((%ln + %ln - null))', h1, h2)
         return {ctx.node() for ctx in res}
 
     while True:
         old_heads = unficl.heads()
         clstart = len(unficl)
         _pullbundle2(pullop)
         if requirements.NARROW_REQUIREMENT in repo.requirements:
             # XXX narrow clones filter the heads on the server side during
             # XXX getbundle and result in partial replies as well.
             # XXX Disable pull bundles in this case as band aid to avoid
             # XXX extra round trips.
             break
         if clstart == len(unficl):
             break
         if all(unficl.hasnode(n) for n in pullop.rheads):
             break
         new_heads = headsofdiff(unficl.heads(), old_heads)
         pullop.common = headsofunion(new_heads, pullop.common)
         pullop.rheads = set(pullop.rheads) - pullop.common
 
 
 def add_confirm_callback(repo, pullop):
     """adds a finalize callback to transaction which can be used to show stats
     to user and confirm the pull before committing transaction"""
 
     tr = pullop.trmanager.transaction()
     scmutil.registersummarycallback(
         repo, tr, txnname=b'pull', as_validator=True
     )
     reporef = weakref.ref(repo.unfiltered())
 
     def prompt(tr):
         repo = reporef()
         cm = _(b'accept incoming changes (yn)?$$ &Yes $$ &No')
         if repo.ui.promptchoice(cm):
             raise error.Abort(b"user aborted")
 
     tr.addvalidator(b'900-pull-prompt', prompt)
 
 
 def pull(
     repo,
     remote,
     heads=None,
     force=False,
     bookmarks=(),
     opargs=None,
     streamclonerequested=None,
     includepats=None,
     excludepats=None,
     depth=None,
     confirm=None,
 ):
     """Fetch repository data from a remote.
 
     This is the main function used to retrieve data from a remote repository.
 
     ``repo`` is the local repository to clone into.
     ``remote`` is a peer instance.
     ``heads`` is an iterable of revisions we want to pull. ``None`` (the
     default) means to pull everything from the remote.
     ``bookmarks`` is an iterable of bookmarks requesting to be pulled. By
     default, all remote bookmarks are pulled.
     ``opargs`` are additional keyword arguments to pass to ``pulloperation``
     initialization.
     ``streamclonerequested`` is a boolean indicating whether a "streaming
     clone" is requested. A "streaming clone" is essentially a raw file copy
     of revlogs from the server. This only works when the local repository is
     empty. The default value of ``None`` means to respect the server
     configuration for preferring stream clones.
     ``includepats`` and ``excludepats`` define explicit file patterns to
     include and exclude in storage, respectively. If not defined, narrow
     patterns from the repo instance are used, if available.
     ``depth`` is an integer indicating the DAG depth of history we're
     interested in. If defined, for each revision specified in ``heads``, we
     will fetch up to this many of its ancestors and data associated with them.
     ``confirm`` is a boolean indicating whether the pull should be confirmed
     before committing the transaction. This overrides HGPLAIN.
 
     Returns the ``pulloperation`` created for this pull.
     """
     if opargs is None:
         opargs = {}
 
     # We allow the narrow patterns to be passed in explicitly to provide more
     # flexibility for API consumers.
     if includepats or excludepats:
         includepats = includepats or set()
         excludepats = excludepats or set()
     else:
         includepats, excludepats = repo.narrowpats
 
     narrowspec.validatepatterns(includepats)
     narrowspec.validatepatterns(excludepats)
 
     pullop = pulloperation(
         repo,
         remote,
         heads,
         force,
         bookmarks=bookmarks,
         streamclonerequested=streamclonerequested,
         includepats=includepats,
         excludepats=excludepats,
         depth=depth,
         **pycompat.strkwargs(opargs)
     )
 
     peerlocal = pullop.remote.local()
     if peerlocal:
         missing = set(peerlocal.requirements) - pullop.repo.supported
         if missing:
             msg = _(
                 b"required features are not"
                 b" supported in the destination:"
                 b" %s"
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
         wlock = repo.wlock()
     with wlock, repo.lock(), pullop.trmanager:
         if confirm or (
             repo.ui.configbool(b"pull", b"confirm") and not repo.ui.plain()
         ):
             add_confirm_callback(repo, pullop)
 
         # Use the modern wire protocol, if available.
         if remote.capable(b'command-changesetdata'):
             exchangev2.pull(pullop)
         else:
             # This should ideally be in _pullbundle2(). However, it needs to run
             # before discovery to avoid extra work.
             _maybeapplyclonebundle(pullop)
             streamclone.maybeperformlegacystreamclone(pullop)
             _pulldiscovery(pullop)
             if pullop.canusebundle2:
                 _fullpullbundle2(repo, pullop)
             _pullchangeset(pullop)
             _pullphase(pullop)
             _pullbookmarks(pullop)
             _pullobsolete(pullop)
 
     # storing remotenames
     if repo.ui.configbool(b'experimental', b'remotenames'):
         logexchange.pullremotenames(repo, remote)
 
     return pullop
 
 
 # list of steps to perform discovery before pull
 pulldiscoveryorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 pulldiscoverymapping = {}
 
 
 def pulldiscovery(stepname):
     """decorator for function performing discovery before pull
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated function will be added in order (this
     may matter).
 
     You can only use this decorator for a new step, if you want to wrap a step
     from an extension, change the pulldiscovery dictionary directly."""
 
     def dec(func):
         assert stepname not in pulldiscoverymapping
         pulldiscoverymapping[stepname] = func
         pulldiscoveryorder.append(stepname)
         return func
 
     return dec
 
 
 def _pulldiscovery(pullop):
     """Run all discovery steps"""
     for stepname in pulldiscoveryorder:
         step = pulldiscoverymapping[stepname]
         step(pullop)
 
 
 @pulldiscovery(b'b1:bookmarks')
 def _pullbookmarkbundle1(pullop):
     """fetch bookmark data in bundle1 case
 
     If not using bundle2, we have to fetch bookmarks before changeset
     discovery to reduce the chance and impact of race conditions."""
     if pullop.remotebookmarks is not None:
         return
     if pullop.canusebundle2 and b'listkeys' in pullop.remotebundle2caps:
         # all known bundle2 servers now support listkeys, but lets be nice with
         # new implementation.
         return
     books = listkeys(pullop.remote, b'bookmarks')
     pullop.remotebookmarks = bookmod.unhexlifybookmarks(books)
 
 
 @pulldiscovery(b'changegroup')
 def _pulldiscoverychangegroup(pullop):
     """discovery phase for the pull
 
     Current handle changeset discovery only, will change handle all discovery
     at some point."""
     tmp = discovery.findcommonincoming(
         pullop.repo, pullop.remote, heads=pullop.heads, force=pullop.force
     )
     common, fetch, rheads = tmp
     has_node = pullop.repo.unfiltered().changelog.index.has_node
     if fetch and rheads:
         # If a remote heads is filtered locally, put in back in common.
         #
         # This is a hackish solution to catch most of "common but locally
         # hidden situation".  We do not performs discovery on unfiltered
         # repository because it end up doing a pathological amount of round
         # trip for w huge amount of changeset we do not care about.
         #
         # If a set of such "common but filtered" changeset exist on the server
         # but are not including a remote heads, we'll not be able to detect it,
         scommon = set(common)
         for n in rheads:
             if has_node(n):
                 if n not in scommon:
                     common.append(n)
         if set(rheads).issubset(set(common)):
             fetch = []
     pullop.common = common
     pullop.fetch = fetch
     pullop.rheads = rheads
 
 
 def _pullbundle2(pullop):
     """pull data using bundle2
 
     For now, the only supported data are changegroup."""
     kwargs = {b'bundlecaps': caps20to10(pullop.repo, role=b'client')}
 
     # make ui easier to access
     ui = pullop.repo.ui
 
     # At the moment we don't do stream clones over bundle2. If that is
     # implemented then here's where the check for that will go.
     streaming = streamclone.canperformstreamclone(pullop, bundle2=True)[0]
 
     # declare pull perimeters
     kwargs[b'common'] = pullop.common
     kwargs[b'heads'] = pullop.heads or pullop.rheads
 
     # check server supports narrow and then adding includepats and excludepats
     servernarrow = pullop.remote.capable(wireprototypes.NARROWCAP)
     if servernarrow and pullop.includepats:
         kwargs[b'includepats'] = pullop.includepats
     if servernarrow and pullop.excludepats:
         kwargs[b'excludepats'] = pullop.excludepats
 
     if streaming:
         kwargs[b'cg'] = False
         kwargs[b'stream'] = True
         pullop.stepsdone.add(b'changegroup')
         pullop.stepsdone.add(b'phases')
 
     else:
         # pulling changegroup
         pullop.stepsdone.add(b'changegroup')
 
         kwargs[b'cg'] = pullop.fetch
 
         legacyphase = b'phases' in ui.configlist(b'devel', b'legacy.exchange')
         hasbinaryphase = b'heads' in pullop.remotebundle2caps.get(b'phases', ())
         if not legacyphase and hasbinaryphase:
             kwargs[b'phases'] = True
             pullop.stepsdone.add(b'phases')
 
         if b'listkeys' in pullop.remotebundle2caps:
             if b'phases' not in pullop.stepsdone:
                 kwargs[b'listkeys'] = [b'phases']
 
     bookmarksrequested = False
     legacybookmark = b'bookmarks' in ui.configlist(b'devel', b'legacy.exchange')
     hasbinarybook = b'bookmarks' in pullop.remotebundle2caps
 
     if pullop.remotebookmarks is not None:
         pullop.stepsdone.add(b'request-bookmarks')
 
     if (
         b'request-bookmarks' not in pullop.stepsdone
         and pullop.remotebookmarks is None
         and not legacybookmark
         and hasbinarybook
     ):
         kwargs[b'bookmarks'] = True
         bookmarksrequested = True
 
     if b'listkeys' in pullop.remotebundle2caps:
         if b'request-bookmarks' not in pullop.stepsdone:
             # make sure to always includes bookmark data when migrating
             # `hg incoming --bundle` to using this function.
             pullop.stepsdone.add(b'request-bookmarks')
             kwargs.setdefault(b'listkeys', []).append(b'bookmarks')
 
     # If this is a full pull / clone and the server supports the clone bundles
     # feature, tell the server whether we attempted a clone bundle. The
     # presence of this flag indicates the client supports clone bundles. This
     # will enable the server to treat clients that support clone bundles
     # differently from those that don't.
     if (
         pullop.remote.capable(b'clonebundles')
         and pullop.heads is None
         and list(pullop.common) == [nullid]
     ):
         kwargs[b'cbattempted'] = pullop.clonebundleattempted
 
     if streaming:
         pullop.repo.ui.status(_(b'streaming all changes\n'))
     elif not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
     else:
         if pullop.heads is None and list(pullop.common) == [nullid]:
             pullop.repo.ui.status(_(b"requesting all changes\n"))
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps)
         if obsolete.commonversion(remoteversions) is not None:
             kwargs[b'obsmarkers'] = True
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
         bundle = e.callcommand(b'getbundle', args).result()
 
         try:
             op = bundle2.bundleoperation(
                 pullop.repo, pullop.gettransaction, source=b'pull'
             )
             op.modes[b'bookmarks'] = b'records'
             bundle2.processbundle(pullop.repo, bundle, op=op)
         except bundle2.AbortFromPart as exc:
             pullop.repo.ui.error(_(b'remote: abort: %s\n') % exc)
             raise error.Abort(_(b'pull failed on remote'), hint=exc.hint)
         except error.BundleValueError as exc:
             raise error.Abort(_(b'missing support for %s') % exc)
 
     if pullop.fetch:
         pullop.cgresult = bundle2.combinechangegroupresults(op)
 
     # processing phases change
     for namespace, value in op.records[b'listkeys']:
         if namespace == b'phases':
             _pullapplyphases(pullop, value)
 
     # processing bookmark update
     if bookmarksrequested:
         books = {}
         for record in op.records[b'bookmarks']:
             books[record[b'bookmark']] = record[b"node"]
         pullop.remotebookmarks = books
     else:
         for namespace, value in op.records[b'listkeys']:
             if namespace == b'bookmarks':
                 pullop.remotebookmarks = bookmod.unhexlifybookmarks(value)
 
     # bookmark data were either already there or pulled in the bundle
     if pullop.remotebookmarks is not None:
         _pullbookmarks(pullop)
 
 
 def _pullbundle2extraprepare(pullop, kwargs):
     """hook function so that extensions can extend the getbundle call"""
 
 
 def _pullchangeset(pullop):
     """pull changeset from unbundle into the local repo"""
     # We delay the open of the transaction as late as possible so we
     # don't open transaction for nothing or you break future useful
     # rollback call
     if b'changegroup' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'changegroup')
     if not pullop.fetch:
         pullop.repo.ui.status(_(b"no changes found\n"))
         pullop.cgresult = 0
         return
     tr = pullop.gettransaction()
     if pullop.heads is None and list(pullop.common) == [nullid]:
         pullop.repo.ui.status(_(b"requesting all changes\n"))
     elif pullop.heads is None and pullop.remote.capable(b'changegroupsubset'):
         # issue1320, avoid a race if remote changed after discovery
         pullop.heads = pullop.rheads
 
     if pullop.remote.capable(b'getbundle'):
         # TODO: get bundlecaps from remote
         cg = pullop.remote.getbundle(
             b'pull', common=pullop.common, heads=pullop.heads or pullop.rheads
         )
     elif pullop.heads is None:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroup',
                 {
                     b'nodes': pullop.fetch,
                     b'source': b'pull',
                 },
             ).result()
 
     elif not pullop.remote.capable(b'changegroupsubset'):
         raise error.Abort(
             _(
                 b"partial pull cannot be done because "
                 b"other repository doesn't support "
                 b"changegroupsubset."
             )
         )
     else:
         with pullop.remote.commandexecutor() as e:
             cg = e.callcommand(
                 b'changegroupsubset',
                 {
                     b'bases': pullop.fetch,
                     b'heads': pullop.heads,
                     b'source': b'pull',
                 },
             ).result()
 
     bundleop = bundle2.applybundle(
         pullop.repo, cg, tr, b'pull', pullop.remote.url()
     )
     pullop.cgresult = bundle2.combinechangegroupresults(bundleop)
 
 
 def _pullphase(pullop):
     # Get remote phases data from remote
     if b'phases' in pullop.stepsdone:
         return
     remotephases = listkeys(pullop.remote, b'phases')
     _pullapplyphases(pullop, remotephases)
 
 
 def _pullapplyphases(pullop, remotephases):
     """apply phase movement from observed remote state"""
     if b'phases' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'phases')
     publishing = bool(remotephases.get(b'publishing', False))
     if remotephases and not publishing:
         # remote is new and non-publishing
         pheads, _dr = phases.analyzeremotephases(
             pullop.repo, pullop.pulledsubset, remotephases
         )
         dheads = pullop.pulledsubset
     else:
         # Remote is old or publishing all common changesets
         # should be seen as public
         pheads = pullop.pulledsubset
         dheads = []
     unfi = pullop.repo.unfiltered()
     phase = unfi._phasecache.phase
     rev = unfi.changelog.index.get_rev
     public = phases.public
     draft = phases.draft
 
     # exclude changesets already public locally and update the others
     pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public]
     if pheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, public, pheads)
 
     # exclude changesets already draft locally and update the others
     dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft]
     if dheads:
         tr = pullop.gettransaction()
         phases.advanceboundary(pullop.repo, tr, draft, dheads)
 
 
 def _pullbookmarks(pullop):
     """process the remote bookmark information to update the local one"""
     if b'bookmarks' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'bookmarks')
     repo = pullop.repo
     remotebookmarks = pullop.remotebookmarks
     bookmod.updatefromremote(
         repo.ui,
         repo,
         remotebookmarks,
         pullop.remote.url(),
         pullop.gettransaction,
         explicit=pullop.explicitbookmarks,
     )
 
 
 def _pullobsolete(pullop):
     """utility function to pull obsolete markers from a remote
 
     The `gettransaction` is function that return the pull transaction, creating
     one if necessary. We return the transaction to inform the calling code that
     a new transaction have been created (when applicable).
 
     Exists mostly to allow overriding for experimentation purpose"""
     if b'obsmarkers' in pullop.stepsdone:
         return
     pullop.stepsdone.add(b'obsmarkers')
     tr = None
     if obsolete.isenabled(pullop.repo, obsolete.exchangeopt):
         pullop.repo.ui.debug(b'fetching remote obsolete markers\n')
         remoteobs = listkeys(pullop.remote, b'obsolete')
         if b'dump0' in remoteobs:
             tr = pullop.gettransaction()
             markers = []
             for key in sorted(remoteobs, reverse=True):
                 if key.startswith(b'dump'):
                     data = util.b85decode(remoteobs[key])
                     version, newmarks = obsolete._readmarkers(data)
                     markers += newmarks
             if markers:
                 pullop.repo.obsstore.add(tr, markers)
             pullop.repo.invalidatevolatilesets()
     return tr
 
 
 def applynarrowacl(repo, kwargs):
     """Apply narrow fetch access control.
 
     This massages the named arguments for getbundle wire protocol commands
     so requested data is filtered through access control rules.
     """
     ui = repo.ui
     # TODO this assumes existence of HTTP and is a layering violation.
     username = ui.shortuser(ui.environ.get(b'REMOTE_USER') or ui.username())
     user_includes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.includes',
         ui.configlist(_NARROWACL_SECTION, b'default.includes'),
     )
     user_excludes = ui.configlist(
         _NARROWACL_SECTION,
         username + b'.excludes',
         ui.configlist(_NARROWACL_SECTION, b'default.excludes'),
     )
     if not user_includes:
         raise error.Abort(
             _(b"%s configuration for user %s is empty")
             % (_NARROWACL_SECTION, username)
         )
 
     user_includes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_includes
     ]
     user_excludes = [
         b'path:.' if p == b'*' else b'path:' + p for p in user_excludes
     ]
 
     req_includes = set(kwargs.get('includepats', []))
     req_excludes = set(kwargs.get('excludepats', []))
 
     req_includes, req_excludes, invalid_includes = narrowspec.restrictpatterns(
         req_includes, req_excludes, user_includes, user_excludes
     )
 
     if invalid_includes:
         raise error.Abort(
             _(b"The following includes are not accessible for %s: %s")
             % (username, stringutil.pprint(invalid_includes))
         )
 
     new_args = {}
     new_args.update(kwargs)
     new_args['narrow'] = True
     new_args['narrow_acl'] = True
     new_args['includepats'] = req_includes
     if req_excludes:
         new_args['excludepats'] = req_excludes
 
     return new_args
 
 
 def _computeellipsis(repo, common, heads, known, match, depth=None):
     """Compute the shape of a narrowed DAG.
 
     Args:
       repo: The repository we're transferring.
       common: The roots of the DAG range we're transferring.
               May be just [nullid], which means all ancestors of heads.
       heads: The heads of the DAG range we're transferring.
       match: The narrowmatcher that allows us to identify relevant changes.
       depth: If not None, only consider nodes to be full nodes if they are at
              most depth changesets away from one of heads.
 
     Returns:
       A tuple of (visitnodes, relevant_nodes, ellipsisroots) where:
 
         visitnodes: The list of nodes (either full or ellipsis) which
                     need to be sent to the client.
         relevant_nodes: The set of changelog nodes which change a file inside
                  the narrowspec. The client needs these as non-ellipsis nodes.
         ellipsisroots: A dict of {rev: parents} that is used in
                        narrowchangegroup to produce ellipsis nodes with the
                        correct parents.
     """
     cl = repo.changelog
     mfl = repo.manifestlog
 
     clrev = cl.rev
 
     commonrevs = {clrev(n) for n in common} | {nullrev}
     headsrevs = {clrev(n) for n in heads}
 
     if depth:
         revdepth = {h: 0 for h in headsrevs}
 
     ellipsisheads = collections.defaultdict(set)
     ellipsisroots = collections.defaultdict(set)
 
     def addroot(head, curchange):
         """Add a root to an ellipsis head, splitting heads with 3 roots."""
         ellipsisroots[head].add(curchange)
         # Recursively split ellipsis heads with 3 roots by finding the
         # roots' youngest common descendant which is an elided merge commit.
         # That descendant takes 2 of the 3 roots as its own, and becomes a
         # root of the head.
         while len(ellipsisroots[head]) > 2:
             child, roots = splithead(head)
             splitroots(head, child, roots)
             head = child  # Recurse in case we just added a 3rd root
 
     def splitroots(head, child, roots):
         ellipsisroots[head].difference_update(roots)
         ellipsisroots[head].add(child)
         ellipsisroots[child].update(roots)
         ellipsisroots[child].discard(child)
 
     def splithead(head):
         r1, r2, r3 = sorted(ellipsisroots[head])
         for nr1, nr2 in ((r2, r3), (r1, r3), (r1, r2)):
             mid = repo.revs(
                 b'sort(merge() & %d::%d & %d::%d, -rev)', nr1, head, nr2, head
             )
             for j in mid:
                 if j == nr2:
                     return nr2, (nr1, nr2)
                 if j not in ellipsisroots or len(ellipsisroots[j]) < 2:
                     return j, (nr1, nr2)
         raise error.Abort(
             _(
                 b'Failed to split up ellipsis node! head: %d, '
                 b'roots: %d %d %d'
             )
             % (head, r1, r2, r3)
         )
 
     missing = list(cl.findmissingrevs(common=commonrevs, heads=headsrevs))
     visit = reversed(missing)
     relevant_nodes = set()
     visitnodes = [cl.node(m) for m in missing]
     required = set(headsrevs) | known
     for rev in visit:
         clrev = cl.changelogrevision(rev)
         ps = [prev for prev in cl.parentrevs(rev) if prev != nullrev]
         if depth is not None:
             curdepth = revdepth[rev]
             for p in ps:
                 revdepth[p] = min(curdepth + 1, revdepth.get(p, depth + 1))
         needed = False
         shallow_enough = depth is None or revdepth[rev] <= depth
         if shallow_enough:
             curmf = mfl[clrev.manifest].read()
             if ps:
                 # We choose to not trust the changed files list in
                 # changesets because it's not always correct. TODO: could
                 # we trust it for the non-merge case?
                 p1mf = mfl[cl.changelogrevision(ps[0]).manifest].read()
                 needed = bool(curmf.diff(p1mf, match))
                 if not needed and len(ps) > 1:
                     # For merge changes, the list of changed files is not
                     # helpful, since we need to emit the merge if a file
                     # in the narrow spec has changed on either side of the
                     # merge. As a result, we do a manifest diff to check.
                     p2mf = mfl[cl.changelogrevision(ps[1]).manifest].read()
                     needed = bool(curmf.diff(p2mf, match))
             else:
                 # For a root node, we need to include the node if any
                 # files in the node match the narrowspec.
                 needed = any(curmf.walk(match))
 
         if needed:
             for head in ellipsisheads[rev]:
                 addroot(head, rev)
             for p in ps:
                 required.add(p)
             relevant_nodes.add(cl.node(rev))
         else:
             if not ps:
                 ps = [nullrev]
             if rev in required:
                 for head in ellipsisheads[rev]:
                     addroot(head, rev)
                 for p in ps:
                     ellipsisheads[p].add(rev)
             else:
                 for p in ps:
                     ellipsisheads[p] |= ellipsisheads[rev]
 
     # add common changesets as roots of their reachable ellipsis heads
     for c in commonrevs:
         for head in ellipsisheads[c]:
             addroot(head, c)
     return visitnodes, relevant_nodes, ellipsisroots
 
 
 def caps20to10(repo, role):
     """return a set with appropriate options to use bundle20 during getbundle"""
     caps = {b'HG20'}
     capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role=role))
     caps.add(b'bundle2=' + urlreq.quote(capsblob))
     return caps
 
 
 # List of names of steps to perform for a bundle2 for getbundle, order matters.
 getbundle2partsorder = []
 
 # Mapping between step name and function
 #
 # This exists to help extensions wrap steps if necessary
 getbundle2partsmapping = {}
 
 
 def getbundle2partsgenerator(stepname, idx=None):
     """decorator for function generating bundle2 part for getbundle
 
     The function is added to the step -> function mapping and appended to the
     list of steps.  Beware that decorated functions will be added in order
     (this may matter).
 
     You can only use this decorator for new steps, if you want to wrap a step
     from an extension, attack the getbundle2partsmapping dictionary directly."""
 
     def dec(func):
         assert stepname not in getbundle2partsmapping
         getbundle2partsmapping[stepname] = func
         if idx is None:
             getbundle2partsorder.append(stepname)
         else:
             getbundle2partsorder.insert(idx, stepname)
         return func
 
     return dec
 
 
 def bundle2requested(bundlecaps):
     if bundlecaps is not None:
         return any(cap.startswith(b'HG2') for cap in bundlecaps)
     return False
 
 
 def getbundlechunks(
-    repo, source, heads=None, common=None, bundlecaps=None, **kwargs
+    repo,
+    source,
+    heads=None,
+    common=None,
+    bundlecaps=None,
+    remote_sidedata=None,
+    **kwargs
 ):
     """Return chunks constituting a bundle's raw data.
 
     Could be a bundle HG10 or a bundle HG20 depending on bundlecaps
     passed.
 
     Returns a 2-tuple of a dict with metadata about the generated bundle
     and an iterator over raw chunks (of varying sizes).
     """
     kwargs = pycompat.byteskwargs(kwargs)
     info = {}
     usebundle2 = bundle2requested(bundlecaps)
     # bundle10 case
     if not usebundle2:
         if bundlecaps and not kwargs.get(b'cg', True):
             raise ValueError(
                 _(b'request for bundle10 must include changegroup')
             )
 
         if kwargs:
             raise ValueError(
                 _(b'unsupported getbundle arguments: %s')
                 % b', '.join(sorted(kwargs.keys()))
             )
         outgoing = _computeoutgoing(repo, heads, common)
         info[b'bundleversion'] = 1
         return (
             info,
             changegroup.makestream(
-                repo, outgoing, b'01', source, bundlecaps=bundlecaps
+                repo,
+                outgoing,
+                b'01',
+                source,
+                bundlecaps=bundlecaps,
+                remote_sidedata=remote_sidedata,
             ),
         )
 
     # bundle20 case
     info[b'bundleversion'] = 2
     b2caps = {}
     for bcaps in bundlecaps:
         if bcaps.startswith(b'bundle2='):
             blob = urlreq.unquote(bcaps[len(b'bundle2=') :])
             b2caps.update(bundle2.decodecaps(blob))
     bundler = bundle2.bundle20(repo.ui, b2caps)
 
     kwargs[b'heads'] = heads
     kwargs[b'common'] = common
 
     for name in getbundle2partsorder:
         func = getbundle2partsmapping[name]
         func(
             bundler,
             repo,
             source,
             bundlecaps=bundlecaps,
             b2caps=b2caps,
+            remote_sidedata=remote_sidedata,
             **pycompat.strkwargs(kwargs)
         )
 
     info[b'prefercompressed'] = bundler.prefercompressed
 
     return info, bundler.getchunks()
 
 
 @getbundle2partsgenerator(b'stream2')
 def _getbundlestream2(bundler, repo, *args, **kwargs):
     return bundle2.addpartbundlestream2(bundler, repo, **kwargs)
 
 
 @getbundle2partsgenerator(b'changegroup')
 def _getbundlechangegrouppart(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
+    remote_sidedata=None,
     **kwargs
 ):
     """add a changegroup part to the requested bundle"""
     if not kwargs.get('cg', True) or not b2caps:
         return
 
     version = b'01'
     cgversions = b2caps.get(b'changegroup')
     if cgversions:  # 3.1 and 3.2 ship with an empty value
         cgversions = [
             v
             for v in cgversions
             if v in changegroup.supportedoutgoingversions(repo)
         ]
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
 
     outgoing = _computeoutgoing(repo, heads, common)
     if not outgoing.missing:
         return
 
     if kwargs.get('narrow', False):
         include = sorted(filter(bool, kwargs.get('includepats', [])))
         exclude = sorted(filter(bool, kwargs.get('excludepats', [])))
         matcher = narrowspec.match(repo.root, include=include, exclude=exclude)
     else:
         matcher = None
 
     cgstream = changegroup.makestream(
-        repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
+        repo,
+        outgoing,
+        version,
+        source,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
     )
 
     part = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
         part.addparam(b'version', version)
 
     part.addparam(b'nbchanges', b'%d' % len(outgoing.missing), mandatory=False)
 
     if scmutil.istreemanifest(repo):
         part.addparam(b'treemanifest', b'1')
 
     if b'exp-sidedata-flag' in repo.requirements:
         part.addparam(b'exp-sidedata', b'1')
 
     if (
         kwargs.get('narrow', False)
         and kwargs.get('narrow_acl', False)
         and (include or exclude)
     ):
         # this is mandatory because otherwise ACL clients won't work
         narrowspecpart = bundler.newpart(b'Narrow:responsespec')
         narrowspecpart.data = b'%s\0%s' % (
             b'\n'.join(include),
             b'\n'.join(exclude),
         )
 
 
 @getbundle2partsgenerator(b'bookmarks')
 def _getbundlebookmarkpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add a bookmark part to the requested bundle"""
     if not kwargs.get('bookmarks', False):
         return
     if not b2caps or b'bookmarks' not in b2caps:
         raise error.Abort(_(b'no common bookmarks exchange method'))
     books = bookmod.listbinbookmarks(repo)
     data = bookmod.binaryencode(books)
     if data:
         bundler.newpart(b'bookmarks', data=data)
 
 
 @getbundle2partsgenerator(b'listkeys')
 def _getbundlelistkeysparts(
     bundler, repo, source, bundlecaps=None, b2caps=None, **kwargs
 ):
     """add parts containing listkeys namespaces to the requested bundle"""
     listkeys = kwargs.get('listkeys', ())
     for namespace in listkeys:
         part = bundler.newpart(b'listkeys')
         part.addparam(b'namespace', namespace)
         keys = repo.listkeys(namespace).items()
         part.data = pushkey.encodekeys(keys)
 
 
 @getbundle2partsgenerator(b'obsmarkers')
 def _getbundleobsmarkerpart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add an obsolescence markers part to the requested bundle"""
     if kwargs.get('obsmarkers', False):
         if heads is None:
             heads = repo.heads()
         subset = [c.node() for c in repo.set(b'::%ln', heads)]
         markers = repo.obsstore.relevantmarkers(subset)
         markers = obsutil.sortedmarkers(markers)
         bundle2.buildobsmarkerspart(bundler, markers)
 
 
 @getbundle2partsgenerator(b'phases')
 def _getbundlephasespart(
     bundler, repo, source, bundlecaps=None, b2caps=None, heads=None, **kwargs
 ):
     """add phase heads part to the requested bundle"""
     if kwargs.get('phases', False):
         if not b2caps or b'heads' not in b2caps.get(b'phases'):
             raise error.Abort(_(b'no common phases exchange method'))
         if heads is None:
             heads = repo.heads()
 
         headsbyphase = collections.defaultdict(set)
         if repo.publishing():
             headsbyphase[phases.public] = heads
         else:
             # find the appropriate heads to move
 
             phase = repo._phasecache.phase
             node = repo.changelog.node
             rev = repo.changelog.rev
             for h in heads:
                 headsbyphase[phase(repo, rev(h))].add(h)
             seenphases = list(headsbyphase.keys())
 
             # We do not handle anything but public and draft phase for now)
             if seenphases:
                 assert max(seenphases) <= phases.draft
 
             # if client is pulling non-public changesets, we need to find
             # intermediate public heads.
             draftheads = headsbyphase.get(phases.draft, set())
             if draftheads:
                 publicheads = headsbyphase.get(phases.public, set())
 
                 revset = b'heads(only(%ln, %ln) and public())'
                 extraheads = repo.revs(revset, draftheads, publicheads)
                 for r in extraheads:
                     headsbyphase[phases.public].add(node(r))
 
         # transform data in a format used by the encoding function
         phasemapping = {
             phase: sorted(headsbyphase[phase]) for phase in phases.allphases
         }
 
         # generate the actual part
         phasedata = phases.binaryencode(phasemapping)
         bundler.newpart(b'phase-heads', data=phasedata)
 
 
 @getbundle2partsgenerator(b'hgtagsfnodes')
 def _getbundletagsfnodes(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the .hgtags filenodes mapping.
 
     Only values for heads in this bundle will be transferred.
 
     The part data consists of pairs of 20 byte changeset node and .hgtags
     filenodes raw values.
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     if not b2caps or not (kwargs.get('cg', True) and b'hgtagsfnodes' in b2caps):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addparttagsfnodescache(repo, bundler, outgoing)
 
 
 @getbundle2partsgenerator(b'cache:rev-branch-cache')
 def _getbundlerevbranchcache(
     bundler,
     repo,
     source,
     bundlecaps=None,
     b2caps=None,
     heads=None,
     common=None,
     **kwargs
 ):
     """Transfer the rev-branch-cache mapping
 
     The payload is a series of data related to each branch
 
     1) branch name length
     2) number of open heads
     3) number of closed heads
     4) open heads nodes
     5) closed heads nodes
     """
     # Don't send unless:
     # - changeset are being exchanged,
     # - the client supports it.
     # - narrow bundle isn't in play (not currently compatible).
     if (
         not kwargs.get('cg', True)
         or not b2caps
         or b'rev-branch-cache' not in b2caps
         or kwargs.get('narrow', False)
         or repo.ui.has_section(_NARROWACL_SECTION)
     ):
         return
 
     outgoing = _computeoutgoing(repo, heads, common)
     bundle2.addpartrevbranchcache(repo, bundler, outgoing)
 
 
 def check_heads(repo, their_heads, context):
     """check if the heads of a repo have been modified
 
     Used by peer for unbundling.
     """
     heads = repo.heads()
     heads_hash = hashutil.sha1(b''.join(sorted(heads))).digest()
     if not (
         their_heads == [b'force']
         or their_heads == heads
         or their_heads == [b'hashed', heads_hash]
     ):
         # someone else committed/pushed/unbundled while we
         # were transferring data
         raise error.PushRaced(
             b'repository changed while %s - please try again' % context
         )
 
 
 def unbundle(repo, cg, heads, source, url):
     """Apply a bundle to a repo.
 
     this function makes sure the repo is locked during the application and have
     mechanism to check that no push race occurred between the creation of the
     bundle and its application.
 
     If the push was raced as PushRaced exception is raised."""
     r = 0
     # need a transaction when processing a bundle2 stream
     # [wlock, lock, tr] - needs to be an array so nested functions can modify it
     lockandtr = [None, None, None]
     recordout = None
     # quick fix for output mismatch with bundle2 in 3.4
     captureoutput = repo.ui.configbool(
         b'experimental', b'bundle2-output-capture'
     )
     if url.startswith(b'remote:http:') or url.startswith(b'remote:https:'):
         captureoutput = True
     try:
         # note: outside bundle1, 'heads' is expected to be empty and this
         # 'check_heads' call wil be a no-op
         check_heads(repo, heads, b'uploading changes')
         # push can proceed
         if not isinstance(cg, bundle2.unbundle20):
             # legacy case: bundle1 (changegroup 01)
             txnname = b"\n".join([source, util.hidepassword(url)])
             with repo.lock(), repo.transaction(txnname) as tr:
                 op = bundle2.applybundle(repo, cg, tr, source, url)
                 r = bundle2.combinechangegroupresults(op)
         else:
             r = None
             try:
 
                 def gettransaction():
                     if not lockandtr[2]:
                         if not bookmod.bookmarksinstore(repo):
                             lockandtr[0] = repo.wlock()
                         lockandtr[1] = repo.lock()
                         lockandtr[2] = repo.transaction(source)
                         lockandtr[2].hookargs[b'source'] = source
                         lockandtr[2].hookargs[b'url'] = url
                         lockandtr[2].hookargs[b'bundle2'] = b'1'
                     return lockandtr[2]
 
                 # Do greedy locking by default until we're satisfied with lazy
                 # locking.
                 if not repo.ui.configbool(
                     b'experimental', b'bundle2lazylocking'
                 ):
                     gettransaction()
 
                 op = bundle2.bundleoperation(
                     repo,
                     gettransaction,
                     captureoutput=captureoutput,
                     source=b'push',
                 )
                 try:
                     op = bundle2.processbundle(repo, cg, op=op)
                 finally:
                     r = op.reply
                     if captureoutput and r is not None:
                         repo.ui.pushbuffer(error=True, subproc=True)
 
                         def recordout(output):
                             r.newpart(b'output', data=output, mandatory=False)
 
                 if lockandtr[2] is not None:
                     lockandtr[2].close()
             except BaseException as exc:
                 exc.duringunbundle2 = True
                 if captureoutput and r is not None:
                     parts = exc._bundle2salvagedoutput = r.salvageoutput()
 
                     def recordout(output):
                         part = bundle2.bundlepart(
                             b'output', data=output, mandatory=False
                         )
                         parts.append(part)
 
                 raise
     finally:
         lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0])
         if recordout is not None:
             recordout(repo.ui.popbuffer())
     return r
 
 
 def _maybeapplyclonebundle(pullop):
     """Apply a clone bundle from a remote, if possible."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     if not repo.ui.configbool(b'ui', b'clonebundles'):
         return
 
     # Only run if local repo is empty.
     if len(repo):
         return
 
     if pullop.heads:
         return
 
     if not remote.capable(b'clonebundles'):
         return
 
     with remote.commandexecutor() as e:
         res = e.callcommand(b'clonebundles', {}).result()
 
     # If we call the wire protocol command, that's good enough to record the
     # attempt.
     pullop.clonebundleattempted = True
 
     entries = bundlecaches.parseclonebundlesmanifest(repo, res)
     if not entries:
         repo.ui.note(
             _(
                 b'no clone bundles available on remote; '
                 b'falling back to regular clone\n'
             )
         )
         return
 
     entries = bundlecaches.filterclonebundleentries(
         repo, entries, streamclonerequested=pullop.streamclonerequested
     )
 
     if not entries:
         # There is a thundering herd concern here. However, if a server
         # operator doesn't advertise bundles appropriate for its clients,
         # they deserve what's coming. Furthermore, from a client's
         # perspective, no automatic fallback would mean not being able to
         # clone!
         repo.ui.warn(
             _(
                 b'no compatible clone bundles available on server; '
                 b'falling back to regular clone\n'
             )
         )
         repo.ui.warn(
             _(b'(you may want to report this to the server operator)\n')
         )
         return
 
     entries = bundlecaches.sortclonebundleentries(repo.ui, entries)
 
     url = entries[0][b'URL']
     repo.ui.status(_(b'applying clone bundle from %s\n') % url)
     if trypullbundlefromurl(repo.ui, repo, url):
         repo.ui.status(_(b'finished applying clone bundle\n'))
     # Bundle failed.
     #
     # We abort by default to avoid the thundering herd of
     # clients flooding a server that was expecting expensive
     # clone load to be offloaded.
     elif repo.ui.configbool(b'ui', b'clonebundlefallback'):
         repo.ui.warn(_(b'falling back to normal clone\n'))
     else:
         raise error.Abort(
             _(b'error applying bundle'),
             hint=_(
                 b'if this error persists, consider contacting '
                 b'the server operator or disable clone '
                 b'bundles via '
                 b'"--config ui.clonebundles=false"'
             ),
         )
 
 
 def trypullbundlefromurl(ui, repo, url):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction(b'bundleurl') as tr:
         try:
             fh = urlmod.open(ui, url)
             cg = readbundle(ui, fh, b'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):
                 cg.apply(repo)
             else:
                 bundle2.applybundle(repo, cg, tr, b'clonebundles', url)
             return True
         except urlerr.httperror as e:
             ui.warn(
                 _(b'HTTP error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e)
             )
         except urlerr.urlerror as e:
             ui.warn(
                 _(b'error fetching bundle: %s\n')
                 % stringutil.forcebytestr(e.reason)
             )
 
         return False

mercurial/exchangev2.py

@@ -1,799 +1,807 b''
 # exchangev2.py - repository exchange for wire protocol version 2
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import weakref
 
 from .i18n import _
 from .node import (
     nullid,
     short,
 )
 from . import (
     bookmarks,
     error,
     mdiff,
     narrowspec,
     phases,
     pycompat,
     requirements as requirementsmod,
     setdiscovery,
 )
 from .interfaces import repository
 
 
 def pull(pullop):
     """Pull using wire protocol version 2."""
     repo = pullop.repo
     remote = pullop.remote
 
     usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
 
     # If this is a clone and it was requested to perform a "stream clone",
     # we obtain the raw files data from the remote then fall back to an
     # incremental pull. This is somewhat hacky and is not nearly robust enough
     # for long-term usage.
     if usingrawchangelogandmanifest:
         with repo.transaction(b'clone'):
             _fetchrawstorefiles(repo, remote)
             repo.invalidate(clearfilecache=True)
 
     tr = pullop.trmanager.transaction()
 
     # We don't use the repo's narrow matcher here because the patterns passed
     # to exchange.pull() could be different.
     narrowmatcher = narrowspec.match(
         repo.root,
         # Empty maps to nevermatcher. So always
         # set includes if missing.
         pullop.includepats or {b'path:.'},
         pullop.excludepats,
     )
 
     if pullop.includepats or pullop.excludepats:
         pathfilter = {}
         if pullop.includepats:
             pathfilter[b'include'] = sorted(pullop.includepats)
         if pullop.excludepats:
             pathfilter[b'exclude'] = sorted(pullop.excludepats)
     else:
         pathfilter = None
 
     # Figure out what needs to be fetched.
     common, fetch, remoteheads = _pullchangesetdiscovery(
         repo, remote, pullop.heads, abortwhenunrelated=pullop.force
     )
 
     # And fetch the data.
     pullheads = pullop.heads or remoteheads
     csetres = _fetchchangesets(repo, tr, remote, common, fetch, pullheads)
 
     # New revisions are written to the changelog. But all other updates
     # are deferred. Do those now.
 
     # Ensure all new changesets are draft by default. If the repo is
     # publishing, the phase will be adjusted by the loop below.
     if csetres[b'added']:
         phases.registernew(
             repo, tr, phases.draft, [repo[n].rev() for n in csetres[b'added']]
         )
 
     # And adjust the phase of all changesets accordingly.
     for phasenumber, phase in phases.phasenames.items():
         if phase == b'secret' or not csetres[b'nodesbyphase'][phase]:
             continue
 
         phases.advanceboundary(
             repo,
             tr,
             phasenumber,
             csetres[b'nodesbyphase'][phase],
         )
 
     # Write bookmark updates.
     bookmarks.updatefromremote(
         repo.ui,
         repo,
         csetres[b'bookmarks'],
         remote.url(),
         pullop.gettransaction,
         explicit=pullop.explicitbookmarks,
     )
 
     manres = _fetchmanifests(repo, tr, remote, csetres[b'manifestnodes'])
 
     # We don't properly support shallow changeset and manifest yet. So we apply
     # depth limiting locally.
     if pullop.depth:
         relevantcsetnodes = set()
         clnode = repo.changelog.node
 
         for rev in repo.revs(
             b'ancestors(%ln, %s)', pullheads, pullop.depth - 1
         ):
             relevantcsetnodes.add(clnode(rev))
 
         csetrelevantfilter = lambda n: n in relevantcsetnodes
 
     else:
         csetrelevantfilter = lambda n: True
 
     # If obtaining the raw store files, we need to scan the full repo to
     # derive all the changesets, manifests, and linkrevs.
     if usingrawchangelogandmanifest:
         csetsforfiles = []
         mnodesforfiles = []
         manifestlinkrevs = {}
 
         for rev in repo:
             ctx = repo[rev]
             node = ctx.node()
 
             if not csetrelevantfilter(node):
                 continue
 
             mnode = ctx.manifestnode()
 
             csetsforfiles.append(node)
             mnodesforfiles.append(mnode)
             manifestlinkrevs[mnode] = rev
 
     else:
         csetsforfiles = [n for n in csetres[b'added'] if csetrelevantfilter(n)]
         mnodesforfiles = manres[b'added']
         manifestlinkrevs = manres[b'linkrevs']
 
     # Find all file nodes referenced by added manifests and fetch those
     # revisions.
     fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
     _fetchfilesfromcsets(
         repo,
         tr,
         remote,
         pathfilter,
         fnodes,
         csetsforfiles,
         manifestlinkrevs,
         shallow=bool(pullop.depth),
     )
 
 
 def _checkuserawstorefiledata(pullop):
     """Check whether we should use rawstorefiledata command to retrieve data."""
 
     repo = pullop.repo
     remote = pullop.remote
 
     # Command to obtain raw store data isn't available.
     if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
         return False
 
     # Only honor if user requested stream clone operation.
     if not pullop.streamclonerequested:
         return False
 
     # Only works on empty repos.
     if len(repo):
         return False
 
     # TODO This is super hacky. There needs to be a storage API for this. We
     # also need to check for compatibility with the remote.
     if requirementsmod.REVLOGV1_REQUIREMENT not in repo.requirements:
         return False
 
     return True
 
 
 def _fetchrawstorefiles(repo, remote):
     with remote.commandexecutor() as e:
         objs = e.callcommand(
             b'rawstorefiledata',
             {
                 b'files': [b'changelog', b'manifestlog'],
             },
         ).result()
 
         # First object is a summary of files data that follows.
         overall = next(objs)
 
         progress = repo.ui.makeprogress(
             _(b'clone'), total=overall[b'totalsize'], unit=_(b'bytes')
         )
         with progress:
             progress.update(0)
 
             # Next are pairs of file metadata, data.
             while True:
                 try:
                     filemeta = next(objs)
                 except StopIteration:
                     break
 
                 for k in (b'location', b'path', b'size'):
                     if k not in filemeta:
                         raise error.Abort(
                             _(b'remote file data missing key: %s') % k
                         )
 
                 if filemeta[b'location'] == b'store':
                     vfs = repo.svfs
                 else:
                     raise error.Abort(
                         _(b'invalid location for raw file data: %s')
                         % filemeta[b'location']
                     )
 
                 bytesremaining = filemeta[b'size']
 
                 with vfs.open(filemeta[b'path'], b'wb') as fh:
                     while True:
                         try:
                             chunk = next(objs)
                         except StopIteration:
                             break
 
                         bytesremaining -= len(chunk)
 
                         if bytesremaining < 0:
                             raise error.Abort(
                                 _(
                                     b'received invalid number of bytes for file '
                                     b'data; expected %d, got extra'
                                 )
                                 % filemeta[b'size']
                             )
 
                         progress.increment(step=len(chunk))
                         fh.write(chunk)
 
                         try:
                             if chunk.islast:
                                 break
                         except AttributeError:
                             raise error.Abort(
                                 _(
                                     b'did not receive indefinite length bytestring '
                                     b'for file data'
                                 )
                             )
 
                 if bytesremaining:
                     raise error.Abort(
                         _(
                             b'received invalid number of bytes for'
                             b'file data; expected %d got %d'
                         )
                         % (
                             filemeta[b'size'],
                             filemeta[b'size'] - bytesremaining,
                         )
                     )
 
 
 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
     """Determine which changesets need to be pulled."""
 
     if heads:
         knownnode = repo.changelog.hasnode
         if all(knownnode(head) for head in heads):
             return heads, False, heads
 
     # TODO wire protocol version 2 is capable of more efficient discovery
     # than setdiscovery. Consider implementing something better.
     common, fetch, remoteheads = setdiscovery.findcommonheads(
         repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated
     )
 
     common = set(common)
     remoteheads = set(remoteheads)
 
     # If a remote head is filtered locally, put it back in the common set.
     # See the comment in exchange._pulldiscoverychangegroup() for more.
 
     if fetch and remoteheads:
         has_node = repo.unfiltered().changelog.index.has_node
 
         common |= {head for head in remoteheads if has_node(head)}
 
         if set(remoteheads).issubset(common):
             fetch = []
 
     common.discard(nullid)
 
     return common, fetch, remoteheads
 
 
 def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads):
     # TODO consider adding a step here where we obtain the DAG shape first
     # (or ask the server to slice changesets into chunks for us) so that
     # we can perform multiple fetches in batches. This will facilitate
     # resuming interrupted clones, higher server-side cache hit rates due
     # to smaller segments, etc.
     with remote.commandexecutor() as e:
         objs = e.callcommand(
             b'changesetdata',
             {
                 b'revisions': [
                     {
                         b'type': b'changesetdagrange',
                         b'roots': sorted(common),
                         b'heads': sorted(remoteheads),
                     }
                 ],
                 b'fields': {b'bookmarks', b'parents', b'phase', b'revision'},
             },
         ).result()
 
         # The context manager waits on all response data when exiting. So
         # we need to remain in the context manager in order to stream data.
         return _processchangesetdata(repo, tr, objs)
 
 
 def _processchangesetdata(repo, tr, objs):
     repo.hook(b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs))
 
     urepo = repo.unfiltered()
     cl = urepo.changelog
 
     cl.delayupdate(tr)
 
     # The first emitted object is a header describing the data that
     # follows.
     meta = next(objs)
 
     progress = repo.ui.makeprogress(
         _(b'changesets'), unit=_(b'chunks'), total=meta.get(b'totalitems')
     )
 
     manifestnodes = {}
     added = []
 
     def linkrev(node):
         repo.ui.debug(b'add changeset %s\n' % short(node))
         # Linkrev for changelog is always self.
         return len(cl)
 
     def ondupchangeset(cl, rev):
         added.append(cl.node(rev))
 
     def onchangeset(cl, rev):
         progress.increment()
 
         revision = cl.changelogrevision(rev)
         added.append(cl.node(rev))
 
         # We need to preserve the mapping of changelog revision to node
         # so we can set the linkrev accordingly when manifests are added.
         manifestnodes[rev] = revision.manifest
 
         repo.register_changeset(rev, revision)
 
     nodesbyphase = {phase: set() for phase in phases.phasenames.values()}
     remotebookmarks = {}
 
     # addgroup() expects a 7-tuple describing revisions. This normalizes
     # the wire data to that format.
     #
     # This loop also aggregates non-revision metadata, such as phase
     # data.
     def iterrevisions():
         for cset in objs:
             node = cset[b'node']
 
             if b'phase' in cset:
                 nodesbyphase[cset[b'phase']].add(node)
 
             for mark in cset.get(b'bookmarks', []):
                 remotebookmarks[mark] = node
 
             # TODO add mechanism for extensions to examine records so they
             # can siphon off custom data fields.
 
             extrafields = {}
 
             for field, size in cset.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             # Some entries might only be metadata only updates.
             if b'revision' not in extrafields:
                 continue
 
             data = extrafields[b'revision']
 
             yield (
                 node,
                 cset[b'parents'][0],
                 cset[b'parents'][1],
                 # Linknode is always itself for changesets.
                 cset[b'node'],
                 # We always send full revisions. So delta base is not set.
                 nullid,
                 mdiff.trivialdiffheader(len(data)) + data,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported
+                {},
             )
 
     cl.addgroup(
         iterrevisions(),
         linkrev,
         weakref.proxy(tr),
         alwayscache=True,
         addrevisioncb=onchangeset,
         duplicaterevisioncb=ondupchangeset,
     )
 
     progress.complete()
 
     return {
         b'added': added,
         b'nodesbyphase': nodesbyphase,
         b'bookmarks': remotebookmarks,
         b'manifestnodes': manifestnodes,
     }
 
 
 def _fetchmanifests(repo, tr, remote, manifestnodes):
     rootmanifest = repo.manifestlog.getstorage(b'')
 
     # Some manifests can be shared between changesets. Filter out revisions
     # we already know about.
     fetchnodes = []
     linkrevs = {}
     seen = set()
 
     for clrev, node in sorted(pycompat.iteritems(manifestnodes)):
         if node in seen:
             continue
 
         try:
             rootmanifest.rev(node)
         except error.LookupError:
             fetchnodes.append(node)
             linkrevs[node] = clrev
 
         seen.add(node)
 
     # TODO handle tree manifests
 
     # addgroup() expects 7-tuple describing revisions. This normalizes
     # the wire data to that format.
     def iterrevisions(objs, progress):
         for manifest in objs:
             node = manifest[b'node']
 
             extrafields = {}
 
             for field, size in manifest.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = manifest[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             yield (
                 node,
                 manifest[b'parents'][0],
                 manifest[b'parents'][1],
                 # The value passed in is passed to the lookup function passed
                 # to addgroup(). We already have a map of manifest node to
                 # changelog revision number. So we just pass in the
                 # manifest node here and use linkrevs.__getitem__ as the
                 # resolution function.
                 node,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
 
     progress = repo.ui.makeprogress(
         _(b'manifests'), unit=_(b'chunks'), total=len(fetchnodes)
     )
 
     commandmeta = remote.apidescriptor[b'commands'][b'manifestdata']
     batchsize = commandmeta.get(b'recommendedbatchsize', 10000)
     # TODO make size configurable on client?
 
     # We send commands 1 at a time to the remote. This is not the most
     # efficient because we incur a round trip at the end of each batch.
     # However, the existing frame-based reactor keeps consuming server
     # data in the background. And this results in response data buffering
     # in memory. This can consume gigabytes of memory.
     # TODO send multiple commands in a request once background buffering
     # issues are resolved.
 
     added = []
 
     for i in pycompat.xrange(0, len(fetchnodes), batchsize):
         batch = [node for node in fetchnodes[i : i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             objs = e.callcommand(
                 b'manifestdata',
                 {
                     b'tree': b'',
                     b'nodes': batch,
                     b'fields': {b'parents', b'revision'},
                     b'haveparents': True,
                 },
             ).result()
 
             # Chomp off header object.
             next(objs)
 
             def onchangeset(cl, rev):
                 added.append(cl.node(rev))
 
             rootmanifest.addgroup(
                 iterrevisions(objs, progress),
                 linkrevs.__getitem__,
                 weakref.proxy(tr),
                 addrevisioncb=onchangeset,
                 duplicaterevisioncb=onchangeset,
             )
 
     progress.complete()
 
     return {
         b'added': added,
         b'linkrevs': linkrevs,
     }
 
 
 def _derivefilesfrommanifests(repo, matcher, manifestnodes):
     """Determine what file nodes are relevant given a set of manifest nodes.
 
     Returns a dict mapping file paths to dicts of file node to first manifest
     node.
     """
     ml = repo.manifestlog
     fnodes = collections.defaultdict(dict)
 
     progress = repo.ui.makeprogress(
         _(b'scanning manifests'), total=len(manifestnodes)
     )
 
     with progress:
         for manifestnode in manifestnodes:
             m = ml.get(b'', manifestnode)
 
             # TODO this will pull in unwanted nodes because it takes the storage
             # delta into consideration. What we really want is something that
             # takes the delta between the manifest's parents. And ideally we
             # would ignore file nodes that are known locally. For now, ignore
             # both these limitations. This will result in incremental fetches
             # requesting data we already have. So this is far from ideal.
             md = m.readfast()
 
             for path, fnode in md.items():
                 if matcher(path):
                     fnodes[path].setdefault(fnode, manifestnode)
 
             progress.increment()
 
     return fnodes
 
 
 def _fetchfiles(repo, tr, remote, fnodes, linkrevs):
     """Fetch file data from explicit file revisions."""
 
     def iterrevisions(objs, progress):
         for filerevision in objs:
             node = filerevision[b'node']
 
             extrafields = {}
 
             for field, size in filerevision.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = filerevision[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             yield (
                 node,
                 filerevision[b'parents'][0],
                 filerevision[b'parents'][1],
                 node,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
 
     progress = repo.ui.makeprogress(
         _(b'files'),
         unit=_(b'chunks'),
         total=sum(len(v) for v in pycompat.itervalues(fnodes)),
     )
 
     # TODO make batch size configurable
     batchsize = 10000
     fnodeslist = [x for x in sorted(fnodes.items())]
 
     for i in pycompat.xrange(0, len(fnodeslist), batchsize):
         batch = [x for x in fnodeslist[i : i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             fs = []
             locallinkrevs = {}
 
             for path, nodes in batch:
                 fs.append(
                     (
                         path,
                         e.callcommand(
                             b'filedata',
                             {
                                 b'path': path,
                                 b'nodes': sorted(nodes),
                                 b'fields': {b'parents', b'revision'},
                                 b'haveparents': True,
                             },
                         ),
                     )
                 )
 
                 locallinkrevs[path] = {
                     node: linkrevs[manifestnode]
                     for node, manifestnode in pycompat.iteritems(nodes)
                 }
 
             for path, f in fs:
                 objs = f.result()
 
                 # Chomp off header objects.
                 next(objs)
 
                 store = repo.file(path)
                 store.addgroup(
                     iterrevisions(objs, progress),
                     locallinkrevs[path].__getitem__,
                     weakref.proxy(tr),
                 )
 
 
 def _fetchfilesfromcsets(
     repo, tr, remote, pathfilter, fnodes, csets, manlinkrevs, shallow=False
 ):
     """Fetch file data from explicit changeset revisions."""
 
     def iterrevisions(objs, remaining, progress):
         while remaining:
             filerevision = next(objs)
 
             node = filerevision[b'node']
 
             extrafields = {}
 
             for field, size in filerevision.get(b'fieldsfollowing', []):
                 extrafields[field] = next(objs)
 
             if b'delta' in extrafields:
                 basenode = filerevision[b'deltabasenode']
                 delta = extrafields[b'delta']
             elif b'revision' in extrafields:
                 basenode = nullid
                 revision = extrafields[b'revision']
                 delta = mdiff.trivialdiffheader(len(revision)) + revision
             else:
                 continue
 
             if b'linknode' in filerevision:
                 linknode = filerevision[b'linknode']
             else:
                 linknode = node
 
             yield (
                 node,
                 filerevision[b'parents'][0],
                 filerevision[b'parents'][1],
                 linknode,
                 basenode,
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
             remaining -= 1
 
     progress = repo.ui.makeprogress(
         _(b'files'),
         unit=_(b'chunks'),
         total=sum(len(v) for v in pycompat.itervalues(fnodes)),
     )
 
     commandmeta = remote.apidescriptor[b'commands'][b'filesdata']
     batchsize = commandmeta.get(b'recommendedbatchsize', 50000)
 
     shallowfiles = repository.REPO_FEATURE_SHALLOW_FILE_STORAGE in repo.features
     fields = {b'parents', b'revision'}
     clrev = repo.changelog.rev
 
     # There are no guarantees that we'll have ancestor revisions if
     # a) this repo has shallow file storage b) shallow data fetching is enabled.
     # Force remote to not delta against possibly unknown revisions when these
     # conditions hold.
     haveparents = not (shallowfiles or shallow)
 
     # Similarly, we may not have calculated linkrevs for all incoming file
     # revisions. Ask the remote to do work for us in this case.
     if not haveparents:
         fields.add(b'linknode')
 
     for i in pycompat.xrange(0, len(csets), batchsize):
         batch = [x for x in csets[i : i + batchsize]]
         if not batch:
             continue
 
         with remote.commandexecutor() as e:
             args = {
                 b'revisions': [
                     {
                         b'type': b'changesetexplicit',
                         b'nodes': batch,
                     }
                 ],
                 b'fields': fields,
                 b'haveparents': haveparents,
             }
 
             if pathfilter:
                 args[b'pathfilter'] = pathfilter
 
             objs = e.callcommand(b'filesdata', args).result()
 
             # First object is an overall header.
             overall = next(objs)
 
             # We have overall['totalpaths'] segments.
             for i in pycompat.xrange(overall[b'totalpaths']):
                 header = next(objs)
 
                 path = header[b'path']
                 store = repo.file(path)
 
                 linkrevs = {
                     fnode: manlinkrevs[mnode]
                     for fnode, mnode in pycompat.iteritems(fnodes[path])
                 }
 
                 def getlinkrev(node):
                     if node in linkrevs:
                         return linkrevs[node]
                     else:
                         return clrev(node)
 
                 store.addgroup(
                     iterrevisions(objs, header[b'totalitems'], progress),
                     getlinkrev,
                     weakref.proxy(tr),
                     maybemissingparents=shallow,
                 )

mercurial/localrepo.py

@@ -1,3702 +1,3709 b''
 # localrepo.py - read/write repository class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import functools
 import os
 import random
 import sys
 import time
 import weakref
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
     nullrev,
     short,
 )
 from .pycompat import (
     delattr,
     getattr,
 )
 from . import (
     bookmarks,
     branchmap,
     bundle2,
     bundlecaches,
     changegroup,
     color,
     commit,
     context,
     dirstate,
     dirstateguard,
     discovery,
     encoding,
     error,
     exchange,
     extensions,
     filelog,
     hook,
     lock as lockmod,
     match as matchmod,
     mergestate as mergestatemod,
     mergeutil,
     namespaces,
     narrowspec,
     obsolete,
     pathutil,
     phases,
     pushkey,
     pycompat,
     rcutil,
     repoview,
     requirements as requirementsmod,
     revlog,
     revset,
     revsetlang,
     scmutil,
     sparse,
     store as storemod,
     subrepoutil,
     tags as tagsmod,
     transaction,
     txnutil,
     util,
     vfs as vfsmod,
 )
 
 from .interfaces import (
     repository,
     util as interfaceutil,
 )
 
 from .utils import (
     hashutil,
     procutil,
     stringutil,
 )
 
 from .revlogutils import (
     concurrency_checker as revlogchecker,
     constants as revlogconst,
 )
 
 release = lockmod.release
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # set of (path, vfs-location) tuples. vfs-location is:
 # - 'plain for vfs relative paths
 # - '' for svfs relative paths
 _cachedfiles = set()
 
 
 class _basefilecache(scmutil.filecache):
     """All filecache usage on repo are done for logic that should be unfiltered"""
 
     def __get__(self, repo, type=None):
         if repo is None:
             return self
         # proxy to unfiltered __dict__ since filtered repo has no entry
         unfi = repo.unfiltered()
         try:
             return unfi.__dict__[self.sname]
         except KeyError:
             pass
         return super(_basefilecache, self).__get__(unfi, type)
 
     def set(self, repo, value):
         return super(_basefilecache, self).set(repo.unfiltered(), value)
 
 
 class repofilecache(_basefilecache):
     """filecache for files in .hg but outside of .hg/store"""
 
     def __init__(self, *paths):
         super(repofilecache, self).__init__(*paths)
         for path in paths:
             _cachedfiles.add((path, b'plain'))
 
     def join(self, obj, fname):
         return obj.vfs.join(fname)
 
 
 class storecache(_basefilecache):
     """filecache for files in the store"""
 
     def __init__(self, *paths):
         super(storecache, self).__init__(*paths)
         for path in paths:
             _cachedfiles.add((path, b''))
 
     def join(self, obj, fname):
         return obj.sjoin(fname)
 
 
 class mixedrepostorecache(_basefilecache):
     """filecache for a mix files in .hg/store and outside"""
 
     def __init__(self, *pathsandlocations):
         # scmutil.filecache only uses the path for passing back into our
         # join(), so we can safely pass a list of paths and locations
         super(mixedrepostorecache, self).__init__(*pathsandlocations)
         _cachedfiles.update(pathsandlocations)
 
     def join(self, obj, fnameandlocation):
         fname, location = fnameandlocation
         if location == b'plain':
             return obj.vfs.join(fname)
         else:
             if location != b'':
                 raise error.ProgrammingError(
                     b'unexpected location: %s' % location
                 )
             return obj.sjoin(fname)
 
 
 def isfilecached(repo, name):
     """check if a repo has already cached "name" filecache-ed property
 
     This returns (cachedobj-or-None, iscached) tuple.
     """
     cacheentry = repo.unfiltered()._filecache.get(name, None)
     if not cacheentry:
         return None, False
     return cacheentry.obj, True
 
 
 class unfilteredpropertycache(util.propertycache):
     """propertycache that apply to unfiltered repo only"""
 
     def __get__(self, repo, type=None):
         unfi = repo.unfiltered()
         if unfi is repo:
             return super(unfilteredpropertycache, self).__get__(unfi)
         return getattr(unfi, self.name)
 
 
 class filteredpropertycache(util.propertycache):
     """propertycache that must take filtering in account"""
 
     def cachevalue(self, obj, value):
         object.__setattr__(obj, self.name, value)
 
 
 def hasunfilteredcache(repo, name):
     """check if a repo has an unfilteredpropertycache value for <name>"""
     return name in vars(repo.unfiltered())
 
 
 def unfilteredmethod(orig):
     """decorate method that always need to be run on unfiltered version"""
 
     @functools.wraps(orig)
     def wrapper(repo, *args, **kwargs):
         return orig(repo.unfiltered(), *args, **kwargs)
 
     return wrapper
 
 
 moderncaps = {
     b'lookup',
     b'branchmap',
     b'pushkey',
     b'known',
     b'getbundle',
     b'unbundle',
 }
 legacycaps = moderncaps.union({b'changegroupsubset'})
 
 
 @interfaceutil.implementer(repository.ipeercommandexecutor)
 class localcommandexecutor(object):
     def __init__(self, peer):
         self._peer = peer
         self._sent = False
         self._closed = False
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, exctb):
         self.close()
 
     def callcommand(self, command, args):
         if self._sent:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after sendcommands()'
             )
 
         if self._closed:
             raise error.ProgrammingError(
                 b'callcommand() cannot be used after close()'
             )
 
         # We don't need to support anything fancy. Just call the named
         # method on the peer and return a resolved future.
         fn = getattr(self._peer, pycompat.sysstr(command))
 
         f = pycompat.futures.Future()
 
         try:
             result = fn(**pycompat.strkwargs(args))
         except Exception:
             pycompat.future_set_exception_info(f, sys.exc_info()[1:])
         else:
             f.set_result(result)
 
         return f
 
     def sendcommands(self):
         self._sent = True
 
     def close(self):
         self._closed = True
 
 
 @interfaceutil.implementer(repository.ipeercommands)
 class localpeer(repository.peer):
     '''peer for a local repo; reflects only the most recent API'''
 
     def __init__(self, repo, caps=None):
         super(localpeer, self).__init__()
 
         if caps is None:
             caps = moderncaps.copy()
         self._repo = repo.filtered(b'served')
         self.ui = repo.ui
         self._caps = repo._restrictcapabilities(caps)
 
     # Begin of _basepeer interface.
 
     def url(self):
         return self._repo.url()
 
     def local(self):
         return self._repo
 
     def peer(self):
         return self
 
     def canpush(self):
         return True
 
     def close(self):
         self._repo.close()
 
     # End of _basepeer interface.
 
     # Begin of _basewirecommands interface.
 
     def branchmap(self):
         return self._repo.branchmap()
 
     def capabilities(self):
         return self._caps
 
     def clonebundles(self):
         return self._repo.tryread(bundlecaches.CB_MANIFEST_FILE)
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         """Used to test argument passing over the wire"""
         return b"%s %s %s %s %s" % (
             one,
             two,
             pycompat.bytestr(three),
             pycompat.bytestr(four),
             pycompat.bytestr(five),
         )
 
     def getbundle(
-        self, source, heads=None, common=None, bundlecaps=None, **kwargs
+        self,
+        source,
+        heads=None,
+        common=None,
+        bundlecaps=None,
+        remote_sidedata=None,
+        **kwargs
     ):
         chunks = exchange.getbundlechunks(
             self._repo,
             source,
             heads=heads,
             common=common,
             bundlecaps=bundlecaps,
+            remote_sidedata=remote_sidedata,
             **kwargs
         )[1]
         cb = util.chunkbuffer(chunks)
 
         if exchange.bundle2requested(bundlecaps):
             # When requesting a bundle2, getbundle returns a stream to make the
             # wire level function happier. We need to build a proper object
             # from it in local peer.
             return bundle2.getunbundler(self.ui, cb)
         else:
             return changegroup.getunbundler(b'01', cb, None)
 
     def heads(self):
         return self._repo.heads()
 
     def known(self, nodes):
         return self._repo.known(nodes)
 
     def listkeys(self, namespace):
         return self._repo.listkeys(namespace)
 
     def lookup(self, key):
         return self._repo.lookup(key)
 
     def pushkey(self, namespace, key, old, new):
         return self._repo.pushkey(namespace, key, old, new)
 
     def stream_out(self):
         raise error.Abort(_(b'cannot perform stream clone against local peer'))
 
     def unbundle(self, bundle, heads, url):
         """apply a bundle on a repo
 
         This function handles the repo locking itself."""
         try:
             try:
                 bundle = exchange.readbundle(self.ui, bundle, None)
                 ret = exchange.unbundle(self._repo, bundle, heads, b'push', url)
                 if util.safehasattr(ret, b'getchunks'):
                     # This is a bundle20 object, turn it into an unbundler.
                     # This little dance should be dropped eventually when the
                     # API is finally improved.
                     stream = util.chunkbuffer(ret.getchunks())
                     ret = bundle2.getunbundler(self.ui, stream)
                 return ret
             except Exception as exc:
                 # If the exception contains output salvaged from a bundle2
                 # reply, we need to make sure it is printed before continuing
                 # to fail. So we build a bundle2 with such output and consume
                 # it directly.
                 #
                 # This is not very elegant but allows a "simple" solution for
                 # issue4594
                 output = getattr(exc, '_bundle2salvagedoutput', ())
                 if output:
                     bundler = bundle2.bundle20(self._repo.ui)
                     for out in output:
                         bundler.addpart(out)
                     stream = util.chunkbuffer(bundler.getchunks())
                     b = bundle2.getunbundler(self.ui, stream)
                     bundle2.processbundle(self._repo, b)
                 raise
         except error.PushRaced as exc:
             raise error.ResponseError(
                 _(b'push failed:'), stringutil.forcebytestr(exc)
             )
 
     # End of _basewirecommands interface.
 
     # Begin of peer interface.
 
     def commandexecutor(self):
         return localcommandexecutor(self)
 
     # End of peer interface.
 
 
 @interfaceutil.implementer(repository.ipeerlegacycommands)
 class locallegacypeer(localpeer):
     """peer extension which implements legacy methods too; used for tests with
     restricted capabilities"""
 
     def __init__(self, repo):
         super(locallegacypeer, self).__init__(repo, caps=legacycaps)
 
     # Begin of baselegacywirecommands interface.
 
     def between(self, pairs):
         return self._repo.between(pairs)
 
     def branches(self, nodes):
         return self._repo.branches(nodes)
 
     def changegroup(self, nodes, source):
         outgoing = discovery.outgoing(
             self._repo, missingroots=nodes, ancestorsof=self._repo.heads()
         )
         return changegroup.makechangegroup(self._repo, outgoing, b'01', source)
 
     def changegroupsubset(self, bases, heads, source):
         outgoing = discovery.outgoing(
             self._repo, missingroots=bases, ancestorsof=heads
         )
         return changegroup.makechangegroup(self._repo, outgoing, b'01', source)
 
     # End of baselegacywirecommands interface.
 
 
 # Functions receiving (ui, features) that extensions can register to impact
 # the ability to load repositories with custom requirements. Only
 # functions defined in loaded extensions are called.
 #
 # The function receives a set of requirement strings that the repository
 # is capable of opening. Functions will typically add elements to the
 # set to reflect that the extension knows how to handle that requirements.
 featuresetupfuncs = set()
 
 
 def _getsharedvfs(hgvfs, requirements):
     """returns the vfs object pointing to root of shared source
     repo for a shared repository
 
     hgvfs is vfs pointing at .hg/ of current repo (shared one)
     requirements is a set of requirements of current repo (shared one)
     """
     # The ``shared`` or ``relshared`` requirements indicate the
     # store lives in the path contained in the ``.hg/sharedpath`` file.
     # This is an absolute path for ``shared`` and relative to
     # ``.hg/`` for ``relshared``.
     sharedpath = hgvfs.read(b'sharedpath').rstrip(b'\n')
     if requirementsmod.RELATIVE_SHARED_REQUIREMENT in requirements:
         sharedpath = hgvfs.join(sharedpath)
 
     sharedvfs = vfsmod.vfs(sharedpath, realpath=True)
 
     if not sharedvfs.exists():
         raise error.RepoError(
             _(b'.hg/sharedpath points to nonexistent directory %s')
             % sharedvfs.base
         )
     return sharedvfs
 
 
 def _readrequires(vfs, allowmissing):
     """reads the require file present at root of this vfs
     and return a set of requirements
 
     If allowmissing is True, we suppress ENOENT if raised"""
     # requires file contains a newline-delimited list of
     # features/capabilities the opener (us) must have in order to use
     # the repository. This file was introduced in Mercurial 0.9.2,
     # which means very old repositories may not have one. We assume
     # a missing file translates to no requirements.
     try:
         requirements = set(vfs.read(b'requires').splitlines())
     except IOError as e:
         if not (allowmissing and e.errno == errno.ENOENT):
             raise
         requirements = set()
     return requirements
 
 
 def makelocalrepository(baseui, path, intents=None):
     """Create a local repository object.
 
     Given arguments needed to construct a local repository, this function
     performs various early repository loading functionality (such as
     reading the ``.hg/requires`` and ``.hg/hgrc`` files), validates that
     the repository can be opened, derives a type suitable for representing
     that repository, and returns an instance of it.
 
     The returned object conforms to the ``repository.completelocalrepository``
     interface.
 
     The repository type is derived by calling a series of factory functions
     for each aspect/interface of the final repository. These are defined by
     ``REPO_INTERFACES``.
 
     Each factory function is called to produce a type implementing a specific
     interface. The cumulative list of returned types will be combined into a
     new type and that type will be instantiated to represent the local
     repository.
 
     The factory functions each receive various state that may be consulted
     as part of deriving a type.
 
     Extensions should wrap these factory functions to customize repository type
     creation. Note that an extension's wrapped function may be called even if
     that extension is not loaded for the repo being constructed. Extensions
     should check if their ``__name__`` appears in the
     ``extensionmodulenames`` set passed to the factory function and no-op if
     not.
     """
     ui = baseui.copy()
     # Prevent copying repo configuration.
     ui.copy = baseui.copy
 
     # Working directory VFS rooted at repository root.
     wdirvfs = vfsmod.vfs(path, expandpath=True, realpath=True)
 
     # Main VFS for .hg/ directory.
     hgpath = wdirvfs.join(b'.hg')
     hgvfs = vfsmod.vfs(hgpath, cacheaudited=True)
     # Whether this repository is shared one or not
     shared = False
     # If this repository is shared, vfs pointing to shared repo
     sharedvfs = None
 
     # The .hg/ path should exist and should be a directory. All other
     # cases are errors.
     if not hgvfs.isdir():
         try:
             hgvfs.stat()
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
         except ValueError as e:
             # Can be raised on Python 3.8 when path is invalid.
             raise error.Abort(
                 _(b'invalid path %s: %s') % (path, stringutil.forcebytestr(e))
             )
 
         raise error.RepoError(_(b'repository %s not found') % path)
 
     requirements = _readrequires(hgvfs, True)
     shared = (
         requirementsmod.SHARED_REQUIREMENT in requirements
         or requirementsmod.RELATIVE_SHARED_REQUIREMENT in requirements
     )
     storevfs = None
     if shared:
         # This is a shared repo
         sharedvfs = _getsharedvfs(hgvfs, requirements)
         storevfs = vfsmod.vfs(sharedvfs.join(b'store'))
     else:
         storevfs = vfsmod.vfs(hgvfs.join(b'store'))
 
     # if .hg/requires contains the sharesafe requirement, it means
     # there exists a `.hg/store/requires` too and we should read it
     # NOTE: presence of SHARESAFE_REQUIREMENT imply that store requirement
     # is present. We never write SHARESAFE_REQUIREMENT for a repo if store
     # is not present, refer checkrequirementscompat() for that
     #
     # However, if SHARESAFE_REQUIREMENT is not present, it means that the
     # repository was shared the old way. We check the share source .hg/requires
     # for SHARESAFE_REQUIREMENT to detect whether the current repository needs
     # to be reshared
     hint = _(b"see `hg help config.format.use-share-safe` for more information")
     if requirementsmod.SHARESAFE_REQUIREMENT in requirements:
 
         if (
             shared
             and requirementsmod.SHARESAFE_REQUIREMENT
             not in _readrequires(sharedvfs, True)
         ):
             mismatch_warn = ui.configbool(
                 b'share', b'safe-mismatch.source-not-safe.warn'
             )
             mismatch_config = ui.config(
                 b'share', b'safe-mismatch.source-not-safe'
             )
             if mismatch_config in (
                 b'downgrade-allow',
                 b'allow',
                 b'downgrade-abort',
             ):
                 # prevent cyclic import localrepo -> upgrade -> localrepo
                 from . import upgrade
 
                 upgrade.downgrade_share_to_non_safe(
                     ui,
                     hgvfs,
                     sharedvfs,
                     requirements,
                     mismatch_config,
                     mismatch_warn,
                 )
             elif mismatch_config == b'abort':
                 raise error.Abort(
                     _(b"share source does not support share-safe requirement"),
                     hint=hint,
                 )
             else:
                 raise error.Abort(
                     _(
                         b"share-safe mismatch with source.\nUnrecognized"
                         b" value '%s' of `share.safe-mismatch.source-not-safe`"
                         b" set."
                     )
                     % mismatch_config,
                     hint=hint,
                 )
         else:
             requirements |= _readrequires(storevfs, False)
     elif shared:
         sourcerequires = _readrequires(sharedvfs, False)
         if requirementsmod.SHARESAFE_REQUIREMENT in sourcerequires:
             mismatch_config = ui.config(b'share', b'safe-mismatch.source-safe')
             mismatch_warn = ui.configbool(
                 b'share', b'safe-mismatch.source-safe.warn'
             )
             if mismatch_config in (
                 b'upgrade-allow',
                 b'allow',
                 b'upgrade-abort',
             ):
                 # prevent cyclic import localrepo -> upgrade -> localrepo
                 from . import upgrade
 
                 upgrade.upgrade_share_to_safe(
                     ui,
                     hgvfs,
                     storevfs,
                     requirements,
                     mismatch_config,
                     mismatch_warn,
                 )
             elif mismatch_config == b'abort':
                 raise error.Abort(
                     _(
                         b'version mismatch: source uses share-safe'
                         b' functionality while the current share does not'
                     ),
                     hint=hint,
                 )
             else:
                 raise error.Abort(
                     _(
                         b"share-safe mismatch with source.\nUnrecognized"
                         b" value '%s' of `share.safe-mismatch.source-safe` set."
                     )
                     % mismatch_config,
                     hint=hint,
                 )
 
     # The .hg/hgrc file may load extensions or contain config options
     # that influence repository construction. Attempt to load it and
     # process any new extensions that it may have pulled in.
     if loadhgrc(ui, wdirvfs, hgvfs, requirements, sharedvfs):
         afterhgrcload(ui, wdirvfs, hgvfs, requirements)
         extensions.loadall(ui)
         extensions.populateui(ui)
 
     # Set of module names of extensions loaded for this repository.
     extensionmodulenames = {m.__name__ for n, m in extensions.extensions(ui)}
 
     supportedrequirements = gathersupportedrequirements(ui)
 
     # We first validate the requirements are known.
     ensurerequirementsrecognized(requirements, supportedrequirements)
 
     # Then we validate that the known set is reasonable to use together.
     ensurerequirementscompatible(ui, requirements)
 
     # TODO there are unhandled edge cases related to opening repositories with
     # shared storage. If storage is shared, we should also test for requirements
     # compatibility in the pointed-to repo. This entails loading the .hg/hgrc in
     # that repo, as that repo may load extensions needed to open it. This is a
     # bit complicated because we don't want the other hgrc to overwrite settings
     # in this hgrc.
     #
     # This bug is somewhat mitigated by the fact that we copy the .hg/requires
     # file when sharing repos. But if a requirement is added after the share is
     # performed, thereby introducing a new requirement for the opener, we may
     # will not see that and could encounter a run-time error interacting with
     # that shared store since it has an unknown-to-us requirement.
 
     # At this point, we know we should be capable of opening the repository.
     # Now get on with doing that.
 
     features = set()
 
     # The "store" part of the repository holds versioned data. How it is
     # accessed is determined by various requirements. If `shared` or
     # `relshared` requirements are present, this indicates current repository
     # is a share and store exists in path mentioned in `.hg/sharedpath`
     if shared:
         storebasepath = sharedvfs.base
         cachepath = sharedvfs.join(b'cache')
         features.add(repository.REPO_FEATURE_SHARED_STORAGE)
     else:
         storebasepath = hgvfs.base
         cachepath = hgvfs.join(b'cache')
     wcachepath = hgvfs.join(b'wcache')
 
     # The store has changed over time and the exact layout is dictated by
     # requirements. The store interface abstracts differences across all
     # of them.
     store = makestore(
         requirements,
         storebasepath,
         lambda base: vfsmod.vfs(base, cacheaudited=True),
     )
     hgvfs.createmode = store.createmode
 
     storevfs = store.vfs
     storevfs.options = resolvestorevfsoptions(ui, requirements, features)
 
     # The cache vfs is used to manage cache files.
     cachevfs = vfsmod.vfs(cachepath, cacheaudited=True)
     cachevfs.createmode = store.createmode
     # The cache vfs is used to manage cache files related to the working copy
     wcachevfs = vfsmod.vfs(wcachepath, cacheaudited=True)
     wcachevfs.createmode = store.createmode
 
     # Now resolve the type for the repository object. We do this by repeatedly
     # calling a factory function to produces types for specific aspects of the
     # repo's operation. The aggregate returned types are used as base classes
     # for a dynamically-derived type, which will represent our new repository.
 
     bases = []
     extrastate = {}
 
     for iface, fn in REPO_INTERFACES:
         # We pass all potentially useful state to give extensions tons of
         # flexibility.
         typ = fn()(
             ui=ui,
             intents=intents,
             requirements=requirements,
             features=features,
             wdirvfs=wdirvfs,
             hgvfs=hgvfs,
             store=store,
             storevfs=storevfs,
             storeoptions=storevfs.options,
             cachevfs=cachevfs,
             wcachevfs=wcachevfs,
             extensionmodulenames=extensionmodulenames,
             extrastate=extrastate,
             baseclasses=bases,
         )
 
         if not isinstance(typ, type):
             raise error.ProgrammingError(
                 b'unable to construct type for %s' % iface
             )
 
         bases.append(typ)
 
     # type() allows you to use characters in type names that wouldn't be
     # recognized as Python symbols in source code. We abuse that to add
     # rich information about our constructed repo.
     name = pycompat.sysstr(
         b'derivedrepo:%s<%s>' % (wdirvfs.base, b','.join(sorted(requirements)))
     )
 
     cls = type(name, tuple(bases), {})
 
     return cls(
         baseui=baseui,
         ui=ui,
         origroot=path,
         wdirvfs=wdirvfs,
         hgvfs=hgvfs,
         requirements=requirements,
         supportedrequirements=supportedrequirements,
         sharedpath=storebasepath,
         store=store,
         cachevfs=cachevfs,
         wcachevfs=wcachevfs,
         features=features,
         intents=intents,
     )
 
 
 def loadhgrc(ui, wdirvfs, hgvfs, requirements, sharedvfs=None):
     """Load hgrc files/content into a ui instance.
 
     This is called during repository opening to load any additional
     config files or settings relevant to the current repository.
 
     Returns a bool indicating whether any additional configs were loaded.
 
     Extensions should monkeypatch this function to modify how per-repo
     configs are loaded. For example, an extension may wish to pull in
     configs from alternate files or sources.
 
     sharedvfs is vfs object pointing to source repo if the current one is a
     shared one
     """
     if not rcutil.use_repo_hgrc():
         return False
 
     ret = False
     # first load config from shared source if we has to
     if requirementsmod.SHARESAFE_REQUIREMENT in requirements and sharedvfs:
         try:
             ui.readconfig(sharedvfs.join(b'hgrc'), root=sharedvfs.base)
             ret = True
         except IOError:
             pass
 
     try:
         ui.readconfig(hgvfs.join(b'hgrc'), root=wdirvfs.base)
         ret = True
     except IOError:
         pass
 
     try:
         ui.readconfig(hgvfs.join(b'hgrc-not-shared'), root=wdirvfs.base)
         ret = True
     except IOError:
         pass
 
     return ret
 
 
 def afterhgrcload(ui, wdirvfs, hgvfs, requirements):
     """Perform additional actions after .hg/hgrc is loaded.
 
     This function is called during repository loading immediately after
     the .hg/hgrc file is loaded and before per-repo extensions are loaded.
 
     The function can be used to validate configs, automatically add
     options (including extensions) based on requirements, etc.
     """
 
     # Map of requirements to list of extensions to load automatically when
     # requirement is present.
     autoextensions = {
         b'git': [b'git'],
         b'largefiles': [b'largefiles'],
         b'lfs': [b'lfs'],
     }
 
     for requirement, names in sorted(autoextensions.items()):
         if requirement not in requirements:
             continue
 
         for name in names:
             if not ui.hasconfig(b'extensions', name):
                 ui.setconfig(b'extensions', name, b'', source=b'autoload')
 
 
 def gathersupportedrequirements(ui):
     """Determine the complete set of recognized requirements."""
     # Start with all requirements supported by this file.
     supported = set(localrepository._basesupported)
 
     # Execute ``featuresetupfuncs`` entries if they belong to an extension
     # relevant to this ui instance.
     modules = {m.__name__ for n, m in extensions.extensions(ui)}
 
     for fn in featuresetupfuncs:
         if fn.__module__ in modules:
             fn(ui, supported)
 
     # Add derived requirements from registered compression engines.
     for name in util.compengines:
         engine = util.compengines[name]
         if engine.available() and engine.revlogheader():
             supported.add(b'exp-compression-%s' % name)
             if engine.name() == b'zstd':
                 supported.add(b'revlog-compression-zstd')
 
     return supported
 
 
 def ensurerequirementsrecognized(requirements, supported):
     """Validate that a set of local requirements is recognized.
 
     Receives a set of requirements. Raises an ``error.RepoError`` if there
     exists any requirement in that set that currently loaded code doesn't
     recognize.
 
     Returns a set of supported requirements.
     """
     missing = set()
 
     for requirement in requirements:
         if requirement in supported:
             continue
 
         if not requirement or not requirement[0:1].isalnum():
             raise error.RequirementError(_(b'.hg/requires file is corrupt'))
 
         missing.add(requirement)
 
     if missing:
         raise error.RequirementError(
             _(b'repository requires features unknown to this Mercurial: %s')
             % b' '.join(sorted(missing)),
             hint=_(
                 b'see https://mercurial-scm.org/wiki/MissingRequirement '
                 b'for more information'
             ),
         )
 
 
 def ensurerequirementscompatible(ui, requirements):
     """Validates that a set of recognized requirements is mutually compatible.
 
     Some requirements may not be compatible with others or require
     config options that aren't enabled. This function is called during
     repository opening to ensure that the set of requirements needed
     to open a repository is sane and compatible with config options.
 
     Extensions can monkeypatch this function to perform additional
     checking.
 
     ``error.RepoError`` should be raised on failure.
     """
     if (
         requirementsmod.SPARSE_REQUIREMENT in requirements
         and not sparse.enabled
     ):
         raise error.RepoError(
             _(
                 b'repository is using sparse feature but '
                 b'sparse is not enabled; enable the '
                 b'"sparse" extensions to access'
             )
         )
 
 
 def makestore(requirements, path, vfstype):
     """Construct a storage object for a repository."""
     if requirementsmod.STORE_REQUIREMENT in requirements:
         if requirementsmod.FNCACHE_REQUIREMENT in requirements:
             dotencode = requirementsmod.DOTENCODE_REQUIREMENT in requirements
             return storemod.fncachestore(path, vfstype, dotencode)
 
         return storemod.encodedstore(path, vfstype)
 
     return storemod.basicstore(path, vfstype)
 
 
 def resolvestorevfsoptions(ui, requirements, features):
     """Resolve the options to pass to the store vfs opener.
 
     The returned dict is used to influence behavior of the storage layer.
     """
     options = {}
 
     if requirementsmod.TREEMANIFEST_REQUIREMENT in requirements:
         options[b'treemanifest'] = True
 
     # experimental config: format.manifestcachesize
     manifestcachesize = ui.configint(b'format', b'manifestcachesize')
     if manifestcachesize is not None:
         options[b'manifestcachesize'] = manifestcachesize
 
     # In the absence of another requirement superseding a revlog-related
     # requirement, we have to assume the repo is using revlog version 0.
     # This revlog format is super old and we don't bother trying to parse
     # opener options for it because those options wouldn't do anything
     # meaningful on such old repos.
     if (
         requirementsmod.REVLOGV1_REQUIREMENT in requirements
         or requirementsmod.REVLOGV2_REQUIREMENT in requirements
     ):
         options.update(resolverevlogstorevfsoptions(ui, requirements, features))
     else:  # explicitly mark repo as using revlogv0
         options[b'revlogv0'] = True
 
     if requirementsmod.COPIESSDC_REQUIREMENT in requirements:
         options[b'copies-storage'] = b'changeset-sidedata'
     else:
         writecopiesto = ui.config(b'experimental', b'copies.write-to')
         copiesextramode = (b'changeset-only', b'compatibility')
         if writecopiesto in copiesextramode:
             options[b'copies-storage'] = b'extra'
 
     return options
 
 
 def resolverevlogstorevfsoptions(ui, requirements, features):
     """Resolve opener options specific to revlogs."""
 
     options = {}
     options[b'flagprocessors'] = {}
 
     if requirementsmod.REVLOGV1_REQUIREMENT in requirements:
         options[b'revlogv1'] = True
     if requirementsmod.REVLOGV2_REQUIREMENT in requirements:
         options[b'revlogv2'] = True
 
     if requirementsmod.GENERALDELTA_REQUIREMENT in requirements:
         options[b'generaldelta'] = True
 
     # experimental config: format.chunkcachesize
     chunkcachesize = ui.configint(b'format', b'chunkcachesize')
     if chunkcachesize is not None:
         options[b'chunkcachesize'] = chunkcachesize
 
     deltabothparents = ui.configbool(
         b'storage', b'revlog.optimize-delta-parent-choice'
     )
     options[b'deltabothparents'] = deltabothparents
 
     lazydelta = ui.configbool(b'storage', b'revlog.reuse-external-delta')
     lazydeltabase = False
     if lazydelta:
         lazydeltabase = ui.configbool(
             b'storage', b'revlog.reuse-external-delta-parent'
         )
     if lazydeltabase is None:
         lazydeltabase = not scmutil.gddeltaconfig(ui)
     options[b'lazydelta'] = lazydelta
     options[b'lazydeltabase'] = lazydeltabase
 
     chainspan = ui.configbytes(b'experimental', b'maxdeltachainspan')
     if 0 <= chainspan:
         options[b'maxdeltachainspan'] = chainspan
 
     mmapindexthreshold = ui.configbytes(b'experimental', b'mmapindexthreshold')
     if mmapindexthreshold is not None:
         options[b'mmapindexthreshold'] = mmapindexthreshold
 
     withsparseread = ui.configbool(b'experimental', b'sparse-read')
     srdensitythres = float(
         ui.config(b'experimental', b'sparse-read.density-threshold')
     )
     srmingapsize = ui.configbytes(b'experimental', b'sparse-read.min-gap-size')
     options[b'with-sparse-read'] = withsparseread
     options[b'sparse-read-density-threshold'] = srdensitythres
     options[b'sparse-read-min-gap-size'] = srmingapsize
 
     sparserevlog = requirementsmod.SPARSEREVLOG_REQUIREMENT in requirements
     options[b'sparse-revlog'] = sparserevlog
     if sparserevlog:
         options[b'generaldelta'] = True
 
     sidedata = requirementsmod.SIDEDATA_REQUIREMENT in requirements
     options[b'side-data'] = sidedata
 
     maxchainlen = None
     if sparserevlog:
         maxchainlen = revlogconst.SPARSE_REVLOG_MAX_CHAIN_LENGTH
     # experimental config: format.maxchainlen
     maxchainlen = ui.configint(b'format', b'maxchainlen', maxchainlen)
     if maxchainlen is not None:
         options[b'maxchainlen'] = maxchainlen
 
     for r in requirements:
         # we allow multiple compression engine requirement to co-exist because
         # strickly speaking, revlog seems to support mixed compression style.
         #
         # The compression used for new entries will be "the last one"
         prefix = r.startswith
         if prefix(b'revlog-compression-') or prefix(b'exp-compression-'):
             options[b'compengine'] = r.split(b'-', 2)[2]
 
     options[b'zlib.level'] = ui.configint(b'storage', b'revlog.zlib.level')
     if options[b'zlib.level'] is not None:
         if not (0 <= options[b'zlib.level'] <= 9):
             msg = _(b'invalid value for `storage.revlog.zlib.level` config: %d')
             raise error.Abort(msg % options[b'zlib.level'])
     options[b'zstd.level'] = ui.configint(b'storage', b'revlog.zstd.level')
     if options[b'zstd.level'] is not None:
         if not (0 <= options[b'zstd.level'] <= 22):
             msg = _(b'invalid value for `storage.revlog.zstd.level` config: %d')
             raise error.Abort(msg % options[b'zstd.level'])
 
     if requirementsmod.NARROW_REQUIREMENT in requirements:
         options[b'enableellipsis'] = True
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
     if requirementsmod.NODEMAP_REQUIREMENT in requirements:
         slow_path = ui.config(
             b'storage', b'revlog.persistent-nodemap.slow-path'
         )
         if slow_path not in (b'allow', b'warn', b'abort'):
             default = ui.config_default(
                 b'storage', b'revlog.persistent-nodemap.slow-path'
             )
             msg = _(
                 b'unknown value for config '
                 b'"storage.revlog.persistent-nodemap.slow-path": "%s"\n'
             )
             ui.warn(msg % slow_path)
             if not ui.quiet:
                 ui.warn(_(b'falling back to default value: %s\n') % default)
             slow_path = default
 
         msg = _(
             b"accessing `persistent-nodemap` repository without associated "
             b"fast implementation."
         )
         hint = _(
             b"check `hg help config.format.use-persistent-nodemap` "
             b"for details"
         )
         if not revlog.HAS_FAST_PERSISTENT_NODEMAP:
             if slow_path == b'warn':
                 msg = b"warning: " + msg + b'\n'
                 ui.warn(msg)
                 if not ui.quiet:
                     hint = b'(' + hint + b')\n'
                     ui.warn(hint)
             if slow_path == b'abort':
                 raise error.Abort(msg, hint=hint)
         options[b'persistent-nodemap'] = True
     if ui.configbool(b'storage', b'revlog.persistent-nodemap.mmap'):
         options[b'persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
     return options
 
 
 def makemain(**kwargs):
     """Produce a type conforming to ``ilocalrepositorymain``."""
     return localrepository
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class revlogfilestorage(object):
     """File storage when using revlogs."""
 
     def file(self, path):
         if path.startswith(b'/'):
             path = path[1:]
 
         return filelog.filelog(self.svfs, path)
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
 class revlognarrowfilestorage(object):
     """File storage when using revlogs and narrow files."""
 
     def file(self, path):
         if path.startswith(b'/'):
             path = path[1:]
 
         return filelog.narrowfilelog(self.svfs, path, self._storenarrowmatch)
 
 
 def makefilestorage(requirements, features, **kwargs):
     """Produce a type conforming to ``ilocalrepositoryfilestorage``."""
     features.add(repository.REPO_FEATURE_REVLOG_FILE_STORAGE)
     features.add(repository.REPO_FEATURE_STREAM_CLONE)
 
     if requirementsmod.NARROW_REQUIREMENT in requirements:
         return revlognarrowfilestorage
     else:
         return revlogfilestorage
 
 
 # List of repository interfaces and factory functions for them. Each
 # will be called in order during ``makelocalrepository()`` to iteratively
 # derive the final type for a local repository instance. We capture the
 # function as a lambda so we don't hold a reference and the module-level
 # functions can be wrapped.
 REPO_INTERFACES = [
     (repository.ilocalrepositorymain, lambda: makemain),
     (repository.ilocalrepositoryfilestorage, lambda: makefilestorage),
 ]
 
 
 @interfaceutil.implementer(repository.ilocalrepositorymain)
 class localrepository(object):
     """Main class for representing local repositories.
 
     All local repositories are instances of this class.
 
     Constructed on its own, instances of this class are not usable as
     repository objects. To obtain a usable repository object, call
     ``hg.repository()``, ``localrepo.instance()``, or
     ``localrepo.makelocalrepository()``. The latter is the lowest-level.
     ``instance()`` adds support for creating new repositories.
     ``hg.repository()`` adds more extension integration, including calling
     ``reposetup()``. Generally speaking, ``hg.repository()`` should be
     used.
     """
 
     # obsolete experimental requirements:
     #  - manifestv2: An experimental new manifest format that allowed
     #    for stem compression of long paths. Experiment ended up not
     #    being successful (repository sizes went up due to worse delta
     #    chains), and the code was deleted in 4.6.
     supportedformats = {
         requirementsmod.REVLOGV1_REQUIREMENT,
         requirementsmod.GENERALDELTA_REQUIREMENT,
         requirementsmod.TREEMANIFEST_REQUIREMENT,
         requirementsmod.COPIESSDC_REQUIREMENT,
         requirementsmod.REVLOGV2_REQUIREMENT,
         requirementsmod.SIDEDATA_REQUIREMENT,
         requirementsmod.SPARSEREVLOG_REQUIREMENT,
         requirementsmod.NODEMAP_REQUIREMENT,
         bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT,
         requirementsmod.SHARESAFE_REQUIREMENT,
     }
     _basesupported = supportedformats | {
         requirementsmod.STORE_REQUIREMENT,
         requirementsmod.FNCACHE_REQUIREMENT,
         requirementsmod.SHARED_REQUIREMENT,
         requirementsmod.RELATIVE_SHARED_REQUIREMENT,
         requirementsmod.DOTENCODE_REQUIREMENT,
         requirementsmod.SPARSE_REQUIREMENT,
         requirementsmod.INTERNAL_PHASE_REQUIREMENT,
     }
 
     # list of prefix for file which can be written without 'wlock'
     # Extensions should extend this list when needed
     _wlockfreeprefix = {
         # We migh consider requiring 'wlock' for the next
         # two, but pretty much all the existing code assume
         # wlock is not needed so we keep them excluded for
         # now.
         b'hgrc',
         b'requires',
         # XXX cache is a complicatged business someone
         # should investigate this in depth at some point
         b'cache/',
         # XXX shouldn't be dirstate covered by the wlock?
         b'dirstate',
         # XXX bisect was still a bit too messy at the time
         # this changeset was introduced. Someone should fix
         # the remainig bit and drop this line
         b'bisect.state',
     }
 
     def __init__(
         self,
         baseui,
         ui,
         origroot,
         wdirvfs,
         hgvfs,
         requirements,
         supportedrequirements,
         sharedpath,
         store,
         cachevfs,
         wcachevfs,
         features,
         intents=None,
     ):
         """Create a new local repository instance.
 
         Most callers should use ``hg.repository()``, ``localrepo.instance()``,
         or ``localrepo.makelocalrepository()`` for obtaining a new repository
         object.
 
         Arguments:
 
         baseui
            ``ui.ui`` instance that ``ui`` argument was based off of.
 
         ui
            ``ui.ui`` instance for use by the repository.
 
         origroot
            ``bytes`` path to working directory root of this repository.
 
         wdirvfs
            ``vfs.vfs`` rooted at the working directory.
 
         hgvfs
            ``vfs.vfs`` rooted at .hg/
 
         requirements
            ``set`` of bytestrings representing repository opening requirements.
 
         supportedrequirements
            ``set`` of bytestrings representing repository requirements that we
            know how to open. May be a supetset of ``requirements``.
 
         sharedpath
            ``bytes`` Defining path to storage base directory. Points to a
            ``.hg/`` directory somewhere.
 
         store
            ``store.basicstore`` (or derived) instance providing access to
            versioned storage.
 
         cachevfs
            ``vfs.vfs`` used for cache files.
 
         wcachevfs
            ``vfs.vfs`` used for cache files related to the working copy.
 
         features
            ``set`` of bytestrings defining features/capabilities of this
            instance.
 
         intents
            ``set`` of system strings indicating what this repo will be used
            for.
         """
         self.baseui = baseui
         self.ui = ui
         self.origroot = origroot
         # vfs rooted at working directory.
         self.wvfs = wdirvfs
         self.root = wdirvfs.base
         # vfs rooted at .hg/. Used to access most non-store paths.
         self.vfs = hgvfs
         self.path = hgvfs.base
         self.requirements = requirements
         self.supported = supportedrequirements
         self.sharedpath = sharedpath
         self.store = store
         self.cachevfs = cachevfs
         self.wcachevfs = wcachevfs
         self.features = features
 
         self.filtername = None
 
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             self.vfs.audit = self._getvfsward(self.vfs.audit)
         # A list of callback to shape the phase if no data were found.
         # Callback are in the form: func(repo, roots) --> processed root.
         # This list it to be filled by extension during repo setup
         self._phasedefaults = []
 
         color.setup(self.ui)
 
         self.spath = self.store.path
         self.svfs = self.store.vfs
         self.sjoin = self.store.join
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             if util.safehasattr(self.svfs, b'vfs'):  # this is filtervfs
                 self.svfs.vfs.audit = self._getsvfsward(self.svfs.vfs.audit)
             else:  # standard vfs
                 self.svfs.audit = self._getsvfsward(self.svfs.audit)
 
         self._dirstatevalidatewarned = False
 
         self._branchcaches = branchmap.BranchMapCache()
         self._revbranchcache = None
         self._filterpats = {}
         self._datafilters = {}
         self._transref = self._lockref = self._wlockref = None
 
         # A cache for various files under .hg/ that tracks file changes,
         # (used by the filecache decorator)
         #
         # Maps a property name to its util.filecacheentry
         self._filecache = {}
 
         # hold sets of revision to be filtered
         # should be cleared when something might have changed the filter value:
         # - new changesets,
         # - phase change,
         # - new obsolescence marker,
         # - working directory parent change,
         # - bookmark changes
         self.filteredrevcache = {}
 
         # post-dirstate-status hooks
         self._postdsstatus = []
 
         # generic mapping between names and nodes
         self.names = namespaces.namespaces()
 
         # Key to signature value.
         self._sparsesignaturecache = {}
         # Signature to cached matcher instance.
         self._sparsematchercache = {}
 
         self._extrafilterid = repoview.extrafilter(ui)
 
         self.filecopiesmode = None
         if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements:
             self.filecopiesmode = b'changeset-sidedata'
 
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
 
         def checkvfs(path, mode=None):
             ret = origfunc(path, mode=mode)
             repo = rref()
             if (
                 repo is None
                 or not util.safehasattr(repo, b'_wlockref')
                 or not util.safehasattr(repo, b'_lockref')
             ):
                 return
             if mode in (None, b'r', b'rb'):
                 return
             if path.startswith(repo.path):
                 # truncate name relative to the repository (.hg)
                 path = path[len(repo.path) + 1 :]
             if path.startswith(b'cache/'):
                 msg = b'accessing cache with vfs instead of cachevfs: "%s"'
                 repo.ui.develwarn(msg % path, stacklevel=3, config=b"cache-vfs")
             # path prefixes covered by 'lock'
             vfs_path_prefixes = (
                 b'journal.',
                 b'undo.',
                 b'strip-backup/',
                 b'cache/',
             )
             if any(path.startswith(prefix) for prefix in vfs_path_prefixes):
                 if repo._currentlock(repo._lockref) is None:
                     repo.ui.develwarn(
                         b'write with no lock: "%s"' % path,
                         stacklevel=3,
                         config=b'check-locks',
                     )
             elif repo._currentlock(repo._wlockref) is None:
                 # rest of vfs files are covered by 'wlock'
                 #
                 # exclude special files
                 for prefix in self._wlockfreeprefix:
                     if path.startswith(prefix):
                         return
                 repo.ui.develwarn(
                     b'write with no wlock: "%s"' % path,
                     stacklevel=3,
                     config=b'check-locks',
                 )
             return ret
 
         return checkvfs
 
     def _getsvfsward(self, origfunc):
         """build a ward for self.svfs"""
         rref = weakref.ref(self)
 
         def checksvfs(path, mode=None):
             ret = origfunc(path, mode=mode)
             repo = rref()
             if repo is None or not util.safehasattr(repo, b'_lockref'):
                 return
             if mode in (None, b'r', b'rb'):
                 return
             if path.startswith(repo.sharedpath):
                 # truncate name relative to the repository (.hg)
                 path = path[len(repo.sharedpath) + 1 :]
             if repo._currentlock(repo._lockref) is None:
                 repo.ui.develwarn(
                     b'write with no lock: "%s"' % path, stacklevel=4
                 )
             return ret
 
         return checksvfs
 
     def close(self):
         self._writecaches()
 
     def _writecaches(self):
         if self._revbranchcache:
             self._revbranchcache.write()
 
     def _restrictcapabilities(self, caps):
         if self.ui.configbool(b'experimental', b'bundle2-advertise'):
             caps = set(caps)
             capsblob = bundle2.encodecaps(
                 bundle2.getrepocaps(self, role=b'client')
             )
             caps.add(b'bundle2=' + urlreq.quote(capsblob))
         return caps
 
     # Don't cache auditor/nofsauditor, or you'll end up with reference cycle:
     # self -> auditor -> self._checknested -> self
 
     @property
     def auditor(self):
         # This is only used by context.workingctx.match in order to
         # detect files in subrepos.
         return pathutil.pathauditor(self.root, callback=self._checknested)
 
     @property
     def nofsauditor(self):
         # This is only used by context.basectx.match in order to detect
         # files in subrepos.
         return pathutil.pathauditor(
             self.root, callback=self._checknested, realfs=False, cached=True
         )
 
     def _checknested(self, path):
         """Determine if path is a legal nested repository."""
         if not path.startswith(self.root):
             return False
         subpath = path[len(self.root) + 1 :]
         normsubpath = util.pconvert(subpath)
 
         # XXX: Checking against the current working copy is wrong in
         # the sense that it can reject things like
         #
         #   $ hg cat -r 10 sub/x.txt
         #
         # if sub/ is no longer a subrepository in the working copy
         # parent revision.
         #
         # However, it can of course also allow things that would have
         # been rejected before, such as the above cat command if sub/
         # is a subrepository now, but was a normal directory before.
         # The old path auditor would have rejected by mistake since it
         # panics when it sees sub/.hg/.
         #
         # All in all, checking against the working copy seems sensible
         # since we want to prevent access to nested repositories on
         # the filesystem *now*.
         ctx = self[None]
         parts = util.splitpath(subpath)
         while parts:
             prefix = b'/'.join(parts)
             if prefix in ctx.substate:
                 if prefix == normsubpath:
                     return True
                 else:
                     sub = ctx.sub(prefix)
                     return sub.checknested(subpath[len(prefix) + 1 :])
             else:
                 parts.pop()
         return False
 
     def peer(self):
         return localpeer(self)  # not cached to avoid reference cycle
 
     def unfiltered(self):
         """Return unfiltered version of the repository
 
         Intended to be overwritten by filtered repo."""
         return self
 
     def filtered(self, name, visibilityexceptions=None):
         """Return a filtered version of a repository
 
         The `name` parameter is the identifier of the requested view. This
         will return a repoview object set "exactly" to the specified view.
 
         This function does not apply recursive filtering to a repository. For
         example calling `repo.filtered("served")` will return a repoview using
         the "served" view, regardless of the initial view used by `repo`.
 
         In other word, there is always only one level of `repoview` "filtering".
         """
         if self._extrafilterid is not None and b'%' not in name:
             name = name + b'%' + self._extrafilterid
 
         cls = repoview.newtype(self.unfiltered().__class__)
         return cls(self, name, visibilityexceptions)
 
     @mixedrepostorecache(
         (b'bookmarks', b'plain'),
         (b'bookmarks.current', b'plain'),
         (b'bookmarks', b''),
         (b'00changelog.i', b''),
     )
     def _bookmarks(self):
         # Since the multiple files involved in the transaction cannot be
         # written atomically (with current repository format), there is a race
         # condition here.
         #
         # 1) changelog content A is read
         # 2) outside transaction update changelog to content B
         # 3) outside transaction update bookmark file referring to content B
         # 4) bookmarks file content is read and filtered against changelog-A
         #
         # When this happens, bookmarks against nodes missing from A are dropped.
         #
         # Having this happening during read is not great, but it become worse
         # when this happen during write because the bookmarks to the "unknown"
         # nodes will be dropped for good. However, writes happen within locks.
         # This locking makes it possible to have a race free consistent read.
         # For this purpose data read from disc before locking  are
         # "invalidated" right after the locks are taken. This invalidations are
         # "light", the `filecache` mechanism keep the data in memory and will
         # reuse them if the underlying files did not changed. Not parsing the
         # same data multiple times helps performances.
         #
         # Unfortunately in the case describe above, the files tracked by the
         # bookmarks file cache might not have changed, but the in-memory
         # content is still "wrong" because we used an older changelog content
         # to process the on-disk data. So after locking, the changelog would be
         # refreshed but `_bookmarks` would be preserved.
         # Adding `00changelog.i` to the list of tracked file is not
         # enough, because at the time we build the content for `_bookmarks` in
         # (4), the changelog file has already diverged from the content used
         # for loading `changelog` in (1)
         #
         # To prevent the issue, we force the changelog to be explicitly
         # reloaded while computing `_bookmarks`. The data race can still happen
         # without the lock (with a narrower window), but it would no longer go
         # undetected during the lock time refresh.
         #
         # The new schedule is as follow
         #
         # 1) filecache logic detect that `_bookmarks` needs to be computed
         # 2) cachestat for `bookmarks` and `changelog` are captured (for book)
         # 3) We force `changelog` filecache to be tested
         # 4) cachestat for `changelog` are captured (for changelog)
         # 5) `_bookmarks` is computed and cached
         #
         # The step in (3) ensure we have a changelog at least as recent as the
         # cache stat computed in (1). As a result at locking time:
         #  * if the changelog did not changed since (1) -> we can reuse the data
         #  * otherwise -> the bookmarks get refreshed.
         self._refreshchangelog()
         return bookmarks.bmstore(self)
 
     def _refreshchangelog(self):
         """make sure the in memory changelog match the on-disk one"""
         if 'changelog' in vars(self) and self.currenttransaction() is None:
             del self.changelog
 
     @property
     def _activebookmark(self):
         return self._bookmarks.active
 
     # _phasesets depend on changelog. what we need is to call
     # _phasecache.invalidate() if '00changelog.i' was changed, but it
     # can't be easily expressed in filecache mechanism.
     @storecache(b'phaseroots', b'00changelog.i')
     def _phasecache(self):
         return phases.phasecache(self, self._phasedefaults)
 
     @storecache(b'obsstore')
     def obsstore(self):
         return obsolete.makestore(self.ui, self)
 
     @storecache(b'00changelog.i')
     def changelog(self):
         # load dirstate before changelog to avoid race see issue6303
         self.dirstate.prefetch_parents()
         return self.store.changelog(
             txnutil.mayhavepending(self.root),
             concurrencychecker=revlogchecker.get_checker(self.ui, b'changelog'),
         )
 
     @storecache(b'00manifest.i')
     def manifestlog(self):
         return self.store.manifestlog(self, self._storenarrowmatch)
 
     @repofilecache(b'dirstate')
     def dirstate(self):
         return self._makedirstate()
 
     def _makedirstate(self):
         """Extension point for wrapping the dirstate per-repo."""
         sparsematchfn = lambda: sparse.matcher(self)
 
         return dirstate.dirstate(
             self.vfs, self.ui, self.root, self._dirstatevalidate, sparsematchfn
         )
 
     def _dirstatevalidate(self, node):
         try:
             self.changelog.rev(node)
             return node
         except error.LookupError:
             if not self._dirstatevalidatewarned:
                 self._dirstatevalidatewarned = True
                 self.ui.warn(
                     _(b"warning: ignoring unknown working parent %s!\n")
                     % short(node)
                 )
             return nullid
 
     @storecache(narrowspec.FILENAME)
     def narrowpats(self):
         """matcher patterns for this repository's narrowspec
 
         A tuple of (includes, excludes).
         """
         return narrowspec.load(self)
 
     @storecache(narrowspec.FILENAME)
     def _storenarrowmatch(self):
         if requirementsmod.NARROW_REQUIREMENT not in self.requirements:
             return matchmod.always()
         include, exclude = self.narrowpats
         return narrowspec.match(self.root, include=include, exclude=exclude)
 
     @storecache(narrowspec.FILENAME)
     def _narrowmatch(self):
         if requirementsmod.NARROW_REQUIREMENT not in self.requirements:
             return matchmod.always()
         narrowspec.checkworkingcopynarrowspec(self)
         include, exclude = self.narrowpats
         return narrowspec.match(self.root, include=include, exclude=exclude)
 
     def narrowmatch(self, match=None, includeexact=False):
         """matcher corresponding the the repo's narrowspec
 
         If `match` is given, then that will be intersected with the narrow
         matcher.
 
         If `includeexact` is True, then any exact matches from `match` will
         be included even if they're outside the narrowspec.
         """
         if match:
             if includeexact and not self._narrowmatch.always():
                 # do not exclude explicitly-specified paths so that they can
                 # be warned later on
                 em = matchmod.exact(match.files())
                 nm = matchmod.unionmatcher([self._narrowmatch, em])
                 return matchmod.intersectmatchers(match, nm)
             return matchmod.intersectmatchers(match, self._narrowmatch)
         return self._narrowmatch
 
     def setnarrowpats(self, newincludes, newexcludes):
         narrowspec.save(self, newincludes, newexcludes)
         self.invalidate(clearfilecache=True)
 
     @unfilteredpropertycache
     def _quick_access_changeid_null(self):
         return {
             b'null': (nullrev, nullid),
             nullrev: (nullrev, nullid),
             nullid: (nullrev, nullid),
         }
 
     @unfilteredpropertycache
     def _quick_access_changeid_wc(self):
         # also fast path access to the working copy parents
         # however, only do it for filter that ensure wc is visible.
         quick = self._quick_access_changeid_null.copy()
         cl = self.unfiltered().changelog
         for node in self.dirstate.parents():
             if node == nullid:
                 continue
             rev = cl.index.get_rev(node)
             if rev is None:
                 # unknown working copy parent case:
                 #
                 #   skip the fast path and let higher code deal with it
                 continue
             pair = (rev, node)
             quick[rev] = pair
             quick[node] = pair
             # also add the parents of the parents
             for r in cl.parentrevs(rev):
                 if r == nullrev:
                     continue
                 n = cl.node(r)
                 pair = (r, n)
                 quick[r] = pair
                 quick[n] = pair
         p1node = self.dirstate.p1()
         if p1node != nullid:
             quick[b'.'] = quick[p1node]
         return quick
 
     @unfilteredmethod
     def _quick_access_changeid_invalidate(self):
         if '_quick_access_changeid_wc' in vars(self):
             del self.__dict__['_quick_access_changeid_wc']
 
     @property
     def _quick_access_changeid(self):
         """an helper dictionnary for __getitem__ calls
 
         This contains a list of symbol we can recognise right away without
         further processing.
         """
         if self.filtername in repoview.filter_has_wc:
             return self._quick_access_changeid_wc
         return self._quick_access_changeid_null
 
     def __getitem__(self, changeid):
         # dealing with special cases
         if changeid is None:
             return context.workingctx(self)
         if isinstance(changeid, context.basectx):
             return changeid
 
         # dealing with multiple revisions
         if isinstance(changeid, slice):
             # wdirrev isn't contiguous so the slice shouldn't include it
             return [
                 self[i]
                 for i in pycompat.xrange(*changeid.indices(len(self)))
                 if i not in self.changelog.filteredrevs
             ]
 
         # dealing with some special values
         quick_access = self._quick_access_changeid.get(changeid)
         if quick_access is not None:
             rev, node = quick_access
             return context.changectx(self, rev, node, maybe_filtered=False)
         if changeid == b'tip':
             node = self.changelog.tip()
             rev = self.changelog.rev(node)
             return context.changectx(self, rev, node)
 
         # dealing with arbitrary values
         try:
             if isinstance(changeid, int):
                 node = self.changelog.node(changeid)
                 rev = changeid
             elif changeid == b'.':
                 # this is a hack to delay/avoid loading obsmarkers
                 # when we know that '.' won't be hidden
                 node = self.dirstate.p1()
                 rev = self.unfiltered().changelog.rev(node)
             elif len(changeid) == 20:
                 try:
                     node = changeid
                     rev = self.changelog.rev(changeid)
                 except error.FilteredLookupError:
                     changeid = hex(changeid)  # for the error message
                     raise
                 except LookupError:
                     # check if it might have come from damaged dirstate
                     #
                     # XXX we could avoid the unfiltered if we had a recognizable
                     # exception for filtered changeset access
                     if (
                         self.local()
                         and changeid in self.unfiltered().dirstate.parents()
                     ):
                         msg = _(b"working directory has unknown parent '%s'!")
                         raise error.Abort(msg % short(changeid))
                     changeid = hex(changeid)  # for the error message
                     raise
 
             elif len(changeid) == 40:
                 node = bin(changeid)
                 rev = self.changelog.rev(node)
             else:
                 raise error.ProgrammingError(
                     b"unsupported changeid '%s' of type %s"
                     % (changeid, pycompat.bytestr(type(changeid)))
                 )
 
             return context.changectx(self, rev, node)
 
         except (error.FilteredIndexError, error.FilteredLookupError):
             raise error.FilteredRepoLookupError(
                 _(b"filtered revision '%s'") % pycompat.bytestr(changeid)
             )
         except (IndexError, LookupError):
             raise error.RepoLookupError(
                 _(b"unknown revision '%s'") % pycompat.bytestr(changeid)
             )
         except error.WdirUnsupported:
             return context.workingctx(self)
 
     def __contains__(self, changeid):
         """True if the given changeid exists"""
         try:
             self[changeid]
             return True
         except error.RepoLookupError:
             return False
 
     def __nonzero__(self):
         return True
 
     __bool__ = __nonzero__
 
     def __len__(self):
         # no need to pay the cost of repoview.changelog
         unfi = self.unfiltered()
         return len(unfi.changelog)
 
     def __iter__(self):
         return iter(self.changelog)
 
     def revs(self, expr, *args):
         """Find revisions matching a revset.
 
         The revset is specified as a string ``expr`` that may contain
         %-formatting to escape certain types. See ``revsetlang.formatspec``.
 
         Revset aliases from the configuration are not expanded. To expand
         user aliases, consider calling ``scmutil.revrange()`` or
         ``repo.anyrevs([expr], user=True)``.
 
         Returns a smartset.abstractsmartset, which is a list-like interface
         that contains integer revisions.
         """
         tree = revsetlang.spectree(expr, *args)
         return revset.makematcher(tree)(self)
 
     def set(self, expr, *args):
         """Find revisions matching a revset and emit changectx instances.
 
         This is a convenience wrapper around ``revs()`` that iterates the
         result and is a generator of changectx instances.
 
         Revset aliases from the configuration are not expanded. To expand
         user aliases, consider calling ``scmutil.revrange()``.
         """
         for r in self.revs(expr, *args):
             yield self[r]
 
     def anyrevs(self, specs, user=False, localalias=None):
         """Find revisions matching one of the given revsets.
 
         Revset aliases from the configuration are not expanded by default. To
         expand user aliases, specify ``user=True``. To provide some local
         definitions overriding user aliases, set ``localalias`` to
         ``{name: definitionstring}``.
         """
         if specs == [b'null']:
             return revset.baseset([nullrev])
         if specs == [b'.']:
             quick_data = self._quick_access_changeid.get(b'.')
             if quick_data is not None:
                 return revset.baseset([quick_data[0]])
         if user:
             m = revset.matchany(
                 self.ui,
                 specs,
                 lookup=revset.lookupfn(self),
                 localalias=localalias,
             )
         else:
             m = revset.matchany(None, specs, localalias=localalias)
         return m(self)
 
     def url(self):
         return b'file:' + self.root
 
     def hook(self, name, throw=False, **args):
         """Call a hook, passing this repo instance.
 
         This a convenience method to aid invoking hooks. Extensions likely
         won't call this unless they have registered a custom hook or are
         replacing code that is expected to call a hook.
         """
         return hook.hook(self.ui, self, name, throw, **args)
 
     @filteredpropertycache
     def _tagscache(self):
         """Returns a tagscache object that contains various tags related
         caches."""
 
         # This simplifies its cache management by having one decorated
         # function (this one) and the rest simply fetch things from it.
         class tagscache(object):
             def __init__(self):
                 # These two define the set of tags for this repository. tags
                 # maps tag name to node; tagtypes maps tag name to 'global' or
                 # 'local'. (Global tags are defined by .hgtags across all
                 # heads, and local tags are defined in .hg/localtags.)
                 # They constitute the in-memory cache of tags.
                 self.tags = self.tagtypes = None
 
                 self.nodetagscache = self.tagslist = None
 
         cache = tagscache()
         cache.tags, cache.tagtypes = self._findtags()
 
         return cache
 
     def tags(self):
         '''return a mapping of tag to node'''
         t = {}
         if self.changelog.filteredrevs:
             tags, tt = self._findtags()
         else:
             tags = self._tagscache.tags
         rev = self.changelog.rev
         for k, v in pycompat.iteritems(tags):
             try:
                 # ignore tags to unknown nodes
                 rev(v)
                 t[k] = v
             except (error.LookupError, ValueError):
                 pass
         return t
 
     def _findtags(self):
         """Do the hard work of finding tags.  Return a pair of dicts
         (tags, tagtypes) where tags maps tag name to node, and tagtypes
         maps tag name to a string like \'global\' or \'local\'.
         Subclasses or extensions are free to add their own tags, but
         should be aware that the returned dicts will be retained for the
         duration of the localrepo object."""
 
         # XXX what tagtype should subclasses/extensions use?  Currently
         # mq and bookmarks add tags, but do not set the tagtype at all.
         # Should each extension invent its own tag type?  Should there
         # be one tagtype for all such "virtual" tags?  Or is the status
         # quo fine?
 
         # map tag name to (node, hist)
         alltags = tagsmod.findglobaltags(self.ui, self)
         # map tag name to tag type
         tagtypes = {tag: b'global' for tag in alltags}
 
         tagsmod.readlocaltags(self.ui, self, alltags, tagtypes)
 
         # Build the return dicts.  Have to re-encode tag names because
         # the tags module always uses UTF-8 (in order not to lose info
         # writing to the cache), but the rest of Mercurial wants them in
         # local encoding.
         tags = {}
         for (name, (node, hist)) in pycompat.iteritems(alltags):
             if node != nullid:
                 tags[encoding.tolocal(name)] = node
         tags[b'tip'] = self.changelog.tip()
         tagtypes = {
             encoding.tolocal(name): value
             for (name, value) in pycompat.iteritems(tagtypes)
         }
         return (tags, tagtypes)
 
     def tagtype(self, tagname):
         """
         return the type of the given tag. result can be:
 
         'local'  : a local tag
         'global' : a global tag
         None     : tag does not exist
         """
 
         return self._tagscache.tagtypes.get(tagname)
 
     def tagslist(self):
         '''return a list of tags ordered by revision'''
         if not self._tagscache.tagslist:
             l = []
             for t, n in pycompat.iteritems(self.tags()):
                 l.append((self.changelog.rev(n), t, n))
             self._tagscache.tagslist = [(t, n) for r, t, n in sorted(l)]
 
         return self._tagscache.tagslist
 
     def nodetags(self, node):
         '''return the tags associated with a node'''
         if not self._tagscache.nodetagscache:
             nodetagscache = {}
             for t, n in pycompat.iteritems(self._tagscache.tags):
                 nodetagscache.setdefault(n, []).append(t)
             for tags in pycompat.itervalues(nodetagscache):
                 tags.sort()
             self._tagscache.nodetagscache = nodetagscache
         return self._tagscache.nodetagscache.get(node, [])
 
     def nodebookmarks(self, node):
         """return the list of bookmarks pointing to the specified node"""
         return self._bookmarks.names(node)
 
     def branchmap(self):
         """returns a dictionary {branch: [branchheads]} with branchheads
         ordered by increasing revision number"""
         return self._branchcaches[self]
 
     @unfilteredmethod
     def revbranchcache(self):
         if not self._revbranchcache:
             self._revbranchcache = branchmap.revbranchcache(self.unfiltered())
         return self._revbranchcache
 
     def register_changeset(self, rev, changelogrevision):
         self.revbranchcache().setdata(rev, changelogrevision)
 
     def branchtip(self, branch, ignoremissing=False):
         """return the tip node for a given branch
 
         If ignoremissing is True, then this method will not raise an error.
         This is helpful for callers that only expect None for a missing branch
         (e.g. namespace).
 
         """
         try:
             return self.branchmap().branchtip(branch)
         except KeyError:
             if not ignoremissing:
                 raise error.RepoLookupError(_(b"unknown branch '%s'") % branch)
             else:
                 pass
 
     def lookup(self, key):
         node = scmutil.revsymbol(self, key).node()
         if node is None:
             raise error.RepoLookupError(_(b"unknown revision '%s'") % key)
         return node
 
     def lookupbranch(self, key):
         if self.branchmap().hasbranch(key):
             return key
 
         return scmutil.revsymbol(self, key).branch()
 
     def known(self, nodes):
         cl = self.changelog
         get_rev = cl.index.get_rev
         filtered = cl.filteredrevs
         result = []
         for n in nodes:
             r = get_rev(n)
             resp = not (r is None or r in filtered)
             result.append(resp)
         return result
 
     def local(self):
         return self
 
     def publishing(self):
         # it's safe (and desirable) to trust the publish flag unconditionally
         # so that we don't finalize changes shared between users via ssh or nfs
         return self.ui.configbool(b'phases', b'publish', untrusted=True)
 
     def cancopy(self):
         # so statichttprepo's override of local() works
         if not self.local():
             return False
         if not self.publishing():
             return True
         # if publishing we can't copy if there is filtered content
         return not self.filtered(b'visible').changelog.filteredrevs
 
     def shared(self):
         '''the type of shared repository (None if not shared)'''
         if self.sharedpath != self.path:
             return b'store'
         return None
 
     def wjoin(self, f, *insidef):
         return self.vfs.reljoin(self.root, f, *insidef)
 
     def setparents(self, p1, p2=nullid):
         self[None].setparents(p1, p2)
         self._quick_access_changeid_invalidate()
 
     def filectx(self, path, changeid=None, fileid=None, changectx=None):
         """changeid must be a changeset revision, if specified.
         fileid can be a file revision or node."""
         return context.filectx(
             self, path, changeid, fileid, changectx=changectx
         )
 
     def getcwd(self):
         return self.dirstate.getcwd()
 
     def pathto(self, f, cwd=None):
         return self.dirstate.pathto(f, cwd)
 
     def _loadfilter(self, filter):
         if filter not in self._filterpats:
             l = []
             for pat, cmd in self.ui.configitems(filter):
                 if cmd == b'!':
                     continue
                 mf = matchmod.match(self.root, b'', [pat])
                 fn = None
                 params = cmd
                 for name, filterfn in pycompat.iteritems(self._datafilters):
                     if cmd.startswith(name):
                         fn = filterfn
                         params = cmd[len(name) :].lstrip()
                         break
                 if not fn:
                     fn = lambda s, c, **kwargs: procutil.filter(s, c)
                     fn.__name__ = 'commandfilter'
                 # Wrap old filters not supporting keyword arguments
                 if not pycompat.getargspec(fn)[2]:
                     oldfn = fn
                     fn = lambda s, c, oldfn=oldfn, **kwargs: oldfn(s, c)
                     fn.__name__ = 'compat-' + oldfn.__name__
                 l.append((mf, fn, params))
             self._filterpats[filter] = l
         return self._filterpats[filter]
 
     def _filter(self, filterpats, filename, data):
         for mf, fn, cmd in filterpats:
             if mf(filename):
                 self.ui.debug(
                     b"filtering %s through %s\n"
                     % (filename, cmd or pycompat.sysbytes(fn.__name__))
                 )
                 data = fn(data, cmd, ui=self.ui, repo=self, filename=filename)
                 break
 
         return data
 
     @unfilteredpropertycache
     def _encodefilterpats(self):
         return self._loadfilter(b'encode')
 
     @unfilteredpropertycache
     def _decodefilterpats(self):
         return self._loadfilter(b'decode')
 
     def adddatafilter(self, name, filter):
         self._datafilters[name] = filter
 
     def wread(self, filename):
         if self.wvfs.islink(filename):
             data = self.wvfs.readlink(filename)
         else:
             data = self.wvfs.read(filename)
         return self._filter(self._encodefilterpats, filename, data)
 
     def wwrite(self, filename, data, flags, backgroundclose=False, **kwargs):
         """write ``data`` into ``filename`` in the working directory
 
         This returns length of written (maybe decoded) data.
         """
         data = self._filter(self._decodefilterpats, filename, data)
         if b'l' in flags:
             self.wvfs.symlink(data, filename)
         else:
             self.wvfs.write(
                 filename, data, backgroundclose=backgroundclose, **kwargs
             )
             if b'x' in flags:
                 self.wvfs.setflags(filename, False, True)
             else:
                 self.wvfs.setflags(filename, False, False)
         return len(data)
 
     def wwritedata(self, filename, data):
         return self._filter(self._decodefilterpats, filename, data)
 
     def currenttransaction(self):
         """return the current transaction or None if non exists"""
         if self._transref:
             tr = self._transref()
         else:
             tr = None
 
         if tr and tr.running():
             return tr
         return None
 
     def transaction(self, desc, report=None):
         if self.ui.configbool(b'devel', b'all-warnings') or self.ui.configbool(
             b'devel', b'check-locks'
         ):
             if self._currentlock(self._lockref) is None:
                 raise error.ProgrammingError(b'transaction requires locking')
         tr = self.currenttransaction()
         if tr is not None:
             return tr.nest(name=desc)
 
         # abort here if the journal already exists
         if self.svfs.exists(b"journal"):
             raise error.RepoError(
                 _(b"abandoned transaction found"),
                 hint=_(b"run 'hg recover' to clean up transaction"),
             )
 
         idbase = b"%.40f#%f" % (random.random(), time.time())
         ha = hex(hashutil.sha1(idbase).digest())
         txnid = b'TXN:' + ha
         self.hook(b'pretxnopen', throw=True, txnname=desc, txnid=txnid)
 
         self._writejournal(desc)
         renames = [(vfs, x, undoname(x)) for vfs, x in self._journalfiles()]
         if report:
             rp = report
         else:
             rp = self.ui.warn
         vfsmap = {b'plain': self.vfs, b'store': self.svfs}  # root of .hg/
         # we must avoid cyclic reference between repo and transaction.
         reporef = weakref.ref(self)
         # Code to track tag movement
         #
         # Since tags are all handled as file content, it is actually quite hard
         # to track these movement from a code perspective. So we fallback to a
         # tracking at the repository level. One could envision to track changes
         # to the '.hgtags' file through changegroup apply but that fails to
         # cope with case where transaction expose new heads without changegroup
         # being involved (eg: phase movement).
         #
         # For now, We gate the feature behind a flag since this likely comes
         # with performance impacts. The current code run more often than needed
         # and do not use caches as much as it could.  The current focus is on
         # the behavior of the feature so we disable it by default. The flag
         # will be removed when we are happy with the performance impact.
         #
         # Once this feature is no longer experimental move the following
         # documentation to the appropriate help section:
         #
         # The ``HG_TAG_MOVED`` variable will be set if the transaction touched
         # tags (new or changed or deleted tags). In addition the details of
         # these changes are made available in a file at:
         #     ``REPOROOT/.hg/changes/tags.changes``.
         # Make sure you check for HG_TAG_MOVED before reading that file as it
         # might exist from a previous transaction even if no tag were touched
         # in this one. Changes are recorded in a line base format::
         #
         #     <action> <hex-node> <tag-name>\n
         #
         # Actions are defined as follow:
         #   "-R": tag is removed,
         #   "+A": tag is added,
         #   "-M": tag is moved (old value),
         #   "+M": tag is moved (new value),
         tracktags = lambda x: None
         # experimental config: experimental.hook-track-tags
         shouldtracktags = self.ui.configbool(
             b'experimental', b'hook-track-tags'
         )
         if desc != b'strip' and shouldtracktags:
             oldheads = self.changelog.headrevs()
 
             def tracktags(tr2):
                 repo = reporef()
                 oldfnodes = tagsmod.fnoderevs(repo.ui, repo, oldheads)
                 newheads = repo.changelog.headrevs()
                 newfnodes = tagsmod.fnoderevs(repo.ui, repo, newheads)
                 # notes: we compare lists here.
                 # As we do it only once buiding set would not be cheaper
                 changes = tagsmod.difftags(repo.ui, repo, oldfnodes, newfnodes)
                 if changes:
                     tr2.hookargs[b'tag_moved'] = b'1'
                     with repo.vfs(
                         b'changes/tags.changes', b'w', atomictemp=True
                     ) as changesfile:
                         # note: we do not register the file to the transaction
                         # because we needs it to still exist on the transaction
                         # is close (for txnclose hooks)
                         tagsmod.writediff(changesfile, changes)
 
         def validate(tr2):
             """will run pre-closing hooks"""
             # XXX the transaction API is a bit lacking here so we take a hacky
             # path for now
             #
             # We cannot add this as a "pending" hooks since the 'tr.hookargs'
             # dict is copied before these run. In addition we needs the data
             # available to in memory hooks too.
             #
             # Moreover, we also need to make sure this runs before txnclose
             # hooks and there is no "pending" mechanism that would execute
             # logic only if hooks are about to run.
             #
             # Fixing this limitation of the transaction is also needed to track
             # other families of changes (bookmarks, phases, obsolescence).
             #
             # This will have to be fixed before we remove the experimental
             # gating.
             tracktags(tr2)
             repo = reporef()
 
             singleheadopt = (b'experimental', b'single-head-per-branch')
             singlehead = repo.ui.configbool(*singleheadopt)
             if singlehead:
                 singleheadsub = repo.ui.configsuboptions(*singleheadopt)[1]
                 accountclosed = singleheadsub.get(
                     b"account-closed-heads", False
                 )
                 if singleheadsub.get(b"public-changes-only", False):
                     filtername = b"immutable"
                 else:
                     filtername = b"visible"
                 scmutil.enforcesinglehead(
                     repo, tr2, desc, accountclosed, filtername
                 )
             if hook.hashook(repo.ui, b'pretxnclose-bookmark'):
                 for name, (old, new) in sorted(
                     tr.changes[b'bookmarks'].items()
                 ):
                     args = tr.hookargs.copy()
                     args.update(bookmarks.preparehookargs(name, old, new))
                     repo.hook(
                         b'pretxnclose-bookmark',
                         throw=True,
                         **pycompat.strkwargs(args)
                     )
             if hook.hashook(repo.ui, b'pretxnclose-phase'):
                 cl = repo.unfiltered().changelog
                 for revs, (old, new) in tr.changes[b'phases']:
                     for rev in revs:
                         args = tr.hookargs.copy()
                         node = hex(cl.node(rev))
                         args.update(phases.preparehookargs(node, old, new))
                         repo.hook(
                             b'pretxnclose-phase',
                             throw=True,
                             **pycompat.strkwargs(args)
                         )
 
             repo.hook(
                 b'pretxnclose', throw=True, **pycompat.strkwargs(tr.hookargs)
             )
 
         def releasefn(tr, success):
             repo = reporef()
             if repo is None:
                 # If the repo has been GC'd (and this release function is being
                 # called from transaction.__del__), there's not much we can do,
                 # so just leave the unfinished transaction there and let the
                 # user run `hg recover`.
                 return
             if success:
                 # this should be explicitly invoked here, because
                 # in-memory changes aren't written out at closing
                 # transaction, if tr.addfilegenerator (via
                 # dirstate.write or so) isn't invoked while
                 # transaction running
                 repo.dirstate.write(None)
             else:
                 # discard all changes (including ones already written
                 # out) in this transaction
                 narrowspec.restorebackup(self, b'journal.narrowspec')
                 narrowspec.restorewcbackup(self, b'journal.narrowspec.dirstate')
                 repo.dirstate.restorebackup(None, b'journal.dirstate')
 
                 repo.invalidate(clearfilecache=True)
 
         tr = transaction.transaction(
             rp,
             self.svfs,
             vfsmap,
             b"journal",
             b"undo",
             aftertrans(renames),
             self.store.createmode,
             validator=validate,
             releasefn=releasefn,
             checkambigfiles=_cachedfiles,
             name=desc,
         )
         tr.changes[b'origrepolen'] = len(self)
         tr.changes[b'obsmarkers'] = set()
         tr.changes[b'phases'] = []
         tr.changes[b'bookmarks'] = {}
 
         tr.hookargs[b'txnid'] = txnid
         tr.hookargs[b'txnname'] = desc
         tr.hookargs[b'changes'] = tr.changes
         # note: writing the fncache only during finalize mean that the file is
         # outdated when running hooks. As fncache is used for streaming clone,
         # this is not expected to break anything that happen during the hooks.
         tr.addfinalize(b'flush-fncache', self.store.write)
 
         def txnclosehook(tr2):
             """To be run if transaction is successful, will schedule a hook run"""
             # Don't reference tr2 in hook() so we don't hold a reference.
             # This reduces memory consumption when there are multiple
             # transactions per lock. This can likely go away if issue5045
             # fixes the function accumulation.
             hookargs = tr2.hookargs
 
             def hookfunc(unused_success):
                 repo = reporef()
                 if hook.hashook(repo.ui, b'txnclose-bookmark'):
                     bmchanges = sorted(tr.changes[b'bookmarks'].items())
                     for name, (old, new) in bmchanges:
                         args = tr.hookargs.copy()
                         args.update(bookmarks.preparehookargs(name, old, new))
                         repo.hook(
                             b'txnclose-bookmark',
                             throw=False,
                             **pycompat.strkwargs(args)
                         )
 
                 if hook.hashook(repo.ui, b'txnclose-phase'):
                     cl = repo.unfiltered().changelog
                     phasemv = sorted(
                         tr.changes[b'phases'], key=lambda r: r[0][0]
                     )
                     for revs, (old, new) in phasemv:
                         for rev in revs:
                             args = tr.hookargs.copy()
                             node = hex(cl.node(rev))
                             args.update(phases.preparehookargs(node, old, new))
                             repo.hook(
                                 b'txnclose-phase',
                                 throw=False,
                                 **pycompat.strkwargs(args)
                             )
 
                 repo.hook(
                     b'txnclose', throw=False, **pycompat.strkwargs(hookargs)
                 )
 
             reporef()._afterlock(hookfunc)
 
         tr.addfinalize(b'txnclose-hook', txnclosehook)
         # Include a leading "-" to make it happen before the transaction summary
         # reports registered via scmutil.registersummarycallback() whose names
         # are 00-txnreport etc. That way, the caches will be warm when the
         # callbacks run.
         tr.addpostclose(b'-warm-cache', self._buildcacheupdater(tr))
 
         def txnaborthook(tr2):
             """To be run if transaction is aborted"""
             reporef().hook(
                 b'txnabort', throw=False, **pycompat.strkwargs(tr2.hookargs)
             )
 
         tr.addabort(b'txnabort-hook', txnaborthook)
         # avoid eager cache invalidation. in-memory data should be identical
         # to stored data if transaction has no error.
         tr.addpostclose(b'refresh-filecachestats', self._refreshfilecachestats)
         self._transref = weakref.ref(tr)
         scmutil.registersummarycallback(self, tr, desc)
         return tr
 
     def _journalfiles(self):
         return (
             (self.svfs, b'journal'),
             (self.svfs, b'journal.narrowspec'),
             (self.vfs, b'journal.narrowspec.dirstate'),
             (self.vfs, b'journal.dirstate'),
             (self.vfs, b'journal.branch'),
             (self.vfs, b'journal.desc'),
             (bookmarks.bookmarksvfs(self), b'journal.bookmarks'),
             (self.svfs, b'journal.phaseroots'),
         )
 
     def undofiles(self):
         return [(vfs, undoname(x)) for vfs, x in self._journalfiles()]
 
     @unfilteredmethod
     def _writejournal(self, desc):
         self.dirstate.savebackup(None, b'journal.dirstate')
         narrowspec.savewcbackup(self, b'journal.narrowspec.dirstate')
         narrowspec.savebackup(self, b'journal.narrowspec')
         self.vfs.write(
             b"journal.branch", encoding.fromlocal(self.dirstate.branch())
         )
         self.vfs.write(b"journal.desc", b"%d\n%s\n" % (len(self), desc))
         bookmarksvfs = bookmarks.bookmarksvfs(self)
         bookmarksvfs.write(
             b"journal.bookmarks", bookmarksvfs.tryread(b"bookmarks")
         )
         self.svfs.write(b"journal.phaseroots", self.svfs.tryread(b"phaseroots"))
 
     def recover(self):
         with self.lock():
             if self.svfs.exists(b"journal"):
                 self.ui.status(_(b"rolling back interrupted transaction\n"))
                 vfsmap = {
                     b'': self.svfs,
                     b'plain': self.vfs,
                 }
                 transaction.rollback(
                     self.svfs,
                     vfsmap,
                     b"journal",
                     self.ui.warn,
                     checkambigfiles=_cachedfiles,
                 )
                 self.invalidate()
                 return True
             else:
                 self.ui.warn(_(b"no interrupted transaction available\n"))
                 return False
 
     def rollback(self, dryrun=False, force=False):
         wlock = lock = dsguard = None
         try:
             wlock = self.wlock()
             lock = self.lock()
             if self.svfs.exists(b"undo"):
                 dsguard = dirstateguard.dirstateguard(self, b'rollback')
 
                 return self._rollback(dryrun, force, dsguard)
             else:
                 self.ui.warn(_(b"no rollback information available\n"))
                 return 1
         finally:
             release(dsguard, lock, wlock)
 
     @unfilteredmethod  # Until we get smarter cache management
     def _rollback(self, dryrun, force, dsguard):
         ui = self.ui
         try:
             args = self.vfs.read(b'undo.desc').splitlines()
             (oldlen, desc, detail) = (int(args[0]), args[1], None)
             if len(args) >= 3:
                 detail = args[2]
             oldtip = oldlen - 1
 
             if detail and ui.verbose:
                 msg = _(
                     b'repository tip rolled back to revision %d'
                     b' (undo %s: %s)\n'
                 ) % (oldtip, desc, detail)
             else:
                 msg = _(
                     b'repository tip rolled back to revision %d (undo %s)\n'
                 ) % (oldtip, desc)
         except IOError:
             msg = _(b'rolling back unknown transaction\n')
             desc = None
 
         if not force and self[b'.'] != self[b'tip'] and desc == b'commit':
             raise error.Abort(
                 _(
                     b'rollback of last commit while not checked out '
                     b'may lose data'
                 ),
                 hint=_(b'use -f to force'),
             )
 
         ui.status(msg)
         if dryrun:
             return 0
 
         parents = self.dirstate.parents()
         self.destroying()
         vfsmap = {b'plain': self.vfs, b'': self.svfs}
         transaction.rollback(
             self.svfs, vfsmap, b'undo', ui.warn, checkambigfiles=_cachedfiles
         )
         bookmarksvfs = bookmarks.bookmarksvfs(self)
         if bookmarksvfs.exists(b'undo.bookmarks'):
             bookmarksvfs.rename(
                 b'undo.bookmarks', b'bookmarks', checkambig=True
             )
         if self.svfs.exists(b'undo.phaseroots'):
             self.svfs.rename(b'undo.phaseroots', b'phaseroots', checkambig=True)
         self.invalidate()
 
         has_node = self.changelog.index.has_node
         parentgone = any(not has_node(p) for p in parents)
         if parentgone:
             # prevent dirstateguard from overwriting already restored one
             dsguard.close()
 
             narrowspec.restorebackup(self, b'undo.narrowspec')
             narrowspec.restorewcbackup(self, b'undo.narrowspec.dirstate')
             self.dirstate.restorebackup(None, b'undo.dirstate')
             try:
                 branch = self.vfs.read(b'undo.branch')
                 self.dirstate.setbranch(encoding.tolocal(branch))
             except IOError:
                 ui.warn(
                     _(
                         b'named branch could not be reset: '
                         b'current branch is still \'%s\'\n'
                     )
                     % self.dirstate.branch()
                 )
 
             parents = tuple([p.rev() for p in self[None].parents()])
             if len(parents) > 1:
                 ui.status(
                     _(
                         b'working directory now based on '
                         b'revisions %d and %d\n'
                     )
                     % parents
                 )
             else:
                 ui.status(
                     _(b'working directory now based on revision %d\n') % parents
                 )
             mergestatemod.mergestate.clean(self)
 
         # TODO: if we know which new heads may result from this rollback, pass
         # them to destroy(), which will prevent the branchhead cache from being
         # invalidated.
         self.destroyed()
         return 0
 
     def _buildcacheupdater(self, newtransaction):
         """called during transaction to build the callback updating cache
 
         Lives on the repository to help extension who might want to augment
         this logic. For this purpose, the created transaction is passed to the
         method.
         """
         # we must avoid cyclic reference between repo and transaction.
         reporef = weakref.ref(self)
 
         def updater(tr):
             repo = reporef()
             repo.updatecaches(tr)
 
         return updater
 
     @unfilteredmethod
     def updatecaches(self, tr=None, full=False):
         """warm appropriate caches
 
         If this function is called after a transaction closed. The transaction
         will be available in the 'tr' argument. This can be used to selectively
         update caches relevant to the changes in that transaction.
 
         If 'full' is set, make sure all caches the function knows about have
         up-to-date data. Even the ones usually loaded more lazily.
         """
         if tr is not None and tr.hookargs.get(b'source') == b'strip':
             # During strip, many caches are invalid but
             # later call to `destroyed` will refresh them.
             return
 
         if tr is None or tr.changes[b'origrepolen'] < len(self):
             # accessing the 'served' branchmap should refresh all the others,
             self.ui.debug(b'updating the branch cache\n')
             self.filtered(b'served').branchmap()
             self.filtered(b'served.hidden').branchmap()
 
         if full:
             unfi = self.unfiltered()
 
             self.changelog.update_caches(transaction=tr)
             self.manifestlog.update_caches(transaction=tr)
 
             rbc = unfi.revbranchcache()
             for r in unfi.changelog:
                 rbc.branchinfo(r)
             rbc.write()
 
             # ensure the working copy parents are in the manifestfulltextcache
             for ctx in self[b'.'].parents():
                 ctx.manifest()  # accessing the manifest is enough
 
             # accessing fnode cache warms the cache
             tagsmod.fnoderevs(self.ui, unfi, unfi.changelog.revs())
             # accessing tags warm the cache
             self.tags()
             self.filtered(b'served').tags()
 
             # The `full` arg is documented as updating even the lazily-loaded
             # caches immediately, so we're forcing a write to cause these caches
             # to be warmed up even if they haven't explicitly been requested
             # yet (if they've never been used by hg, they won't ever have been
             # written, even if they're a subset of another kind of cache that
             # *has* been used).
             for filt in repoview.filtertable.keys():
                 filtered = self.filtered(filt)
                 filtered.branchmap().write(filtered)
 
     def invalidatecaches(self):
 
         if '_tagscache' in vars(self):
             # can't use delattr on proxy
             del self.__dict__['_tagscache']
 
         self._branchcaches.clear()
         self.invalidatevolatilesets()
         self._sparsesignaturecache.clear()
 
     def invalidatevolatilesets(self):
         self.filteredrevcache.clear()
         obsolete.clearobscaches(self)
         self._quick_access_changeid_invalidate()
 
     def invalidatedirstate(self):
         """Invalidates the dirstate, causing the next call to dirstate
         to check if it was modified since the last time it was read,
         rereading it if it has.
 
         This is different to dirstate.invalidate() that it doesn't always
         rereads the dirstate. Use dirstate.invalidate() if you want to
         explicitly read the dirstate again (i.e. restoring it to a previous
         known good state)."""
         if hasunfilteredcache(self, 'dirstate'):
             for k in self.dirstate._filecache:
                 try:
                     delattr(self.dirstate, k)
                 except AttributeError:
                     pass
             delattr(self.unfiltered(), 'dirstate')
 
     def invalidate(self, clearfilecache=False):
         """Invalidates both store and non-store parts other than dirstate
 
         If a transaction is running, invalidation of store is omitted,
         because discarding in-memory changes might cause inconsistency
         (e.g. incomplete fncache causes unintentional failure, but
         redundant one doesn't).
         """
         unfiltered = self.unfiltered()  # all file caches are stored unfiltered
         for k in list(self._filecache.keys()):
             # dirstate is invalidated separately in invalidatedirstate()
             if k == b'dirstate':
                 continue
             if (
                 k == b'changelog'
                 and self.currenttransaction()
                 and self.changelog._delayed
             ):
                 # The changelog object may store unwritten revisions. We don't
                 # want to lose them.
                 # TODO: Solve the problem instead of working around it.
                 continue
 
             if clearfilecache:
                 del self._filecache[k]
             try:
                 delattr(unfiltered, k)
             except AttributeError:
                 pass
         self.invalidatecaches()
         if not self.currenttransaction():
             # TODO: Changing contents of store outside transaction
             # causes inconsistency. We should make in-memory store
             # changes detectable, and abort if changed.
             self.store.invalidatecaches()
 
     def invalidateall(self):
         """Fully invalidates both store and non-store parts, causing the
         subsequent operation to reread any outside changes."""
         # extension should hook this to invalidate its caches
         self.invalidate()
         self.invalidatedirstate()
 
     @unfilteredmethod
     def _refreshfilecachestats(self, tr):
         """Reload stats of cached files so that they are flagged as valid"""
         for k, ce in self._filecache.items():
             k = pycompat.sysstr(k)
             if k == 'dirstate' or k not in self.__dict__:
                 continue
             ce.refresh()
 
     def _lock(
         self,
         vfs,
         lockname,
         wait,
         releasefn,
         acquirefn,
         desc,
     ):
         timeout = 0
         warntimeout = 0
         if wait:
             timeout = self.ui.configint(b"ui", b"timeout")
             warntimeout = self.ui.configint(b"ui", b"timeout.warn")
         # internal config: ui.signal-safe-lock
         signalsafe = self.ui.configbool(b'ui', b'signal-safe-lock')
 
         l = lockmod.trylock(
             self.ui,
             vfs,
             lockname,
             timeout,
             warntimeout,
             releasefn=releasefn,
             acquirefn=acquirefn,
             desc=desc,
             signalsafe=signalsafe,
         )
         return l
 
     def _afterlock(self, callback):
         """add a callback to be run when the repository is fully unlocked
 
         The callback will be executed when the outermost lock is released
         (with wlock being higher level than 'lock')."""
         for ref in (self._wlockref, self._lockref):
             l = ref and ref()
             if l and l.held:
                 l.postrelease.append(callback)
                 break
         else:  # no lock have been found.
             callback(True)
 
     def lock(self, wait=True):
         """Lock the repository store (.hg/store) and return a weak reference
         to the lock. Use this before modifying the store (e.g. committing or
         stripping). If you are opening a transaction, get a lock as well.)
 
         If both 'lock' and 'wlock' must be acquired, ensure you always acquires
         'wlock' first to avoid a dead-lock hazard."""
         l = self._currentlock(self._lockref)
         if l is not None:
             l.lock()
             return l
 
         l = self._lock(
             vfs=self.svfs,
             lockname=b"lock",
             wait=wait,
             releasefn=None,
             acquirefn=self.invalidate,
             desc=_(b'repository %s') % self.origroot,
         )
         self._lockref = weakref.ref(l)
         return l
 
     def wlock(self, wait=True):
         """Lock the non-store parts of the repository (everything under
         .hg except .hg/store) and return a weak reference to the lock.
 
         Use this before modifying files in .hg.
 
         If both 'lock' and 'wlock' must be acquired, ensure you always acquires
         'wlock' first to avoid a dead-lock hazard."""
         l = self._wlockref and self._wlockref()
         if l is not None and l.held:
             l.lock()
             return l
 
         # We do not need to check for non-waiting lock acquisition.  Such
         # acquisition would not cause dead-lock as they would just fail.
         if wait and (
             self.ui.configbool(b'devel', b'all-warnings')
             or self.ui.configbool(b'devel', b'check-locks')
         ):
             if self._currentlock(self._lockref) is not None:
                 self.ui.develwarn(b'"wlock" acquired after "lock"')
 
         def unlock():
             if self.dirstate.pendingparentchange():
                 self.dirstate.invalidate()
             else:
                 self.dirstate.write(None)
 
             self._filecache[b'dirstate'].refresh()
 
         l = self._lock(
             self.vfs,
             b"wlock",
             wait,
             unlock,
             self.invalidatedirstate,
             _(b'working directory of %s') % self.origroot,
         )
         self._wlockref = weakref.ref(l)
         return l
 
     def _currentlock(self, lockref):
         """Returns the lock if it's held, or None if it's not."""
         if lockref is None:
             return None
         l = lockref()
         if l is None or not l.held:
             return None
         return l
 
     def currentwlock(self):
         """Returns the wlock if it's held, or None if it's not."""
         return self._currentlock(self._wlockref)
 
     def checkcommitpatterns(self, wctx, match, status, fail):
         """check for commit arguments that aren't committable"""
         if match.isexact() or match.prefix():
             matched = set(status.modified + status.added + status.removed)
 
             for f in match.files():
                 f = self.dirstate.normalize(f)
                 if f == b'.' or f in matched or f in wctx.substate:
                     continue
                 if f in status.deleted:
                     fail(f, _(b'file not found!'))
                 # Is it a directory that exists or used to exist?
                 if self.wvfs.isdir(f) or wctx.p1().hasdir(f):
                     d = f + b'/'
                     for mf in matched:
                         if mf.startswith(d):
                             break
                     else:
                         fail(f, _(b"no match under directory!"))
                 elif f not in self.dirstate:
                     fail(f, _(b"file not tracked!"))
 
     @unfilteredmethod
     def commit(
         self,
         text=b"",
         user=None,
         date=None,
         match=None,
         force=False,
         editor=None,
         extra=None,
     ):
         """Add a new revision to current repository.
 
         Revision information is gathered from the working directory,
         match can be used to filter the committed files. If editor is
         supplied, it is called to get a commit message.
         """
         if extra is None:
             extra = {}
 
         def fail(f, msg):
             raise error.InputError(b'%s: %s' % (f, msg))
 
         if not match:
             match = matchmod.always()
 
         if not force:
             match.bad = fail
 
         # lock() for recent changelog (see issue4368)
         with self.wlock(), self.lock():
             wctx = self[None]
             merge = len(wctx.parents()) > 1
 
             if not force and merge and not match.always():
                 raise error.Abort(
                     _(
                         b'cannot partially commit a merge '
                         b'(do not specify files or patterns)'
                     )
                 )
 
             status = self.status(match=match, clean=force)
             if force:
                 status.modified.extend(
                     status.clean
                 )  # mq may commit clean files
 
             # check subrepos
             subs, commitsubs, newstate = subrepoutil.precommit(
                 self.ui, wctx, status, match, force=force
             )
 
             # make sure all explicit patterns are matched
             if not force:
                 self.checkcommitpatterns(wctx, match, status, fail)
 
             cctx = context.workingcommitctx(
                 self, status, text, user, date, extra
             )
 
             ms = mergestatemod.mergestate.read(self)
             mergeutil.checkunresolved(ms)
 
             # internal config: ui.allowemptycommit
             if cctx.isempty() and not self.ui.configbool(
                 b'ui', b'allowemptycommit'
             ):
                 self.ui.debug(b'nothing to commit, clearing merge state\n')
                 ms.reset()
                 return None
 
             if merge and cctx.deleted():
                 raise error.Abort(_(b"cannot commit merge with missing files"))
 
             if editor:
                 cctx._text = editor(self, cctx, subs)
             edited = text != cctx._text
 
             # Save commit message in case this transaction gets rolled back
             # (e.g. by a pretxncommit hook).  Leave the content alone on
             # the assumption that the user will use the same editor again.
             msgfn = self.savecommitmessage(cctx._text)
 
             # commit subs and write new state
             if subs:
                 uipathfn = scmutil.getuipathfn(self)
                 for s in sorted(commitsubs):
                     sub = wctx.sub(s)
                     self.ui.status(
                         _(b'committing subrepository %s\n')
                         % uipathfn(subrepoutil.subrelpath(sub))
                     )
                     sr = sub.commit(cctx._text, user, date)
                     newstate[s] = (newstate[s][0], sr)
                 subrepoutil.writestate(self, newstate)
 
             p1, p2 = self.dirstate.parents()
             hookp1, hookp2 = hex(p1), (p2 != nullid and hex(p2) or b'')
             try:
                 self.hook(
                     b"precommit", throw=True, parent1=hookp1, parent2=hookp2
                 )
                 with self.transaction(b'commit'):
                     ret = self.commitctx(cctx, True)
                     # update bookmarks, dirstate and mergestate
                     bookmarks.update(self, [p1, p2], ret)
                     cctx.markcommitted(ret)
                     ms.reset()
             except:  # re-raises
                 if edited:
                     self.ui.write(
                         _(b'note: commit message saved in %s\n') % msgfn
                     )
                     self.ui.write(
                         _(
                             b"note: use 'hg commit --logfile "
                             b".hg/last-message.txt --edit' to reuse it\n"
                         )
                     )
                 raise
 
         def commithook(unused_success):
             # hack for command that use a temporary commit (eg: histedit)
             # temporary commit got stripped before hook release
             if self.changelog.hasnode(ret):
                 self.hook(
                     b"commit", node=hex(ret), parent1=hookp1, parent2=hookp2
                 )
 
         self._afterlock(commithook)
         return ret
 
     @unfilteredmethod
     def commitctx(self, ctx, error=False, origctx=None):
         return commit.commitctx(self, ctx, error=error, origctx=origctx)
 
     @unfilteredmethod
     def destroying(self):
         """Inform the repository that nodes are about to be destroyed.
         Intended for use by strip and rollback, so there's a common
         place for anything that has to be done before destroying history.
 
         This is mostly useful for saving state that is in memory and waiting
         to be flushed when the current lock is released. Because a call to
         destroyed is imminent, the repo will be invalidated causing those
         changes to stay in memory (waiting for the next unlock), or vanish
         completely.
         """
         # When using the same lock to commit and strip, the phasecache is left
         # dirty after committing. Then when we strip, the repo is invalidated,
         # causing those changes to disappear.
         if '_phasecache' in vars(self):
             self._phasecache.write()
 
     @unfilteredmethod
     def destroyed(self):
         """Inform the repository that nodes have been destroyed.
         Intended for use by strip and rollback, so there's a common
         place for anything that has to be done after destroying history.
         """
         # When one tries to:
         # 1) destroy nodes thus calling this method (e.g. strip)
         # 2) use phasecache somewhere (e.g. commit)
         #
         # then 2) will fail because the phasecache contains nodes that were
         # removed. We can either remove phasecache from the filecache,
         # causing it to reload next time it is accessed, or simply filter
         # the removed nodes now and write the updated cache.
         self._phasecache.filterunknown(self)
         self._phasecache.write()
 
         # refresh all repository caches
         self.updatecaches()
 
         # Ensure the persistent tag cache is updated.  Doing it now
         # means that the tag cache only has to worry about destroyed
         # heads immediately after a strip/rollback.  That in turn
         # guarantees that "cachetip == currenttip" (comparing both rev
         # and node) always means no nodes have been added or destroyed.
 
         # XXX this is suboptimal when qrefresh'ing: we strip the current
         # head, refresh the tag cache, then immediately add a new head.
         # But I think doing it this way is necessary for the "instant
         # tag cache retrieval" case to work.
         self.invalidate()
 
     def status(
         self,
         node1=b'.',
         node2=None,
         match=None,
         ignored=False,
         clean=False,
         unknown=False,
         listsubrepos=False,
     ):
         '''a convenience method that calls node1.status(node2)'''
         return self[node1].status(
             node2, match, ignored, clean, unknown, listsubrepos
         )
 
     def addpostdsstatus(self, ps):
         """Add a callback to run within the wlock, at the point at which status
         fixups happen.
 
         On status completion, callback(wctx, status) will be called with the
         wlock held, unless the dirstate has changed from underneath or the wlock
         couldn't be grabbed.
 
         Callbacks should not capture and use a cached copy of the dirstate --
         it might change in the meanwhile. Instead, they should access the
         dirstate via wctx.repo().dirstate.
 
         This list is emptied out after each status run -- extensions should
         make sure it adds to this list each time dirstate.status is called.
         Extensions should also make sure they don't call this for statuses
         that don't involve the dirstate.
         """
 
         # The list is located here for uniqueness reasons -- it is actually
         # managed by the workingctx, but that isn't unique per-repo.
         self._postdsstatus.append(ps)
 
     def postdsstatus(self):
         """Used by workingctx to get the list of post-dirstate-status hooks."""
         return self._postdsstatus
 
     def clearpostdsstatus(self):
         """Used by workingctx to clear post-dirstate-status hooks."""
         del self._postdsstatus[:]
 
     def heads(self, start=None):
         if start is None:
             cl = self.changelog
             headrevs = reversed(cl.headrevs())
             return [cl.node(rev) for rev in headrevs]
 
         heads = self.changelog.heads(start)
         # sort the output in rev descending order
         return sorted(heads, key=self.changelog.rev, reverse=True)
 
     def branchheads(self, branch=None, start=None, closed=False):
         """return a (possibly filtered) list of heads for the given branch
 
         Heads are returned in topological order, from newest to oldest.
         If branch is None, use the dirstate branch.
         If start is not None, return only heads reachable from start.
         If closed is True, return heads that are marked as closed as well.
         """
         if branch is None:
             branch = self[None].branch()
         branches = self.branchmap()
         if not branches.hasbranch(branch):
             return []
         # the cache returns heads ordered lowest to highest
         bheads = list(reversed(branches.branchheads(branch, closed=closed)))
         if start is not None:
             # filter out the heads that cannot be reached from startrev
             fbheads = set(self.changelog.nodesbetween([start], bheads)[2])
             bheads = [h for h in bheads if h in fbheads]
         return bheads
 
     def branches(self, nodes):
         if not nodes:
             nodes = [self.changelog.tip()]
         b = []
         for n in nodes:
             t = n
             while True:
                 p = self.changelog.parents(n)
                 if p[1] != nullid or p[0] == nullid:
                     b.append((t, n, p[0], p[1]))
                     break
                 n = p[0]
         return b
 
     def between(self, pairs):
         r = []
 
         for top, bottom in pairs:
             n, l, i = top, [], 0
             f = 1
 
             while n != bottom and n != nullid:
                 p = self.changelog.parents(n)[0]
                 if i == f:
                     l.append(n)
                     f = f * 2
                 n = p
                 i += 1
 
             r.append(l)
 
         return r
 
     def checkpush(self, pushop):
         """Extensions can override this function if additional checks have
         to be performed before pushing, or call it if they override push
         command.
         """
 
     @unfilteredpropertycache
     def prepushoutgoinghooks(self):
         """Return util.hooks consists of a pushop with repo, remote, outgoing
         methods, which are called before pushing changesets.
         """
         return util.hooks()
 
     def pushkey(self, namespace, key, old, new):
         try:
             tr = self.currenttransaction()
             hookargs = {}
             if tr is not None:
                 hookargs.update(tr.hookargs)
             hookargs = pycompat.strkwargs(hookargs)
             hookargs['namespace'] = namespace
             hookargs['key'] = key
             hookargs['old'] = old
             hookargs['new'] = new
             self.hook(b'prepushkey', throw=True, **hookargs)
         except error.HookAbort as exc:
             self.ui.write_err(_(b"pushkey-abort: %s\n") % exc)
             if exc.hint:
                 self.ui.write_err(_(b"(%s)\n") % exc.hint)
             return False
         self.ui.debug(b'pushing key for "%s:%s"\n' % (namespace, key))
         ret = pushkey.push(self, namespace, key, old, new)
 
         def runhook(unused_success):
             self.hook(
                 b'pushkey',
                 namespace=namespace,
                 key=key,
                 old=old,
                 new=new,
                 ret=ret,
             )
 
         self._afterlock(runhook)
         return ret
 
     def listkeys(self, namespace):
         self.hook(b'prelistkeys', throw=True, namespace=namespace)
         self.ui.debug(b'listing keys for "%s"\n' % namespace)
         values = pushkey.list(self, namespace)
         self.hook(b'listkeys', namespace=namespace, values=values)
         return values
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         '''used to test argument passing over the wire'''
         return b"%s %s %s %s %s" % (
             one,
             two,
             pycompat.bytestr(three),
             pycompat.bytestr(four),
             pycompat.bytestr(five),
         )
 
     def savecommitmessage(self, text):
         fp = self.vfs(b'last-message.txt', b'wb')
         try:
             fp.write(text)
         finally:
             fp.close()
         return self.pathto(fp.name[len(self.root) + 1 :])
 
 
 # used to avoid circular references so destructors work
 def aftertrans(files):
     renamefiles = [tuple(t) for t in files]
 
     def a():
         for vfs, src, dest in renamefiles:
             # if src and dest refer to a same file, vfs.rename is a no-op,
             # leaving both src and dest on disk. delete dest to make sure
             # the rename couldn't be such a no-op.
             vfs.tryunlink(dest)
             try:
                 vfs.rename(src, dest)
             except OSError:  # journal file does not yet exist
                 pass
 
     return a
 
 
 def undoname(fn):
     base, name = os.path.split(fn)
     assert name.startswith(b'journal')
     return os.path.join(base, name.replace(b'journal', b'undo', 1))
 
 
 def instance(ui, path, create, intents=None, createopts=None):
     localpath = util.urllocalpath(path)
     if create:
         createrepository(ui, localpath, createopts=createopts)
 
     return makelocalrepository(ui, localpath, intents=intents)
 
 
 def islocal(path):
     return True
 
 
 def defaultcreateopts(ui, createopts=None):
     """Populate the default creation options for a repository.
 
     A dictionary of explicitly requested creation options can be passed
     in. Missing keys will be populated.
     """
     createopts = dict(createopts or {})
 
     if b'backend' not in createopts:
         # experimental config: storage.new-repo-backend
         createopts[b'backend'] = ui.config(b'storage', b'new-repo-backend')
 
     return createopts
 
 
 def newreporequirements(ui, createopts):
     """Determine the set of requirements for a new local repository.
 
     Extensions can wrap this function to specify custom requirements for
     new repositories.
     """
     # If the repo is being created from a shared repository, we copy
     # its requirements.
     if b'sharedrepo' in createopts:
         requirements = set(createopts[b'sharedrepo'].requirements)
         if createopts.get(b'sharedrelative'):
             requirements.add(requirementsmod.RELATIVE_SHARED_REQUIREMENT)
         else:
             requirements.add(requirementsmod.SHARED_REQUIREMENT)
 
         return requirements
 
     if b'backend' not in createopts:
         raise error.ProgrammingError(
             b'backend key not present in createopts; '
             b'was defaultcreateopts() called?'
         )
 
     if createopts[b'backend'] != b'revlogv1':
         raise error.Abort(
             _(
                 b'unable to determine repository requirements for '
                 b'storage backend: %s'
             )
             % createopts[b'backend']
         )
 
     requirements = {requirementsmod.REVLOGV1_REQUIREMENT}
     if ui.configbool(b'format', b'usestore'):
         requirements.add(requirementsmod.STORE_REQUIREMENT)
         if ui.configbool(b'format', b'usefncache'):
             requirements.add(requirementsmod.FNCACHE_REQUIREMENT)
             if ui.configbool(b'format', b'dotencode'):
                 requirements.add(requirementsmod.DOTENCODE_REQUIREMENT)
 
     compengines = ui.configlist(b'format', b'revlog-compression')
     for compengine in compengines:
         if compengine in util.compengines:
             break
     else:
         raise error.Abort(
             _(
                 b'compression engines %s defined by '
                 b'format.revlog-compression not available'
             )
             % b', '.join(b'"%s"' % e for e in compengines),
             hint=_(
                 b'run "hg debuginstall" to list available '
                 b'compression engines'
             ),
         )
 
     # zlib is the historical default and doesn't need an explicit requirement.
     if compengine == b'zstd':
         requirements.add(b'revlog-compression-zstd')
     elif compengine != b'zlib':
         requirements.add(b'exp-compression-%s' % compengine)
 
     if scmutil.gdinitconfig(ui):
         requirements.add(requirementsmod.GENERALDELTA_REQUIREMENT)
         if ui.configbool(b'format', b'sparse-revlog'):
             requirements.add(requirementsmod.SPARSEREVLOG_REQUIREMENT)
 
     # experimental config: format.exp-use-side-data
     if ui.configbool(b'format', b'exp-use-side-data'):
         requirements.discard(requirementsmod.REVLOGV1_REQUIREMENT)
         requirements.add(requirementsmod.REVLOGV2_REQUIREMENT)
         requirements.add(requirementsmod.SIDEDATA_REQUIREMENT)
     # experimental config: format.exp-use-copies-side-data-changeset
     if ui.configbool(b'format', b'exp-use-copies-side-data-changeset'):
         requirements.discard(requirementsmod.REVLOGV1_REQUIREMENT)
         requirements.add(requirementsmod.REVLOGV2_REQUIREMENT)
         requirements.add(requirementsmod.SIDEDATA_REQUIREMENT)
         requirements.add(requirementsmod.COPIESSDC_REQUIREMENT)
     if ui.configbool(b'experimental', b'treemanifest'):
         requirements.add(requirementsmod.TREEMANIFEST_REQUIREMENT)
 
     revlogv2 = ui.config(b'experimental', b'revlogv2')
     if revlogv2 == b'enable-unstable-format-and-corrupt-my-data':
         requirements.discard(requirementsmod.REVLOGV1_REQUIREMENT)
         # generaldelta is implied by revlogv2.
         requirements.discard(requirementsmod.GENERALDELTA_REQUIREMENT)
         requirements.add(requirementsmod.REVLOGV2_REQUIREMENT)
     # experimental config: format.internal-phase
     if ui.configbool(b'format', b'internal-phase'):
         requirements.add(requirementsmod.INTERNAL_PHASE_REQUIREMENT)
 
     if createopts.get(b'narrowfiles'):
         requirements.add(requirementsmod.NARROW_REQUIREMENT)
 
     if createopts.get(b'lfs'):
         requirements.add(b'lfs')
 
     if ui.configbool(b'format', b'bookmarks-in-store'):
         requirements.add(bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT)
 
     if ui.configbool(b'format', b'use-persistent-nodemap'):
         requirements.add(requirementsmod.NODEMAP_REQUIREMENT)
 
     # if share-safe is enabled, let's create the new repository with the new
     # requirement
     if ui.configbool(b'format', b'use-share-safe'):
         requirements.add(requirementsmod.SHARESAFE_REQUIREMENT)
 
     return requirements
 
 
 def checkrequirementscompat(ui, requirements):
     """Checks compatibility of repository requirements enabled and disabled.
 
     Returns a set of requirements which needs to be dropped because dependend
     requirements are not enabled. Also warns users about it"""
 
     dropped = set()
 
     if requirementsmod.STORE_REQUIREMENT not in requirements:
         if bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT in requirements:
             ui.warn(
                 _(
                     b'ignoring enabled \'format.bookmarks-in-store\' config '
                     b'beacuse it is incompatible with disabled '
                     b'\'format.usestore\' config\n'
                 )
             )
             dropped.add(bookmarks.BOOKMARKS_IN_STORE_REQUIREMENT)
 
         if (
             requirementsmod.SHARED_REQUIREMENT in requirements
             or requirementsmod.RELATIVE_SHARED_REQUIREMENT in requirements
         ):
             raise error.Abort(
                 _(
                     b"cannot create shared repository as source was created"
                     b" with 'format.usestore' config disabled"
                 )
             )
 
         if requirementsmod.SHARESAFE_REQUIREMENT in requirements:
             ui.warn(
                 _(
                     b"ignoring enabled 'format.use-share-safe' config because "
                     b"it is incompatible with disabled 'format.usestore'"
                     b" config\n"
                 )
             )
             dropped.add(requirementsmod.SHARESAFE_REQUIREMENT)
 
     return dropped
 
 
 def filterknowncreateopts(ui, createopts):
     """Filters a dict of repo creation options against options that are known.
 
     Receives a dict of repo creation options and returns a dict of those
     options that we don't know how to handle.
 
     This function is called as part of repository creation. If the
     returned dict contains any items, repository creation will not
     be allowed, as it means there was a request to create a repository
     with options not recognized by loaded code.
 
     Extensions can wrap this function to filter out creation options
     they know how to handle.
     """
     known = {
         b'backend',
         b'lfs',
         b'narrowfiles',
         b'sharedrepo',
         b'sharedrelative',
         b'shareditems',
         b'shallowfilestore',
     }
 
     return {k: v for k, v in createopts.items() if k not in known}
 
 
 def createrepository(ui, path, createopts=None):
     """Create a new repository in a vfs.
 
     ``path`` path to the new repo's working directory.
     ``createopts`` options for the new repository.
 
     The following keys for ``createopts`` are recognized:
 
     backend
        The storage backend to use.
     lfs
        Repository will be created with ``lfs`` requirement. The lfs extension
        will automatically be loaded when the repository is accessed.
     narrowfiles
        Set up repository to support narrow file storage.
     sharedrepo
        Repository object from which storage should be shared.
     sharedrelative
        Boolean indicating if the path to the shared repo should be
        stored as relative. By default, the pointer to the "parent" repo
        is stored as an absolute path.
     shareditems
        Set of items to share to the new repository (in addition to storage).
     shallowfilestore
        Indicates that storage for files should be shallow (not all ancestor
        revisions are known).
     """
     createopts = defaultcreateopts(ui, createopts=createopts)
 
     unknownopts = filterknowncreateopts(ui, createopts)
 
     if not isinstance(unknownopts, dict):
         raise error.ProgrammingError(
             b'filterknowncreateopts() did not return a dict'
         )
 
     if unknownopts:
         raise error.Abort(
             _(
                 b'unable to create repository because of unknown '
                 b'creation option: %s'
             )
             % b', '.join(sorted(unknownopts)),
             hint=_(b'is a required extension not loaded?'),
         )
 
     requirements = newreporequirements(ui, createopts=createopts)
     requirements -= checkrequirementscompat(ui, requirements)
 
     wdirvfs = vfsmod.vfs(path, expandpath=True, realpath=True)
 
     hgvfs = vfsmod.vfs(wdirvfs.join(b'.hg'))
     if hgvfs.exists():
         raise error.RepoError(_(b'repository %s already exists') % path)
 
     if b'sharedrepo' in createopts:
         sharedpath = createopts[b'sharedrepo'].sharedpath
 
         if createopts.get(b'sharedrelative'):
             try:
                 sharedpath = os.path.relpath(sharedpath, hgvfs.base)
             except (IOError, ValueError) as e:
                 # ValueError is raised on Windows if the drive letters differ
                 # on each path.
                 raise error.Abort(
                     _(b'cannot calculate relative path'),
                     hint=stringutil.forcebytestr(e),
                 )
 
     if not wdirvfs.exists():
         wdirvfs.makedirs()
 
     hgvfs.makedir(notindexed=True)
     if b'sharedrepo' not in createopts:
         hgvfs.mkdir(b'cache')
     hgvfs.mkdir(b'wcache')
 
     has_store = requirementsmod.STORE_REQUIREMENT in requirements
     if has_store and b'sharedrepo' not in createopts:
         hgvfs.mkdir(b'store')
 
         # We create an invalid changelog outside the store so very old
         # Mercurial versions (which didn't know about the requirements
         # file) encounter an error on reading the changelog. This
         # effectively locks out old clients and prevents them from
         # mucking with a repo in an unknown format.
         #
         # The revlog header has version 65535, which won't be recognized by
         # such old clients.
         hgvfs.append(
             b'00changelog.i',
             b'\0\0\xFF\xFF dummy changelog to prevent using the old repo '
             b'layout',
         )
 
     # Filter the requirements into working copy and store ones
     wcreq, storereq = scmutil.filterrequirements(requirements)
     # write working copy ones
     scmutil.writerequires(hgvfs, wcreq)
     # If there are store requirements and the current repository
     # is not a shared one, write stored requirements
     # For new shared repository, we don't need to write the store
     # requirements as they are already present in store requires
     if storereq and b'sharedrepo' not in createopts:
         storevfs = vfsmod.vfs(hgvfs.join(b'store'), cacheaudited=True)
         scmutil.writerequires(storevfs, storereq)
 
     # Write out file telling readers where to find the shared store.
     if b'sharedrepo' in createopts:
         hgvfs.write(b'sharedpath', sharedpath)
 
     if createopts.get(b'shareditems'):
         shared = b'\n'.join(sorted(createopts[b'shareditems'])) + b'\n'
         hgvfs.write(b'shared', shared)
 
 
 def poisonrepository(repo):
     """Poison a repository instance so it can no longer be used."""
     # Perform any cleanup on the instance.
     repo.close()
 
     # Our strategy is to replace the type of the object with one that
     # has all attribute lookups result in error.
     #
     # But we have to allow the close() method because some constructors
     # of repos call close() on repo references.
     class poisonedrepository(object):
         def __getattribute__(self, item):
             if item == 'close':
                 return object.__getattribute__(self, item)
 
             raise error.ProgrammingError(
                 b'repo instances should not be used after unshare'
             )
 
         def close(self):
             pass
 
     # We may have a repoview, which intercepts __setattr__. So be sure
     # we operate at the lowest level possible.
     object.__setattr__(repo, '__class__', poisonedrepository)

mercurial/revlog.py

@@ -1,3199 +1,3199 b''
 # revlog.py - storage back-end for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Storage back-end for Mercurial.
 
 This provides efficient delta storage with O(1) retrieve and append
 and O(changes) merge between branches.
 """
 
 from __future__ import absolute_import
 
 import collections
 import contextlib
 import errno
 import io
 import os
 import struct
 import zlib
 
 # import stuff from node for others to import from revlog
 from .node import (
     bin,
     hex,
     nullhex,
     nullid,
     nullrev,
     short,
     wdirfilenodeids,
     wdirhex,
     wdirid,
     wdirrev,
 )
 from .i18n import _
 from .pycompat import getattr
 from .revlogutils.constants import (
     FLAG_GENERALDELTA,
     FLAG_INLINE_DATA,
     REVLOGV0,
     REVLOGV1,
     REVLOGV1_FLAGS,
     REVLOGV2,
     REVLOGV2_FLAGS,
     REVLOG_DEFAULT_FLAGS,
     REVLOG_DEFAULT_FORMAT,
     REVLOG_DEFAULT_VERSION,
 )
 from .revlogutils.flagutil import (
     REVIDX_DEFAULT_FLAGS,
     REVIDX_ELLIPSIS,
     REVIDX_EXTSTORED,
     REVIDX_FLAGS_ORDER,
     REVIDX_HASCOPIESINFO,
     REVIDX_ISCENSORED,
     REVIDX_RAWTEXT_CHANGING_FLAGS,
     REVIDX_SIDEDATA,
 )
 from .thirdparty import attr
 from . import (
     ancestor,
     dagop,
     error,
     mdiff,
     policy,
     pycompat,
     templatefilters,
     util,
 )
 from .interfaces import (
     repository,
     util as interfaceutil,
 )
 from .revlogutils import (
     deltas as deltautil,
     flagutil,
     nodemap as nodemaputil,
     sidedata as sidedatautil,
 )
 from .utils import (
     storageutil,
     stringutil,
 )
 from .pure import parsers as pureparsers
 
 # blanked usage of all the name to prevent pyflakes constraints
 # We need these name available in the module for extensions.
 REVLOGV0
 REVLOGV1
 REVLOGV2
 FLAG_INLINE_DATA
 FLAG_GENERALDELTA
 REVLOG_DEFAULT_FLAGS
 REVLOG_DEFAULT_FORMAT
 REVLOG_DEFAULT_VERSION
 REVLOGV1_FLAGS
 REVLOGV2_FLAGS
 REVIDX_ISCENSORED
 REVIDX_ELLIPSIS
 REVIDX_SIDEDATA
 REVIDX_HASCOPIESINFO
 REVIDX_EXTSTORED
 REVIDX_DEFAULT_FLAGS
 REVIDX_FLAGS_ORDER
 REVIDX_RAWTEXT_CHANGING_FLAGS
 
 parsers = policy.importmod('parsers')
 rustancestor = policy.importrust('ancestor')
 rustdagop = policy.importrust('dagop')
 rustrevlog = policy.importrust('revlog')
 
 # Aliased for performance.
 _zlibdecompress = zlib.decompress
 
 # max size of revlog with inline data
 _maxinline = 131072
 _chunksize = 1048576
 
 # Flag processors for REVIDX_ELLIPSIS.
 def ellipsisreadprocessor(rl, text):
     return text, False
 
 
 def ellipsiswriteprocessor(rl, text):
     return text, False
 
 
 def ellipsisrawprocessor(rl, text):
     return False
 
 
 ellipsisprocessor = (
     ellipsisreadprocessor,
     ellipsiswriteprocessor,
     ellipsisrawprocessor,
 )
 
 
 def getoffset(q):
     return int(q >> 16)
 
 
 def gettype(q):
     return int(q & 0xFFFF)
 
 
 def offset_type(offset, type):
     if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
         raise ValueError(b'unknown revlog index flags')
     return int(int(offset) << 16 | type)
 
 
 def _verify_revision(rl, skipflags, state, node):
     """Verify the integrity of the given revlog ``node`` while providing a hook
     point for extensions to influence the operation."""
     if skipflags:
         state[b'skipread'].add(node)
     else:
         # Side-effect: read content and verify hash.
         rl.revision(node)
 
 
 # True if a fast implementation for persistent-nodemap is available
 #
 # We also consider we have a "fast" implementation in "pure" python because
 # people using pure don't really have performance consideration (and a
 # wheelbarrow of other slowness source)
 HAS_FAST_PERSISTENT_NODEMAP = rustrevlog is not None or util.safehasattr(
     parsers, 'BaseIndexObject'
 )
 
 
 @attr.s(slots=True, frozen=True)
 class _revisioninfo(object):
     """Information about a revision that allows building its fulltext
     node:       expected hash of the revision
     p1, p2:     parent revs of the revision
     btext:      built text cache consisting of a one-element list
     cachedelta: (baserev, uncompressed_delta) or None
     flags:      flags associated to the revision storage
 
     One of btext[0] or cachedelta must be set.
     """
 
     node = attr.ib()
     p1 = attr.ib()
     p2 = attr.ib()
     btext = attr.ib()
     textlen = attr.ib()
     cachedelta = attr.ib()
     flags = attr.ib()
 
 
 @interfaceutil.implementer(repository.irevisiondelta)
 @attr.s(slots=True)
 class revlogrevisiondelta(object):
     node = attr.ib()
     p1node = attr.ib()
     p2node = attr.ib()
     basenode = attr.ib()
     flags = attr.ib()
     baserevisionsize = attr.ib()
     revision = attr.ib()
     delta = attr.ib()
     linknode = attr.ib(default=None)
 
 
 @interfaceutil.implementer(repository.iverifyproblem)
 @attr.s(frozen=True)
 class revlogproblem(object):
     warning = attr.ib(default=None)
     error = attr.ib(default=None)
     node = attr.ib(default=None)
 
 
 # index v0:
 #  4 bytes: offset
 #  4 bytes: compressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 # 20 bytes: parent 1 nodeid
 # 20 bytes: parent 2 nodeid
 # 20 bytes: nodeid
 indexformatv0 = struct.Struct(b">4l20s20s20s")
 indexformatv0_pack = indexformatv0.pack
 indexformatv0_unpack = indexformatv0.unpack
 
 
 class revlogoldindex(list):
     @property
     def nodemap(self):
         msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
         util.nouideprecwarn(msg, b'5.3', stacklevel=2)
         return self._nodemap
 
     @util.propertycache
     def _nodemap(self):
         nodemap = nodemaputil.NodeMap({nullid: nullrev})
         for r in range(0, len(self)):
             n = self[r][7]
             nodemap[n] = r
         return nodemap
 
     def has_node(self, node):
         """return True if the node exist in the index"""
         return node in self._nodemap
 
     def rev(self, node):
         """return a revision for a node
 
         If the node is unknown, raise a RevlogError"""
         return self._nodemap[node]
 
     def get_rev(self, node):
         """return a revision for a node
 
         If the node is unknown, return None"""
         return self._nodemap.get(node)
 
     def append(self, tup):
         self._nodemap[tup[7]] = len(self)
         super(revlogoldindex, self).append(tup)
 
     def __delitem__(self, i):
         if not isinstance(i, slice) or not i.stop == -1 or i.step is not None:
             raise ValueError(b"deleting slices only supports a:-1 with step 1")
         for r in pycompat.xrange(i.start, len(self)):
             del self._nodemap[self[r][7]]
         super(revlogoldindex, self).__delitem__(i)
 
     def clearcaches(self):
         self.__dict__.pop('_nodemap', None)
 
     def __getitem__(self, i):
         if i == -1:
             return (0, 0, 0, -1, -1, -1, -1, nullid)
         return list.__getitem__(self, i)
 
 
 class revlogoldio(object):
     def __init__(self):
         self.size = indexformatv0.size
 
     def parseindex(self, data, inline):
         s = self.size
         index = []
         nodemap = nodemaputil.NodeMap({nullid: nullrev})
         n = off = 0
         l = len(data)
         while off + s <= l:
             cur = data[off : off + s]
             off += s
             e = indexformatv0_unpack(cur)
             # transform to revlogv1 format
             e2 = (
                 offset_type(e[0], 0),
                 e[1],
                 -1,
                 e[2],
                 e[3],
                 nodemap.get(e[4], nullrev),
                 nodemap.get(e[5], nullrev),
                 e[6],
             )
             index.append(e2)
             nodemap[e[6]] = n
             n += 1
 
         index = revlogoldindex(index)
         return index, None
 
     def packentry(self, entry, node, version, rev):
         if gettype(entry[0]):
             raise error.RevlogError(
                 _(b'index entry flags need revlog version 1')
             )
         e2 = (
             getoffset(entry[0]),
             entry[1],
             entry[3],
             entry[4],
             node(entry[5]),
             node(entry[6]),
             entry[7],
         )
         return indexformatv0_pack(*e2)
 
 
 # index ng:
 #  6 bytes: offset
 #  2 bytes: flags
 #  4 bytes: compressed length
 #  4 bytes: uncompressed length
 #  4 bytes: base rev
 #  4 bytes: link rev
 #  4 bytes: parent 1 rev
 #  4 bytes: parent 2 rev
 # 32 bytes: nodeid
 indexformatng = struct.Struct(b">Qiiiiii20s12x")
 indexformatng_pack = indexformatng.pack
 versionformat = struct.Struct(b">I")
 versionformat_pack = versionformat.pack
 versionformat_unpack = versionformat.unpack
 
 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
 # signed integer)
 _maxentrysize = 0x7FFFFFFF
 
 
 class revlogio(object):
     def __init__(self):
         self.size = indexformatng.size
 
     def parseindex(self, data, inline):
         # call the C implementation to parse the index data
         index, cache = parsers.parse_index2(data, inline)
         return index, cache
 
     def packentry(self, entry, node, version, rev):
         p = indexformatng_pack(*entry)
         if rev == 0:
             p = versionformat_pack(version) + p[4:]
         return p
 
 
 indexformatv2 = struct.Struct(pureparsers.Index2Mixin.index_format)
 indexformatv2_pack = indexformatv2.pack
 
 
 class revlogv2io(object):
     def __init__(self):
         self.size = indexformatv2.size
 
     def parseindex(self, data, inline):
         index, cache = parsers.parse_index2(data, inline, revlogv2=True)
         return index, cache
 
     def packentry(self, entry, node, version, rev):
         p = indexformatv2_pack(*entry)
         if rev == 0:
             p = versionformat_pack(version) + p[4:]
         return p
 
 
 NodemapRevlogIO = None
 
 if util.safehasattr(parsers, 'parse_index_devel_nodemap'):
 
     class NodemapRevlogIO(revlogio):
         """A debug oriented IO class that return a PersistentNodeMapIndexObject
 
         The PersistentNodeMapIndexObject object is meant to test the persistent nodemap feature.
         """
 
         def parseindex(self, data, inline):
             index, cache = parsers.parse_index_devel_nodemap(data, inline)
             return index, cache
 
 
 class rustrevlogio(revlogio):
     def parseindex(self, data, inline):
         index, cache = super(rustrevlogio, self).parseindex(data, inline)
         return rustrevlog.MixedIndex(index), cache
 
 
 class revlog(object):
     """
     the underlying revision storage object
 
     A revlog consists of two parts, an index and the revision data.
 
     The index is a file with a fixed record size containing
     information on each revision, including its nodeid (hash), the
     nodeids of its parents, the position and offset of its data within
     the data file, and the revision it's based on. Finally, each entry
     contains a linkrev entry that can serve as a pointer to external
     data.
 
     The revision data itself is a linear collection of data chunks.
     Each chunk represents a revision and is usually represented as a
     delta against the previous chunk. To bound lookup time, runs of
     deltas are limited to about 2 times the length of the original
     version data. This makes retrieval of a version proportional to
     its size, or O(1) relative to the number of revisions.
 
     Both pieces of the revlog are written to in an append-only
     fashion, which means we never need to rewrite a file to insert or
     remove data, and can use some simple techniques to avoid the need
     for locking while reading.
 
     If checkambig, indexfile is opened with checkambig=True at
     writing, to avoid file stat ambiguity.
 
     If mmaplargeindex is True, and an mmapindexthreshold is set, the
     index will be mmapped rather than read if it is larger than the
     configured threshold.
 
     If censorable is True, the revlog can have censored revisions.
 
     If `upperboundcomp` is not None, this is the expected maximal gain from
     compression for the data content.
 
     `concurrencychecker` is an optional function that receives 3 arguments: a
     file handle, a filename, and an expected position. It should check whether
     the current position in the file handle is valid, and log/warn/fail (by
     raising).
     """
 
     _flagserrorclass = error.RevlogError
 
     def __init__(
         self,
         opener,
         indexfile,
         datafile=None,
         checkambig=False,
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
         persistentnodemap=False,
         concurrencychecker=None,
     ):
         """
         create a revlog object
 
         opener is a function that abstracts the file opening operation
         and can be used to implement COW semantics or the like.
 
         """
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
         self.nodemap_file = None
         if persistentnodemap:
             self.nodemap_file = nodemaputil.get_nodemap_file(
                 opener, self.indexfile
             )
 
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
         self._checkambig = checkambig
         self._mmaplargeindex = mmaplargeindex
         self._censorable = censorable
         # 3-tuple of (node, rev, text) for a raw revision.
         self._revisioncache = None
         # Maps rev to chain base rev.
         self._chainbasecache = util.lrucachedict(100)
         # 2-tuple of (offset, data) of raw data from the revlog at an offset.
         self._chunkcache = (0, b'')
         # How much data to read and cache into the raw revlog data cache.
         self._chunkcachesize = 65536
         self._maxchainlen = None
         self._deltabothparents = True
         self.index = None
         self._nodemap_docket = None
         # Mapping of partial identifiers to full nodes.
         self._pcache = {}
         # Mapping of revision integer to full node.
         self._compengine = b'zlib'
         self._compengineopts = {}
         self._maxdeltachainspan = -1
         self._withsparseread = False
         self._sparserevlog = False
         self._srdensitythreshold = 0.50
         self._srmingapsize = 262144
 
         # Make copy of flag processors so each revlog instance can support
         # custom flags.
         self._flagprocessors = dict(flagutil.flagprocessors)
 
         # 2-tuple of file handles being used for active writing.
         self._writinghandles = None
 
         self._loadindex()
 
         self._concurrencychecker = concurrencychecker
 
     def _loadindex(self):
         mmapindexthreshold = None
         opts = self.opener.options
 
         if b'revlogv2' in opts:
             newversionflags = REVLOGV2 | FLAG_INLINE_DATA
         elif b'revlogv1' in opts:
             newversionflags = REVLOGV1 | FLAG_INLINE_DATA
             if b'generaldelta' in opts:
                 newversionflags |= FLAG_GENERALDELTA
         elif b'revlogv0' in self.opener.options:
             newversionflags = REVLOGV0
         else:
             newversionflags = REVLOG_DEFAULT_VERSION
 
         if b'chunkcachesize' in opts:
             self._chunkcachesize = opts[b'chunkcachesize']
         if b'maxchainlen' in opts:
             self._maxchainlen = opts[b'maxchainlen']
         if b'deltabothparents' in opts:
             self._deltabothparents = opts[b'deltabothparents']
         self._lazydelta = bool(opts.get(b'lazydelta', True))
         self._lazydeltabase = False
         if self._lazydelta:
             self._lazydeltabase = bool(opts.get(b'lazydeltabase', False))
         if b'compengine' in opts:
             self._compengine = opts[b'compengine']
         if b'zlib.level' in opts:
             self._compengineopts[b'zlib.level'] = opts[b'zlib.level']
         if b'zstd.level' in opts:
             self._compengineopts[b'zstd.level'] = opts[b'zstd.level']
         if b'maxdeltachainspan' in opts:
             self._maxdeltachainspan = opts[b'maxdeltachainspan']
         if self._mmaplargeindex and b'mmapindexthreshold' in opts:
             mmapindexthreshold = opts[b'mmapindexthreshold']
         self.hassidedata = bool(opts.get(b'side-data', False))
         self._sparserevlog = bool(opts.get(b'sparse-revlog', False))
         withsparseread = bool(opts.get(b'with-sparse-read', False))
         # sparse-revlog forces sparse-read
         self._withsparseread = self._sparserevlog or withsparseread
         if b'sparse-read-density-threshold' in opts:
             self._srdensitythreshold = opts[b'sparse-read-density-threshold']
         if b'sparse-read-min-gap-size' in opts:
             self._srmingapsize = opts[b'sparse-read-min-gap-size']
         if opts.get(b'enableellipsis'):
             self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
 
         # revlog v0 doesn't have flag processors
         for flag, processor in pycompat.iteritems(
             opts.get(b'flagprocessors', {})
         ):
             flagutil.insertflagprocessor(flag, processor, self._flagprocessors)
 
         if self._chunkcachesize <= 0:
             raise error.RevlogError(
                 _(b'revlog chunk cache size %r is not greater than 0')
                 % self._chunkcachesize
             )
         elif self._chunkcachesize & (self._chunkcachesize - 1):
             raise error.RevlogError(
                 _(b'revlog chunk cache size %r is not a power of 2')
                 % self._chunkcachesize
             )
 
         indexdata = b''
         self._initempty = True
         try:
             with self._indexfp() as f:
                 if (
                     mmapindexthreshold is not None
                     and self.opener.fstat(f).st_size >= mmapindexthreshold
                 ):
                     # TODO: should .close() to release resources without
                     # relying on Python GC
                     indexdata = util.buffer(util.mmapread(f))
                 else:
                     indexdata = f.read()
             if len(indexdata) > 0:
                 versionflags = versionformat_unpack(indexdata[:4])[0]
                 self._initempty = False
             else:
                 versionflags = newversionflags
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
 
             versionflags = newversionflags
 
         self.version = versionflags
 
         flags = versionflags & ~0xFFFF
         fmt = versionflags & 0xFFFF
 
         if fmt == REVLOGV0:
             if flags:
                 raise error.RevlogError(
                     _(b'unknown flags (%#04x) in version %d revlog %s')
                     % (flags >> 16, fmt, self.indexfile)
                 )
 
             self._inline = False
             self._generaldelta = False
 
         elif fmt == REVLOGV1:
             if flags & ~REVLOGV1_FLAGS:
                 raise error.RevlogError(
                     _(b'unknown flags (%#04x) in version %d revlog %s')
                     % (flags >> 16, fmt, self.indexfile)
                 )
 
             self._inline = versionflags & FLAG_INLINE_DATA
             self._generaldelta = versionflags & FLAG_GENERALDELTA
 
         elif fmt == REVLOGV2:
             if flags & ~REVLOGV2_FLAGS:
                 raise error.RevlogError(
                     _(b'unknown flags (%#04x) in version %d revlog %s')
                     % (flags >> 16, fmt, self.indexfile)
                 )
 
             self._inline = versionflags & FLAG_INLINE_DATA
             # generaldelta implied by version 2 revlogs.
             self._generaldelta = True
 
         else:
             raise error.RevlogError(
                 _(b'unknown version (%d) in revlog %s') % (fmt, self.indexfile)
             )
         # sparse-revlog can't be on without general-delta (issue6056)
         if not self._generaldelta:
             self._sparserevlog = False
 
         self._storedeltachains = True
 
         devel_nodemap = (
             self.nodemap_file
             and opts.get(b'devel-force-nodemap', False)
             and NodemapRevlogIO is not None
         )
 
         use_rust_index = False
         if rustrevlog is not None:
             if self.nodemap_file is not None:
                 use_rust_index = True
             else:
                 use_rust_index = self.opener.options.get(b'rust.index')
 
         self._io = revlogio()
         if self.version == REVLOGV0:
             self._io = revlogoldio()
         elif fmt == REVLOGV2:
             self._io = revlogv2io()
         elif devel_nodemap:
             self._io = NodemapRevlogIO()
         elif use_rust_index:
             self._io = rustrevlogio()
         try:
             d = self._io.parseindex(indexdata, self._inline)
             index, _chunkcache = d
             use_nodemap = (
                 not self._inline
                 and self.nodemap_file is not None
                 and util.safehasattr(index, 'update_nodemap_data')
             )
             if use_nodemap:
                 nodemap_data = nodemaputil.persisted_data(self)
                 if nodemap_data is not None:
                     docket = nodemap_data[0]
                     if (
                         len(d[0]) > docket.tip_rev
                         and d[0][docket.tip_rev][7] == docket.tip_node
                     ):
                         # no changelog tampering
                         self._nodemap_docket = docket
                         index.update_nodemap_data(*nodemap_data)
         except (ValueError, IndexError):
             raise error.RevlogError(
                 _(b"index %s is corrupted") % self.indexfile
             )
         self.index, self._chunkcache = d
         if not self._chunkcache:
             self._chunkclear()
         # revnum -> (chain-length, sum-delta-length)
         self._chaininfocache = util.lrucachedict(500)
         # revlog header -> revlog compressor
         self._decompressors = {}
 
     @util.propertycache
     def _compressor(self):
         engine = util.compengines[self._compengine]
         return engine.revlogcompressor(self._compengineopts)
 
     def _indexfp(self, mode=b'r'):
         """file object for the revlog's index file"""
         args = {'mode': mode}
         if mode != b'r':
             args['checkambig'] = self._checkambig
         if mode == b'w':
             args['atomictemp'] = True
         return self.opener(self.indexfile, **args)
 
     def _datafp(self, mode=b'r'):
         """file object for the revlog's data file"""
         return self.opener(self.datafile, mode=mode)
 
     @contextlib.contextmanager
     def _datareadfp(self, existingfp=None):
         """file object suitable to read data"""
         # Use explicit file handle, if given.
         if existingfp is not None:
             yield existingfp
 
         # Use a file handle being actively used for writes, if available.
         # There is some danger to doing this because reads will seek the
         # file. However, _writeentry() performs a SEEK_END before all writes,
         # so we should be safe.
         elif self._writinghandles:
             if self._inline:
                 yield self._writinghandles[0]
             else:
                 yield self._writinghandles[1]
 
         # Otherwise open a new file handle.
         else:
             if self._inline:
                 func = self._indexfp
             else:
                 func = self._datafp
             with func() as fp:
                 yield fp
 
     def tiprev(self):
         return len(self.index) - 1
 
     def tip(self):
         return self.node(self.tiprev())
 
     def __contains__(self, rev):
         return 0 <= rev < len(self)
 
     def __len__(self):
         return len(self.index)
 
     def __iter__(self):
         return iter(pycompat.xrange(len(self)))
 
     def revs(self, start=0, stop=None):
         """iterate over all rev in this revlog (from start to stop)"""
         return storageutil.iterrevs(len(self), start=start, stop=stop)
 
     @property
     def nodemap(self):
         msg = (
             b"revlog.nodemap is deprecated, "
             b"use revlog.index.[has_node|rev|get_rev]"
         )
         util.nouideprecwarn(msg, b'5.3', stacklevel=2)
         return self.index.nodemap
 
     @property
     def _nodecache(self):
         msg = b"revlog._nodecache is deprecated, use revlog.index.nodemap"
         util.nouideprecwarn(msg, b'5.3', stacklevel=2)
         return self.index.nodemap
 
     def hasnode(self, node):
         try:
             self.rev(node)
             return True
         except KeyError:
             return False
 
     def candelta(self, baserev, rev):
         """whether two revisions (baserev, rev) can be delta-ed or not"""
         # Disable delta if either rev requires a content-changing flag
         # processor (ex. LFS). This is because such flag processor can alter
         # the rawtext content that the delta will be based on, and two clients
         # could have a same revlog node with different flags (i.e. different
         # rawtext contents) and the delta could be incompatible.
         if (self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS) or (
             self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS
         ):
             return False
         return True
 
     def update_caches(self, transaction):
         if self.nodemap_file is not None:
             if transaction is None:
                 nodemaputil.update_persistent_nodemap(self)
             else:
                 nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def clearcaches(self):
         self._revisioncache = None
         self._chainbasecache.clear()
         self._chunkcache = (0, b'')
         self._pcache = {}
         self._nodemap_docket = None
         self.index.clearcaches()
         # The python code is the one responsible for validating the docket, we
         # end up having to refresh it here.
         use_nodemap = (
             not self._inline
             and self.nodemap_file is not None
             and util.safehasattr(self.index, 'update_nodemap_data')
         )
         if use_nodemap:
             nodemap_data = nodemaputil.persisted_data(self)
             if nodemap_data is not None:
                 self._nodemap_docket = nodemap_data[0]
                 self.index.update_nodemap_data(*nodemap_data)
 
     def rev(self, node):
         try:
             return self.index.rev(node)
         except TypeError:
             raise
         except error.RevlogError:
             # parsers.c radix tree lookup failed
             if node == wdirid or node in wdirfilenodeids:
                 raise error.WdirUnsupported
             raise error.LookupError(node, self.indexfile, _(b'no node'))
 
     # Accessors for index entries.
 
     # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
     # are flags.
     def start(self, rev):
         return int(self.index[rev][0] >> 16)
 
     def flags(self, rev):
         return self.index[rev][0] & 0xFFFF
 
     def length(self, rev):
         return self.index[rev][1]
 
     def sidedata_length(self, rev):
         if self.version & 0xFFFF != REVLOGV2:
             return 0
         return self.index[rev][9]
 
     def rawsize(self, rev):
         """return the length of the uncompressed text for a given revision"""
         l = self.index[rev][2]
         if l >= 0:
             return l
 
         t = self.rawdata(rev)
         return len(t)
 
     def size(self, rev):
         """length of non-raw text (processed by a "read" flag processor)"""
         # fast path: if no "read" flag processor could change the content,
         # size is rawsize. note: ELLIPSIS is known to not change the content.
         flags = self.flags(rev)
         if flags & (flagutil.REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
             return self.rawsize(rev)
 
         return len(self.revision(rev, raw=False))
 
     def chainbase(self, rev):
         base = self._chainbasecache.get(rev)
         if base is not None:
             return base
 
         index = self.index
         iterrev = rev
         base = index[iterrev][3]
         while base != iterrev:
             iterrev = base
             base = index[iterrev][3]
 
         self._chainbasecache[rev] = base
         return base
 
     def linkrev(self, rev):
         return self.index[rev][4]
 
     def parentrevs(self, rev):
         try:
             entry = self.index[rev]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
         return entry[5], entry[6]
 
     # fast parentrevs(rev) where rev isn't filtered
     _uncheckedparentrevs = parentrevs
 
     def node(self, rev):
         try:
             return self.index[rev][7]
         except IndexError:
             if rev == wdirrev:
                 raise error.WdirUnsupported
             raise
 
     # Derived from index values.
 
     def end(self, rev):
         return self.start(rev) + self.length(rev)
 
     def parents(self, node):
         i = self.index
         d = i[self.rev(node)]
         return i[d[5]][7], i[d[6]][7]  # map revisions to nodes inline
 
     def chainlen(self, rev):
         return self._chaininfo(rev)[0]
 
     def _chaininfo(self, rev):
         chaininfocache = self._chaininfocache
         if rev in chaininfocache:
             return chaininfocache[rev]
         index = self.index
         generaldelta = self._generaldelta
         iterrev = rev
         e = index[iterrev]
         clen = 0
         compresseddeltalen = 0
         while iterrev != e[3]:
             clen += 1
             compresseddeltalen += e[1]
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             if iterrev in chaininfocache:
                 t = chaininfocache[iterrev]
                 clen += t[0]
                 compresseddeltalen += t[1]
                 break
             e = index[iterrev]
         else:
             # Add text length of base since decompressing that also takes
             # work. For cache hits the length is already included.
             compresseddeltalen += e[1]
         r = (clen, compresseddeltalen)
         chaininfocache[rev] = r
         return r
 
     def _deltachain(self, rev, stoprev=None):
         """Obtain the delta chain for a revision.
 
         ``stoprev`` specifies a revision to stop at. If not specified, we
         stop at the base of the chain.
 
         Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
         revs in ascending order and ``stopped`` is a bool indicating whether
         ``stoprev`` was hit.
         """
         # Try C implementation.
         try:
             return self.index.deltachain(rev, stoprev, self._generaldelta)
         except AttributeError:
             pass
 
         chain = []
 
         # Alias to prevent attribute lookup in tight loop.
         index = self.index
         generaldelta = self._generaldelta
 
         iterrev = rev
         e = index[iterrev]
         while iterrev != e[3] and iterrev != stoprev:
             chain.append(iterrev)
             if generaldelta:
                 iterrev = e[3]
             else:
                 iterrev -= 1
             e = index[iterrev]
 
         if iterrev == stoprev:
             stopped = True
         else:
             chain.append(iterrev)
             stopped = False
 
         chain.reverse()
         return chain, stopped
 
     def ancestors(self, revs, stoprev=0, inclusive=False):
         """Generate the ancestors of 'revs' in reverse revision order.
         Does not generate revs lower than stoprev.
 
         See the documentation for ancestor.lazyancestors for more details."""
 
         # first, make sure start revisions aren't filtered
         revs = list(revs)
         checkrev = self.node
         for r in revs:
             checkrev(r)
         # and we're sure ancestors aren't filtered as well
 
         if rustancestor is not None:
             lazyancestors = rustancestor.LazyAncestors
             arg = self.index
         else:
             lazyancestors = ancestor.lazyancestors
             arg = self._uncheckedparentrevs
         return lazyancestors(arg, revs, stoprev=stoprev, inclusive=inclusive)
 
     def descendants(self, revs):
         return dagop.descendantrevs(revs, self.revs, self.parentrevs)
 
     def findcommonmissing(self, common=None, heads=None):
         """Return a tuple of the ancestors of common and the ancestors of heads
         that are not ancestors of common. In revset terminology, we return the
         tuple:
 
           ::common, (::heads) - (::common)
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         # we want the ancestors, but inclusive
         class lazyset(object):
             def __init__(self, lazyvalues):
                 self.addedvalues = set()
                 self.lazyvalues = lazyvalues
 
             def __contains__(self, value):
                 return value in self.addedvalues or value in self.lazyvalues
 
             def __iter__(self):
                 added = self.addedvalues
                 for r in added:
                     yield r
                 for r in self.lazyvalues:
                     if not r in added:
                         yield r
 
             def add(self, value):
                 self.addedvalues.add(value)
 
             def update(self, values):
                 self.addedvalues.update(values)
 
         has = lazyset(self.ancestors(common))
         has.add(nullrev)
         has.update(common)
 
         # take all ancestors from heads that aren't in has
         missing = set()
         visit = collections.deque(r for r in heads if r not in has)
         while visit:
             r = visit.popleft()
             if r in missing:
                 continue
             else:
                 missing.add(r)
                 for p in self.parentrevs(r):
                     if p not in has:
                         visit.append(p)
         missing = list(missing)
         missing.sort()
         return has, [self.node(miss) for miss in missing]
 
     def incrementalmissingrevs(self, common=None):
         """Return an object that can be used to incrementally compute the
         revision numbers of the ancestors of arbitrary sets that are not
         ancestors of common. This is an ancestor.incrementalmissingancestors
         object.
 
         'common' is a list of revision numbers. If common is not supplied, uses
         nullrev.
         """
         if common is None:
             common = [nullrev]
 
         if rustancestor is not None:
             return rustancestor.MissingAncestors(self.index, common)
         return ancestor.incrementalmissingancestors(self.parentrevs, common)
 
     def findmissingrevs(self, common=None, heads=None):
         """Return the revision numbers of the ancestors of heads that
         are not ancestors of common.
 
         More specifically, return a list of revision numbers corresponding to
         nodes N such that every N satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of revision numbers.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullrev]
         if heads is None:
             heads = self.headrevs()
 
         inc = self.incrementalmissingrevs(common=common)
         return inc.missingancestors(heads)
 
     def findmissing(self, common=None, heads=None):
         """Return the ancestors of heads that are not ancestors of common.
 
         More specifically, return a list of nodes N such that every N
         satisfies the following constraints:
 
           1. N is an ancestor of some node in 'heads'
           2. N is not an ancestor of any node in 'common'
 
         The list is sorted by revision number, meaning it is
         topologically sorted.
 
         'heads' and 'common' are both lists of node IDs.  If heads is
         not supplied, uses all of the revlog's heads.  If common is not
         supplied, uses nullid."""
         if common is None:
             common = [nullid]
         if heads is None:
             heads = self.heads()
 
         common = [self.rev(n) for n in common]
         heads = [self.rev(n) for n in heads]
 
         inc = self.incrementalmissingrevs(common=common)
         return [self.node(r) for r in inc.missingancestors(heads)]
 
     def nodesbetween(self, roots=None, heads=None):
         """Return a topological path from 'roots' to 'heads'.
 
         Return a tuple (nodes, outroots, outheads) where 'nodes' is a
         topologically sorted list of all nodes N that satisfy both of
         these constraints:
 
           1. N is a descendant of some node in 'roots'
           2. N is an ancestor of some node in 'heads'
 
         Every node is considered to be both a descendant and an ancestor
         of itself, so every reachable node in 'roots' and 'heads' will be
         included in 'nodes'.
 
         'outroots' is the list of reachable nodes in 'roots', i.e., the
         subset of 'roots' that is returned in 'nodes'.  Likewise,
         'outheads' is the subset of 'heads' that is also in 'nodes'.
 
         'roots' and 'heads' are both lists of node IDs.  If 'roots' is
         unspecified, uses nullid as the only root.  If 'heads' is
         unspecified, uses list of all of the revlog's heads."""
         nonodes = ([], [], [])
         if roots is not None:
             roots = list(roots)
             if not roots:
                 return nonodes
             lowestrev = min([self.rev(n) for n in roots])
         else:
             roots = [nullid]  # Everybody's a descendant of nullid
             lowestrev = nullrev
         if (lowestrev == nullrev) and (heads is None):
             # We want _all_ the nodes!
             return ([self.node(r) for r in self], [nullid], list(self.heads()))
         if heads is None:
             # All nodes are ancestors, so the latest ancestor is the last
             # node.
             highestrev = len(self) - 1
             # Set ancestors to None to signal that every node is an ancestor.
             ancestors = None
             # Set heads to an empty dictionary for later discovery of heads
             heads = {}
         else:
             heads = list(heads)
             if not heads:
                 return nonodes
             ancestors = set()
             # Turn heads into a dictionary so we can remove 'fake' heads.
             # Also, later we will be using it to filter out the heads we can't
             # find from roots.
             heads = dict.fromkeys(heads, False)
             # Start at the top and keep marking parents until we're done.
             nodestotag = set(heads)
             # Remember where the top was so we can use it as a limit later.
             highestrev = max([self.rev(n) for n in nodestotag])
             while nodestotag:
                 # grab a node to tag
                 n = nodestotag.pop()
                 # Never tag nullid
                 if n == nullid:
                     continue
                 # A node's revision number represents its place in a
                 # topologically sorted list of nodes.
                 r = self.rev(n)
                 if r >= lowestrev:
                     if n not in ancestors:
                         # If we are possibly a descendant of one of the roots
                         # and we haven't already been marked as an ancestor
                         ancestors.add(n)  # Mark as ancestor
                         # Add non-nullid parents to list of nodes to tag.
                         nodestotag.update(
                             [p for p in self.parents(n) if p != nullid]
                         )
                     elif n in heads:  # We've seen it before, is it a fake head?
                         # So it is, real heads should not be the ancestors of
                         # any other heads.
                         heads.pop(n)
             if not ancestors:
                 return nonodes
             # Now that we have our set of ancestors, we want to remove any
             # roots that are not ancestors.
 
             # If one of the roots was nullid, everything is included anyway.
             if lowestrev > nullrev:
                 # But, since we weren't, let's recompute the lowest rev to not
                 # include roots that aren't ancestors.
 
                 # Filter out roots that aren't ancestors of heads
                 roots = [root for root in roots if root in ancestors]
                 # Recompute the lowest revision
                 if roots:
                     lowestrev = min([self.rev(root) for root in roots])
                 else:
                     # No more roots?  Return empty list
                     return nonodes
             else:
                 # We are descending from nullid, and don't need to care about
                 # any other roots.
                 lowestrev = nullrev
                 roots = [nullid]
         # Transform our roots list into a set.
         descendants = set(roots)
         # Also, keep the original roots so we can filter out roots that aren't
         # 'real' roots (i.e. are descended from other roots).
         roots = descendants.copy()
         # Our topologically sorted list of output nodes.
         orderedout = []
         # Don't start at nullid since we don't want nullid in our output list,
         # and if nullid shows up in descendants, empty parents will look like
         # they're descendants.
         for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
             n = self.node(r)
             isdescendant = False
             if lowestrev == nullrev:  # Everybody is a descendant of nullid
                 isdescendant = True
             elif n in descendants:
                 # n is already a descendant
                 isdescendant = True
                 # This check only needs to be done here because all the roots
                 # will start being marked is descendants before the loop.
                 if n in roots:
                     # If n was a root, check if it's a 'real' root.
                     p = tuple(self.parents(n))
                     # If any of its parents are descendants, it's not a root.
                     if (p[0] in descendants) or (p[1] in descendants):
                         roots.remove(n)
             else:
                 p = tuple(self.parents(n))
                 # A node is a descendant if either of its parents are
                 # descendants.  (We seeded the dependents list with the roots
                 # up there, remember?)
                 if (p[0] in descendants) or (p[1] in descendants):
                     descendants.add(n)
                     isdescendant = True
             if isdescendant and ((ancestors is None) or (n in ancestors)):
                 # Only include nodes that are both descendants and ancestors.
                 orderedout.append(n)
                 if (ancestors is not None) and (n in heads):
                     # We're trying to figure out which heads are reachable
                     # from roots.
                     # Mark this head as having been reached
                     heads[n] = True
                 elif ancestors is None:
                     # Otherwise, we're trying to discover the heads.
                     # Assume this is a head because if it isn't, the next step
                     # will eventually remove it.
                     heads[n] = True
                     # But, obviously its parents aren't.
                     for p in self.parents(n):
                         heads.pop(p, None)
         heads = [head for head, flag in pycompat.iteritems(heads) if flag]
         roots = list(roots)
         assert orderedout
         assert roots
         assert heads
         return (orderedout, roots, heads)
 
     def headrevs(self, revs=None):
         if revs is None:
             try:
                 return self.index.headrevs()
             except AttributeError:
                 return self._headrevs()
         if rustdagop is not None:
             return rustdagop.headrevs(self.index, revs)
         return dagop.headrevs(revs, self._uncheckedparentrevs)
 
     def computephases(self, roots):
         return self.index.computephasesmapsets(roots)
 
     def _headrevs(self):
         count = len(self)
         if not count:
             return [nullrev]
         # we won't iter over filtered rev so nobody is a head at start
         ishead = [0] * (count + 1)
         index = self.index
         for r in self:
             ishead[r] = 1  # I may be an head
             e = index[r]
             ishead[e[5]] = ishead[e[6]] = 0  # my parent are not
         return [r for r, val in enumerate(ishead) if val]
 
     def heads(self, start=None, stop=None):
         """return the list of all nodes that have no children
 
         if start is specified, only heads that are descendants of
         start will be returned
         if stop is specified, it will consider all the revs from stop
         as if they had no children
         """
         if start is None and stop is None:
             if not len(self):
                 return [nullid]
             return [self.node(r) for r in self.headrevs()]
 
         if start is None:
             start = nullrev
         else:
             start = self.rev(start)
 
         stoprevs = {self.rev(n) for n in stop or []}
 
         revs = dagop.headrevssubset(
             self.revs, self.parentrevs, startrev=start, stoprevs=stoprevs
         )
 
         return [self.node(rev) for rev in revs]
 
     def children(self, node):
         """find the children of a given node"""
         c = []
         p = self.rev(node)
         for r in self.revs(start=p + 1):
             prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
             if prevs:
                 for pr in prevs:
                     if pr == p:
                         c.append(self.node(r))
             elif p == nullrev:
                 c.append(self.node(r))
         return c
 
     def commonancestorsheads(self, a, b):
         """calculate all the heads of the common ancestors of nodes a and b"""
         a, b = self.rev(a), self.rev(b)
         ancs = self._commonancestorsheads(a, b)
         return pycompat.maplist(self.node, ancs)
 
     def _commonancestorsheads(self, *revs):
         """calculate all the heads of the common ancestors of revs"""
         try:
             ancs = self.index.commonancestorsheads(*revs)
         except (AttributeError, OverflowError):  # C implementation failed
             ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
         return ancs
 
     def isancestor(self, a, b):
         """return True if node a is an ancestor of node b
 
         A revision is considered an ancestor of itself."""
         a, b = self.rev(a), self.rev(b)
         return self.isancestorrev(a, b)
 
     def isancestorrev(self, a, b):
         """return True if revision a is an ancestor of revision b
 
         A revision is considered an ancestor of itself.
 
         The implementation of this is trivial but the use of
         reachableroots is not."""
         if a == nullrev:
             return True
         elif a == b:
             return True
         elif a > b:
             return False
         return bool(self.reachableroots(a, [b], [a], includepath=False))
 
     def reachableroots(self, minroot, heads, roots, includepath=False):
         """return (heads(::(<roots> and <roots>::<heads>)))
 
         If includepath is True, return (<roots>::<heads>)."""
         try:
             return self.index.reachableroots2(
                 minroot, heads, roots, includepath
             )
         except AttributeError:
             return dagop._reachablerootspure(
                 self.parentrevs, minroot, roots, heads, includepath
             )
 
     def ancestor(self, a, b):
         """calculate the "best" common ancestor of nodes a and b"""
 
         a, b = self.rev(a), self.rev(b)
         try:
             ancs = self.index.ancestors(a, b)
         except (AttributeError, OverflowError):
             ancs = ancestor.ancestors(self.parentrevs, a, b)
         if ancs:
             # choose a consistent winner when there's a tie
             return min(map(self.node, ancs))
         return nullid
 
     def _match(self, id):
         if isinstance(id, int):
             # rev
             return self.node(id)
         if len(id) == 20:
             # possibly a binary node
             # odds of a binary node being all hex in ASCII are 1 in 10**25
             try:
                 node = id
                 self.rev(node)  # quick search the index
                 return node
             except error.LookupError:
                 pass  # may be partial hex id
         try:
             # str(rev)
             rev = int(id)
             if b"%d" % rev != id:
                 raise ValueError
             if rev < 0:
                 rev = len(self) + rev
             if rev < 0 or rev >= len(self):
                 raise ValueError
             return self.node(rev)
         except (ValueError, OverflowError):
             pass
         if len(id) == 40:
             try:
                 # a full hex nodeid?
                 node = bin(id)
                 self.rev(node)
                 return node
             except (TypeError, error.LookupError):
                 pass
 
     def _partialmatch(self, id):
         # we don't care wdirfilenodeids as they should be always full hash
         maybewdir = wdirhex.startswith(id)
         try:
             partial = self.index.partialmatch(id)
             if partial and self.hasnode(partial):
                 if maybewdir:
                     # single 'ff...' match in radix tree, ambiguous with wdir
                     raise error.RevlogError
                 return partial
             if maybewdir:
                 # no 'ff...' match in radix tree, wdir identified
                 raise error.WdirUnsupported
             return None
         except error.RevlogError:
             # parsers.c radix tree lookup gave multiple matches
             # fast path: for unfiltered changelog, radix tree is accurate
             if not getattr(self, 'filteredrevs', None):
                 raise error.AmbiguousPrefixLookupError(
                     id, self.indexfile, _(b'ambiguous identifier')
                 )
             # fall through to slow path that filters hidden revisions
         except (AttributeError, ValueError):
             # we are pure python, or key was too short to search radix tree
             pass
 
         if id in self._pcache:
             return self._pcache[id]
 
         if len(id) <= 40:
             try:
                 # hex(node)[:...]
                 l = len(id) // 2  # grab an even number of digits
                 prefix = bin(id[: l * 2])
                 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
                 nl = [
                     n for n in nl if hex(n).startswith(id) and self.hasnode(n)
                 ]
                 if nullhex.startswith(id):
                     nl.append(nullid)
                 if len(nl) > 0:
                     if len(nl) == 1 and not maybewdir:
                         self._pcache[id] = nl[0]
                         return nl[0]
                     raise error.AmbiguousPrefixLookupError(
                         id, self.indexfile, _(b'ambiguous identifier')
                     )
                 if maybewdir:
                     raise error.WdirUnsupported
                 return None
             except TypeError:
                 pass
 
     def lookup(self, id):
         """locate a node based on:
         - revision number or str(revision number)
         - nodeid or subset of hex nodeid
         """
         n = self._match(id)
         if n is not None:
             return n
         n = self._partialmatch(id)
         if n:
             return n
 
         raise error.LookupError(id, self.indexfile, _(b'no match found'))
 
     def shortest(self, node, minlength=1):
         """Find the shortest unambiguous prefix that matches node."""
 
         def isvalid(prefix):
             try:
                 matchednode = self._partialmatch(prefix)
             except error.AmbiguousPrefixLookupError:
                 return False
             except error.WdirUnsupported:
                 # single 'ff...' match
                 return True
             if matchednode is None:
                 raise error.LookupError(node, self.indexfile, _(b'no node'))
             return True
 
         def maybewdir(prefix):
             return all(c == b'f' for c in pycompat.iterbytestr(prefix))
 
         hexnode = hex(node)
 
         def disambiguate(hexnode, minlength):
             """Disambiguate against wdirid."""
             for length in range(minlength, len(hexnode) + 1):
                 prefix = hexnode[:length]
                 if not maybewdir(prefix):
                     return prefix
 
         if not getattr(self, 'filteredrevs', None):
             try:
                 length = max(self.index.shortest(node), minlength)
                 return disambiguate(hexnode, length)
             except error.RevlogError:
                 if node != wdirid:
                     raise error.LookupError(node, self.indexfile, _(b'no node'))
             except AttributeError:
                 # Fall through to pure code
                 pass
 
         if node == wdirid:
             for length in range(minlength, len(hexnode) + 1):
                 prefix = hexnode[:length]
                 if isvalid(prefix):
                     return prefix
 
         for length in range(minlength, len(hexnode) + 1):
             prefix = hexnode[:length]
             if isvalid(prefix):
                 return disambiguate(hexnode, length)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
         p1, p2 = self.parents(node)
         return storageutil.hashrevisionsha1(text, p1, p2) != node
 
     def _cachesegment(self, offset, data):
         """Add a segment to the revlog cache.
 
         Accepts an absolute offset and the data that is at that location.
         """
         o, d = self._chunkcache
         # try to add to existing cache
         if o + len(d) == offset and len(d) + len(data) < _chunksize:
             self._chunkcache = o, d + data
         else:
             self._chunkcache = offset, data
 
     def _readsegment(self, offset, length, df=None):
         """Load a segment of raw data from the revlog.
 
         Accepts an absolute offset, length to read, and an optional existing
         file handle to read from.
 
         If an existing file handle is passed, it will be seeked and the
         original seek position will NOT be restored.
 
         Returns a str or buffer of raw byte data.
 
         Raises if the requested number of bytes could not be read.
         """
         # Cache data both forward and backward around the requested
         # data, in a fixed size window. This helps speed up operations
         # involving reading the revlog backwards.
         cachesize = self._chunkcachesize
         realoffset = offset & ~(cachesize - 1)
         reallength = (
             (offset + length + cachesize) & ~(cachesize - 1)
         ) - realoffset
         with self._datareadfp(df) as df:
             df.seek(realoffset)
             d = df.read(reallength)
 
         self._cachesegment(realoffset, d)
         if offset != realoffset or reallength != length:
             startoffset = offset - realoffset
             if len(d) - startoffset < length:
                 raise error.RevlogError(
                     _(
                         b'partial read of revlog %s; expected %d bytes from '
                         b'offset %d, got %d'
                     )
                     % (
                         self.indexfile if self._inline else self.datafile,
                         length,
                         realoffset,
                         len(d) - startoffset,
                     )
                 )
 
             return util.buffer(d, startoffset, length)
 
         if len(d) < length:
             raise error.RevlogError(
                 _(
                     b'partial read of revlog %s; expected %d bytes from offset '
                     b'%d, got %d'
                 )
                 % (
                     self.indexfile if self._inline else self.datafile,
                     length,
                     offset,
                     len(d),
                 )
             )
 
         return d
 
     def _getsegment(self, offset, length, df=None):
         """Obtain a segment of raw data from the revlog.
 
         Accepts an absolute offset, length of bytes to obtain, and an
         optional file handle to the already-opened revlog. If the file
         handle is used, it's original seek position will not be preserved.
 
         Requests for data may be returned from a cache.
 
         Returns a str or a buffer instance of raw byte data.
         """
         o, d = self._chunkcache
         l = len(d)
 
         # is it in the cache?
         cachestart = offset - o
         cacheend = cachestart + length
         if cachestart >= 0 and cacheend <= l:
             if cachestart == 0 and cacheend == l:
                 return d  # avoid a copy
             return util.buffer(d, cachestart, cacheend - cachestart)
 
         return self._readsegment(offset, length, df=df)
 
     def _getsegmentforrevs(self, startrev, endrev, df=None):
         """Obtain a segment of raw data corresponding to a range of revisions.
 
         Accepts the start and end revisions and an optional already-open
         file handle to be used for reading. If the file handle is read, its
         seek position will not be preserved.
 
         Requests for data may be satisfied by a cache.
 
         Returns a 2-tuple of (offset, data) for the requested range of
         revisions. Offset is the integer offset from the beginning of the
         revlog and data is a str or buffer of the raw byte data.
 
         Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
         to determine where each revision's data begins and ends.
         """
         # Inlined self.start(startrev) & self.end(endrev) for perf reasons
         # (functions are expensive).
         index = self.index
         istart = index[startrev]
         start = int(istart[0] >> 16)
         if startrev == endrev:
             end = start + istart[1]
         else:
             iend = index[endrev]
             end = int(iend[0] >> 16) + iend[1]
 
         if self._inline:
             start += (startrev + 1) * self._io.size
             end += (endrev + 1) * self._io.size
         length = end - start
 
         return start, self._getsegment(start, length, df=df)
 
     def _chunk(self, rev, df=None):
         """Obtain a single decompressed chunk for a revision.
 
         Accepts an integer revision and an optional already-open file handle
         to be used for reading. If used, the seek position of the file will not
         be preserved.
 
         Returns a str holding uncompressed data for the requested revision.
         """
         return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
 
     def _chunks(self, revs, df=None, targetsize=None):
         """Obtain decompressed chunks for the specified revisions.
 
         Accepts an iterable of numeric revisions that are assumed to be in
         ascending order. Also accepts an optional already-open file handle
         to be used for reading. If used, the seek position of the file will
         not be preserved.
 
         This function is similar to calling ``self._chunk()`` multiple times,
         but is faster.
 
         Returns a list with decompressed data for each requested revision.
         """
         if not revs:
             return []
         start = self.start
         length = self.length
         inline = self._inline
         iosize = self._io.size
         buffer = util.buffer
 
         l = []
         ladd = l.append
 
         if not self._withsparseread:
             slicedchunks = (revs,)
         else:
             slicedchunks = deltautil.slicechunk(
                 self, revs, targetsize=targetsize
             )
 
         for revschunk in slicedchunks:
             firstrev = revschunk[0]
             # Skip trailing revisions with empty diff
             for lastrev in revschunk[::-1]:
                 if length(lastrev) != 0:
                     break
 
             try:
                 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
             except OverflowError:
                 # issue4215 - we can't cache a run of chunks greater than
                 # 2G on Windows
                 return [self._chunk(rev, df=df) for rev in revschunk]
 
             decomp = self.decompress
             for rev in revschunk:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
 
         return l
 
     def _chunkclear(self):
         """Clear the raw chunk cache."""
         self._chunkcache = (0, b'')
 
     def deltaparent(self, rev):
         """return deltaparent of the given revision"""
         base = self.index[rev][3]
         if base == rev:
             return nullrev
         elif self._generaldelta:
             return base
         else:
             return rev - 1
 
     def issnapshot(self, rev):
         """tells whether rev is a snapshot"""
         if not self._sparserevlog:
             return self.deltaparent(rev) == nullrev
         elif util.safehasattr(self.index, b'issnapshot'):
             # directly assign the method to cache the testing and access
             self.issnapshot = self.index.issnapshot
             return self.issnapshot(rev)
         if rev == nullrev:
             return True
         entry = self.index[rev]
         base = entry[3]
         if base == rev:
             return True
         if base == nullrev:
             return True
         p1 = entry[5]
         p2 = entry[6]
         if base == p1 or base == p2:
             return False
         return self.issnapshot(base)
 
     def snapshotdepth(self, rev):
         """number of snapshot in the chain before this one"""
         if not self.issnapshot(rev):
             raise error.ProgrammingError(b'revision %d not a snapshot')
         return len(self._deltachain(rev)[0]) - 1
 
     def revdiff(self, rev1, rev2):
         """return or calculate a delta between two revisions
 
         The delta calculated is in binary form and is intended to be written to
         revlog data directly. So this function needs raw revision data.
         """
         if rev1 != nullrev and self.deltaparent(rev2) == rev1:
             return bytes(self._chunk(rev2))
 
         return mdiff.textdiff(self.rawdata(rev1), self.rawdata(rev2))
 
     def _processflags(self, text, flags, operation, raw=False):
         """deprecated entry point to access flag processors"""
         msg = b'_processflag(...) use the specialized variant'
         util.nouideprecwarn(msg, b'5.2', stacklevel=2)
         if raw:
             return text, flagutil.processflagsraw(self, text, flags)
         elif operation == b'read':
             return flagutil.processflagsread(self, text, flags)
         else:  # write operation
             return flagutil.processflagswrite(self, text, flags)
 
     def revision(self, nodeorrev, _df=None, raw=False):
         """return an uncompressed revision of a given node or revision
         number.
 
         _df - an existing file handle to read from. (internal-only)
         raw - an optional argument specifying if the revision data is to be
         treated as raw data when applying flag transforms. 'raw' should be set
         to True when generating changegroups or in debug commands.
         """
         if raw:
             msg = (
                 b'revlog.revision(..., raw=True) is deprecated, '
                 b'use revlog.rawdata(...)'
             )
             util.nouideprecwarn(msg, b'5.2', stacklevel=2)
         return self._revisiondata(nodeorrev, _df, raw=raw)[0]
 
     def sidedata(self, nodeorrev, _df=None):
         """a map of extra data related to the changeset but not part of the hash
 
         This function currently return a dictionary. However, more advanced
         mapping object will likely be used in the future for a more
         efficient/lazy code.
         """
         return self._revisiondata(nodeorrev, _df)[1]
 
     def _revisiondata(self, nodeorrev, _df=None, raw=False):
         # deal with <nodeorrev> argument type
         if isinstance(nodeorrev, int):
             rev = nodeorrev
             node = self.node(rev)
         else:
             node = nodeorrev
             rev = None
 
         # fast path the special `nullid` rev
         if node == nullid:
             return b"", {}
 
         # ``rawtext`` is the text as stored inside the revlog. Might be the
         # revision or might need to be processed to retrieve the revision.
         rev, rawtext, validated = self._rawtext(node, rev, _df=_df)
 
         if self.version & 0xFFFF == REVLOGV2:
             if rev is None:
                 rev = self.rev(node)
             sidedata = self._sidedata(rev)
         else:
             sidedata = {}
 
         if raw and validated:
             # if we don't want to process the raw text and that raw
             # text is cached, we can exit early.
             return rawtext, sidedata
         if rev is None:
             rev = self.rev(node)
         # the revlog's flag for this revision
         # (usually alter its state or content)
         flags = self.flags(rev)
 
         if validated and flags == REVIDX_DEFAULT_FLAGS:
             # no extra flags set, no flag processor runs, text = rawtext
             return rawtext, sidedata
 
         if raw:
             validatehash = flagutil.processflagsraw(self, rawtext, flags)
             text = rawtext
         else:
             r = flagutil.processflagsread(self, rawtext, flags)
             text, validatehash = r
         if validatehash:
             self.checkhash(text, node, rev=rev)
         if not validated:
             self._revisioncache = (node, rev, rawtext)
 
         return text, sidedata
 
     def _rawtext(self, node, rev, _df=None):
         """return the possibly unvalidated rawtext for a revision
 
         returns (rev, rawtext, validated)
         """
 
         # revision in the cache (could be useful to apply delta)
         cachedrev = None
         # An intermediate text to apply deltas to
         basetext = None
 
         # Check if we have the entry in cache
         # The cache entry looks like (node, rev, rawtext)
         if self._revisioncache:
             if self._revisioncache[0] == node:
                 return (rev, self._revisioncache[2], True)
             cachedrev = self._revisioncache[1]
 
         if rev is None:
             rev = self.rev(node)
 
         chain, stopped = self._deltachain(rev, stoprev=cachedrev)
         if stopped:
             basetext = self._revisioncache[2]
 
         # drop cache to save memory, the caller is expected to
         # update self._revisioncache after validating the text
         self._revisioncache = None
 
         targetsize = None
         rawsize = self.index[rev][2]
         if 0 <= rawsize:
             targetsize = 4 * rawsize
 
         bins = self._chunks(chain, df=_df, targetsize=targetsize)
         if basetext is None:
             basetext = bytes(bins[0])
             bins = bins[1:]
 
         rawtext = mdiff.patches(basetext, bins)
         del basetext  # let us have a chance to free memory early
         return (rev, rawtext, False)
 
     def _sidedata(self, rev):
         """Return the sidedata for a given revision number."""
         index_entry = self.index[rev]
         sidedata_offset = index_entry[8]
         sidedata_size = index_entry[9]
 
         if self._inline:
             sidedata_offset += self._io.size * (1 + rev)
         if sidedata_size == 0:
             return {}
 
         segment = self._getsegment(sidedata_offset, sidedata_size)
         sidedata = sidedatautil.deserialize_sidedata(segment)
         return sidedata
 
     def rawdata(self, nodeorrev, _df=None):
         """return an uncompressed raw data of a given node or revision number.
 
         _df - an existing file handle to read from. (internal-only)
         """
         return self._revisiondata(nodeorrev, _df, raw=True)[0]
 
     def hash(self, text, p1, p2):
         """Compute a node hash.
 
         Available as a function so that subclasses can replace the hash
         as needed.
         """
         return storageutil.hashrevisionsha1(text, p1, p2)
 
     def checkhash(self, text, node, p1=None, p2=None, rev=None):
         """Check node hash integrity.
 
         Available as a function so that subclasses can extend hash mismatch
         behaviors as needed.
         """
         try:
             if p1 is None and p2 is None:
                 p1, p2 = self.parents(node)
             if node != self.hash(text, p1, p2):
                 # Clear the revision cache on hash failure. The revision cache
                 # only stores the raw revision and clearing the cache does have
                 # the side-effect that we won't have a cache hit when the raw
                 # revision data is accessed. But this case should be rare and
                 # it is extra work to teach the cache about the hash
                 # verification state.
                 if self._revisioncache and self._revisioncache[0] == node:
                     self._revisioncache = None
 
                 revornode = rev
                 if revornode is None:
                     revornode = templatefilters.short(hex(node))
                 raise error.RevlogError(
                     _(b"integrity check failed on %s:%s")
                     % (self.indexfile, pycompat.bytestr(revornode))
                 )
         except error.RevlogError:
             if self._censorable and storageutil.iscensoredtext(text):
                 raise error.CensoredNodeError(self.indexfile, node, text)
             raise
 
     def _enforceinlinesize(self, tr, fp=None):
         """Check if the revlog is too big for inline and convert if so.
 
         This should be called after revisions are added to the revlog. If the
         revlog has grown too large to be an inline revlog, it will convert it
         to use multiple index and data files.
         """
         tiprev = len(self) - 1
         if (
             not self._inline
             or (self.start(tiprev) + self.length(tiprev)) < _maxinline
         ):
             return
 
         troffset = tr.findoffset(self.indexfile)
         if troffset is None:
             raise error.RevlogError(
                 _(b"%s not found in the transaction") % self.indexfile
             )
         trindex = 0
         tr.add(self.datafile, 0)
 
         if fp:
             fp.flush()
             fp.close()
             # We can't use the cached file handle after close(). So prevent
             # its usage.
             self._writinghandles = None
 
         with self._indexfp(b'r') as ifh, self._datafp(b'w') as dfh:
             for r in self:
                 dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
                 if troffset <= self.start(r):
                     trindex = r
 
         with self._indexfp(b'w') as fp:
             self.version &= ~FLAG_INLINE_DATA
             self._inline = False
             io = self._io
             for i in self:
                 e = io.packentry(self.index[i], self.node, self.version, i)
                 fp.write(e)
 
             # the temp file replace the real index when we exit the context
             # manager
 
         tr.replace(self.indexfile, trindex * self._io.size)
         nodemaputil.setup_persistent_nodemap(tr, self)
         self._chunkclear()
 
     def _nodeduplicatecallback(self, transaction, node):
         """called when trying to add a node already stored."""
 
     def addrevision(
         self,
         text,
         transaction,
         link,
         p1,
         p2,
         cachedelta=None,
         node=None,
         flags=REVIDX_DEFAULT_FLAGS,
         deltacomputer=None,
         sidedata=None,
     ):
         """add a revision to the log
 
         text - the revision data to add
         transaction - the transaction object used for rollback
         link - the linkrev data to add
         p1, p2 - the parent nodeids of the revision
         cachedelta - an optional precomputed delta
         node - nodeid of revision; typically node is not specified, and it is
             computed by default as hash(text, p1, p2), however subclasses might
             use different hashing method (and override checkhash() in such case)
         flags - the known flags to set on the revision
         deltacomputer - an optional deltacomputer instance shared between
             multiple calls
         """
         if link == nullrev:
             raise error.RevlogError(
                 _(b"attempted to add linkrev -1 to %s") % self.indexfile
             )
 
         if sidedata is None:
             sidedata = {}
         elif not self.hassidedata:
             raise error.ProgrammingError(
                 _(b"trying to add sidedata to a revlog who don't support them")
             )
 
         if flags:
             node = node or self.hash(text, p1, p2)
 
         rawtext, validatehash = flagutil.processflagswrite(self, text, flags)
 
         # If the flag processor modifies the revision data, ignore any provided
         # cachedelta.
         if rawtext != text:
             cachedelta = None
 
         if len(rawtext) > _maxentrysize:
             raise error.RevlogError(
                 _(
                     b"%s: size of %d bytes exceeds maximum revlog storage of 2GiB"
                 )
                 % (self.indexfile, len(rawtext))
             )
 
         node = node or self.hash(rawtext, p1, p2)
         rev = self.index.get_rev(node)
         if rev is not None:
             return rev
 
         if validatehash:
             self.checkhash(rawtext, node, p1=p1, p2=p2)
 
         return self.addrawrevision(
             rawtext,
             transaction,
             link,
             p1,
             p2,
             node,
             flags,
             cachedelta=cachedelta,
             deltacomputer=deltacomputer,
             sidedata=sidedata,
         )
 
     def addrawrevision(
         self,
         rawtext,
         transaction,
         link,
         p1,
         p2,
         node,
         flags,
         cachedelta=None,
         deltacomputer=None,
         sidedata=None,
     ):
         """add a raw revision with known flags, node and parents
         useful when reusing a revision not stored in this revlog (ex: received
         over wire, or read from an external bundle).
         """
         dfh = None
         if not self._inline:
             dfh = self._datafp(b"a+")
         ifh = self._indexfp(b"a+")
         try:
             return self._addrevision(
                 node,
                 rawtext,
                 transaction,
                 link,
                 p1,
                 p2,
                 flags,
                 cachedelta,
                 ifh,
                 dfh,
                 deltacomputer=deltacomputer,
                 sidedata=sidedata,
             )
         finally:
             if dfh:
                 dfh.close()
             ifh.close()
 
     def compress(self, data):
         """Generate a possibly-compressed representation of data."""
         if not data:
             return b'', data
 
         compressed = self._compressor.compress(data)
 
         if compressed:
             # The revlog compressor added the header in the returned data.
             return b'', compressed
 
         if data[0:1] == b'\0':
             return b'', data
         return b'u', data
 
     def decompress(self, data):
         """Decompress a revlog chunk.
 
         The chunk is expected to begin with a header identifying the
         format type so it can be routed to an appropriate decompressor.
         """
         if not data:
             return data
 
         # Revlogs are read much more frequently than they are written and many
         # chunks only take microseconds to decompress, so performance is
         # important here.
         #
         # We can make a few assumptions about revlogs:
         #
         # 1) the majority of chunks will be compressed (as opposed to inline
         #    raw data).
         # 2) decompressing *any* data will likely by at least 10x slower than
         #    returning raw inline data.
         # 3) we want to prioritize common and officially supported compression
         #    engines
         #
         # It follows that we want to optimize for "decompress compressed data
         # when encoded with common and officially supported compression engines"
         # case over "raw data" and "data encoded by less common or non-official
         # compression engines." That is why we have the inline lookup first
         # followed by the compengines lookup.
         #
         # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
         # compressed chunks. And this matters for changelog and manifest reads.
         t = data[0:1]
 
         if t == b'x':
             try:
                 return _zlibdecompress(data)
             except zlib.error as e:
                 raise error.RevlogError(
                     _(b'revlog decompress error: %s')
                     % stringutil.forcebytestr(e)
                 )
         # '\0' is more common than 'u' so it goes first.
         elif t == b'\0':
             return data
         elif t == b'u':
             return util.buffer(data, 1)
 
         try:
             compressor = self._decompressors[t]
         except KeyError:
             try:
                 engine = util.compengines.forrevlogheader(t)
                 compressor = engine.revlogcompressor(self._compengineopts)
                 self._decompressors[t] = compressor
             except KeyError:
                 raise error.RevlogError(_(b'unknown compression type %r') % t)
 
         return compressor.decompress(data)
 
     def _addrevision(
         self,
         node,
         rawtext,
         transaction,
         link,
         p1,
         p2,
         flags,
         cachedelta,
         ifh,
         dfh,
         alwayscache=False,
         deltacomputer=None,
         sidedata=None,
     ):
         """internal function to add revisions to the log
 
         see addrevision for argument descriptions.
 
         note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
 
         if "deltacomputer" is not provided or None, a defaultdeltacomputer will
         be used.
 
         invariants:
         - rawtext is optional (can be None); if not set, cachedelta must be set.
           if both are set, they must correspond to each other.
         """
         if node == nullid:
             raise error.RevlogError(
                 _(b"%s: attempt to add null revision") % self.indexfile
             )
         if node == wdirid or node in wdirfilenodeids:
             raise error.RevlogError(
                 _(b"%s: attempt to add wdir revision") % self.indexfile
             )
 
         if self._inline:
             fh = ifh
         else:
             fh = dfh
 
         btext = [rawtext]
 
         curr = len(self)
         prev = curr - 1
 
         offset = self._get_data_offset(prev)
 
         if self._concurrencychecker:
             if self._inline:
                 # offset is "as if" it were in the .d file, so we need to add on
                 # the size of the entry metadata.
                 self._concurrencychecker(
                     ifh, self.indexfile, offset + curr * self._io.size
                 )
             else:
                 # Entries in the .i are a consistent size.
                 self._concurrencychecker(
                     ifh, self.indexfile, curr * self._io.size
                 )
                 self._concurrencychecker(dfh, self.datafile, offset)
 
         p1r, p2r = self.rev(p1), self.rev(p2)
 
         # full versions are inserted when the needed deltas
         # become comparable to the uncompressed text
         if rawtext is None:
             # need rawtext size, before changed by flag processors, which is
             # the non-raw size. use revlog explicitly to avoid filelog's extra
             # logic that might remove metadata size.
             textlen = mdiff.patchedsize(
                 revlog.size(self, cachedelta[0]), cachedelta[1]
             )
         else:
             textlen = len(rawtext)
 
         if deltacomputer is None:
             deltacomputer = deltautil.deltacomputer(self)
 
         revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
 
         deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
 
         if sidedata:
             serialized_sidedata = sidedatautil.serialize_sidedata(sidedata)
             sidedata_offset = offset + deltainfo.deltalen
         else:
             serialized_sidedata = b""
             # Don't store the offset if the sidedata is empty, that way
             # we can easily detect empty sidedata and they will be no different
             # than ones we manually add.
             sidedata_offset = 0
 
         e = (
             offset_type(offset, flags),
             deltainfo.deltalen,
             textlen,
             deltainfo.base,
             link,
             p1r,
             p2r,
             node,
             sidedata_offset,
             len(serialized_sidedata),
         )
 
         if self.version & 0xFFFF != REVLOGV2:
             e = e[:8]
 
         self.index.append(e)
         entry = self._io.packentry(e, self.node, self.version, curr)
         self._writeentry(
             transaction,
             ifh,
             dfh,
             entry,
             deltainfo.data,
             link,
             offset,
             serialized_sidedata,
         )
 
         rawtext = btext[0]
 
         if alwayscache and rawtext is None:
             rawtext = deltacomputer.buildtext(revinfo, fh)
 
         if type(rawtext) == bytes:  # only accept immutable objects
             self._revisioncache = (node, curr, rawtext)
         self._chainbasecache[curr] = deltainfo.chainbase
         return curr
 
     def _get_data_offset(self, prev):
         """Returns the current offset in the (in-transaction) data file.
         Versions < 2 of the revlog can get this 0(1), revlog v2 needs a docket
         file to store that information: since sidedata can be rewritten to the
         end of the data file within a transaction, you can have cases where, for
         example, rev `n` does not have sidedata while rev `n - 1` does, leading
         to `n - 1`'s sidedata being written after `n`'s data.
 
         TODO cache this in a docket file before getting out of experimental."""
         if self.version & 0xFFFF != REVLOGV2:
             return self.end(prev)
 
         offset = 0
         for rev, entry in enumerate(self.index):
             sidedata_end = entry[8] + entry[9]
             # Sidedata for a previous rev has potentially been written after
             # this rev's end, so take the max.
             offset = max(self.end(rev), offset, sidedata_end)
         return offset
 
     def _writeentry(
         self, transaction, ifh, dfh, entry, data, link, offset, sidedata
     ):
         # Files opened in a+ mode have inconsistent behavior on various
         # platforms. Windows requires that a file positioning call be made
         # when the file handle transitions between reads and writes. See
         # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
         # platforms, Python or the platform itself can be buggy. Some versions
         # of Solaris have been observed to not append at the end of the file
         # if the file was seeked to before the end. See issue4943 for more.
         #
         # We work around this issue by inserting a seek() before writing.
         # Note: This is likely not necessary on Python 3. However, because
         # the file handle is reused for reads and may be seeked there, we need
         # to be careful before changing this.
         ifh.seek(0, os.SEEK_END)
         if dfh:
             dfh.seek(0, os.SEEK_END)
 
         curr = len(self) - 1
         if not self._inline:
             transaction.add(self.datafile, offset)
             transaction.add(self.indexfile, curr * len(entry))
             if data[0]:
                 dfh.write(data[0])
             dfh.write(data[1])
             if sidedata:
                 dfh.write(sidedata)
             ifh.write(entry)
         else:
             offset += curr * self._io.size
             transaction.add(self.indexfile, offset)
             ifh.write(entry)
             ifh.write(data[0])
             ifh.write(data[1])
             if sidedata:
                 ifh.write(sidedata)
             self._enforceinlinesize(transaction, ifh)
         nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(
         self,
         deltas,
         linkmapper,
         transaction,
         alwayscache=False,
         addrevisioncb=None,
         duplicaterevisioncb=None,
     ):
         """
         add a delta group
 
         given a set of deltas, add them to the revision log. the
         first delta is against its parent, which should be in our
         log, the rest are against the previous delta.
 
         If ``addrevisioncb`` is defined, it will be called with arguments of
         this revlog and the node that was added.
         """
 
         if self._writinghandles:
             raise error.ProgrammingError(b'cannot nest addgroup() calls')
 
         r = len(self)
         end = 0
         if r:
             end = self.end(r - 1)
         ifh = self._indexfp(b"a+")
         isize = r * self._io.size
         if self._inline:
             transaction.add(self.indexfile, end + isize)
             dfh = None
         else:
             transaction.add(self.indexfile, isize)
             transaction.add(self.datafile, end)
             dfh = self._datafp(b"a+")
 
         def flush():
             if dfh:
                 dfh.flush()
             ifh.flush()
 
         self._writinghandles = (ifh, dfh)
         empty = True
 
         try:
             deltacomputer = deltautil.deltacomputer(self)
             # loop through our set of deltas
             for data in deltas:
-                node, p1, p2, linknode, deltabase, delta, flags = data
+                node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
                 link = linkmapper(linknode)
                 flags = flags or REVIDX_DEFAULT_FLAGS
 
                 rev = self.index.get_rev(node)
                 if rev is not None:
                     # this can happen if two branches make the same change
                     self._nodeduplicatecallback(transaction, rev)
                     if duplicaterevisioncb:
                         duplicaterevisioncb(self, rev)
                     empty = False
                     continue
 
                 for p in (p1, p2):
                     if not self.index.has_node(p):
                         raise error.LookupError(
                             p, self.indexfile, _(b'unknown parent')
                         )
 
                 if not self.index.has_node(deltabase):
                     raise error.LookupError(
                         deltabase, self.indexfile, _(b'unknown delta base')
                     )
 
                 baserev = self.rev(deltabase)
 
                 if baserev != nullrev and self.iscensored(baserev):
                     # if base is censored, delta must be full replacement in a
                     # single patch operation
                     hlen = struct.calcsize(b">lll")
                     oldlen = self.rawsize(baserev)
                     newlen = len(delta) - hlen
                     if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
                         raise error.CensoredBaseError(
                             self.indexfile, self.node(baserev)
                         )
 
                 if not flags and self._peek_iscensored(baserev, delta, flush):
                     flags |= REVIDX_ISCENSORED
 
                 # We assume consumers of addrevisioncb will want to retrieve
                 # the added revision, which will require a call to
                 # revision(). revision() will fast path if there is a cache
                 # hit. So, we tell _addrevision() to always cache in this case.
                 # We're only using addgroup() in the context of changegroup
                 # generation so the revision data can always be handled as raw
                 # by the flagprocessor.
                 rev = self._addrevision(
                     node,
                     None,
                     transaction,
                     link,
                     p1,
                     p2,
                     flags,
                     (baserev, delta),
                     ifh,
                     dfh,
                     alwayscache=alwayscache,
                     deltacomputer=deltacomputer,
                 )
 
                 if addrevisioncb:
                     addrevisioncb(self, rev)
                 empty = False
 
                 if not dfh and not self._inline:
                     # addrevision switched from inline to conventional
                     # reopen the index
                     ifh.close()
                     dfh = self._datafp(b"a+")
                     ifh = self._indexfp(b"a+")
                     self._writinghandles = (ifh, dfh)
         finally:
             self._writinghandles = None
 
             if dfh:
                 dfh.close()
             ifh.close()
         return not empty
 
     def iscensored(self, rev):
         """Check if a file revision is censored."""
         if not self._censorable:
             return False
 
         return self.flags(rev) & REVIDX_ISCENSORED
 
     def _peek_iscensored(self, baserev, delta, flush):
         """Quickly check if a delta produces a censored revision."""
         if not self._censorable:
             return False
 
         return storageutil.deltaiscensored(delta, baserev, self.rawsize)
 
     def getstrippoint(self, minlink):
         """find the minimum rev that must be stripped to strip the linkrev
 
         Returns a tuple containing the minimum rev and a set of all revs that
         have linkrevs that will be broken by this strip.
         """
         return storageutil.resolvestripinfo(
             minlink,
             len(self) - 1,
             self.headrevs(),
             self.linkrev,
             self.parentrevs,
         )
 
     def strip(self, minlink, transaction):
         """truncate the revlog on the first revision with a linkrev >= minlink
 
         This function is called when we're stripping revision minlink and
         its descendants from the repository.
 
         We have to remove all revisions with linkrev >= minlink, because
         the equivalent changelog revisions will be renumbered after the
         strip.
 
         So we truncate the revlog on the first of these revisions, and
         trust that the caller has saved the revisions that shouldn't be
         removed and that it'll re-add them after this truncation.
         """
         if len(self) == 0:
             return
 
         rev, _ = self.getstrippoint(minlink)
         if rev == len(self):
             return
 
         # first truncate the files on disk
         end = self.start(rev)
         if not self._inline:
             transaction.add(self.datafile, end)
             end = rev * self._io.size
         else:
             end += rev * self._io.size
 
         transaction.add(self.indexfile, end)
 
         # then reset internal state in memory to forget those revisions
         self._revisioncache = None
         self._chaininfocache = util.lrucachedict(500)
         self._chunkclear()
 
         del self.index[rev:-1]
 
     def checksize(self):
         """Check size of index and data files
 
         return a (dd, di) tuple.
         - dd: extra bytes for the "data" file
         - di: extra bytes for the "index" file
 
         A healthy revlog will return (0, 0).
         """
         expected = 0
         if len(self):
             expected = max(0, self.end(len(self) - 1))
 
         try:
             with self._datafp() as f:
                 f.seek(0, io.SEEK_END)
                 actual = f.tell()
             dd = actual - expected
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             dd = 0
 
         try:
             f = self.opener(self.indexfile)
             f.seek(0, io.SEEK_END)
             actual = f.tell()
             f.close()
             s = self._io.size
             i = max(0, actual // s)
             di = actual - (i * s)
             if self._inline:
                 databytes = 0
                 for r in self:
                     databytes += max(0, self.length(r))
                 dd = 0
                 di = actual - len(self) * s - databytes
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             di = 0
 
         return (dd, di)
 
     def files(self):
         res = [self.indexfile]
         if not self._inline:
             res.append(self.datafile)
         return res
 
     def emitrevisions(
         self,
         nodes,
         nodesorder=None,
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
     ):
         if nodesorder not in (b'nodes', b'storage', b'linear', None):
             raise error.ProgrammingError(
                 b'unhandled value for nodesorder: %s' % nodesorder
             )
 
         if nodesorder is None and not self._generaldelta:
             nodesorder = b'storage'
 
         if (
             not self._storedeltachains
             and deltamode != repository.CG_DELTAMODE_PREV
         ):
             deltamode = repository.CG_DELTAMODE_FULL
 
         return storageutil.emitrevisions(
             self,
             nodes,
             nodesorder,
             revlogrevisiondelta,
             deltaparentfn=self.deltaparent,
             candeltafn=self.candelta,
             rawsizefn=self.rawsize,
             revdifffn=self.revdiff,
             flagsfn=self.flags,
             deltamode=deltamode,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
         )
 
     DELTAREUSEALWAYS = b'always'
     DELTAREUSESAMEREVS = b'samerevs'
     DELTAREUSENEVER = b'never'
 
     DELTAREUSEFULLADD = b'fulladd'
 
     DELTAREUSEALL = {b'always', b'samerevs', b'never', b'fulladd'}
 
     def clone(
         self,
         tr,
         destrevlog,
         addrevisioncb=None,
         deltareuse=DELTAREUSESAMEREVS,
         forcedeltabothparents=None,
         sidedatacompanion=None,
     ):
         """Copy this revlog to another, possibly with format changes.
 
         The destination revlog will contain the same revisions and nodes.
         However, it may not be bit-for-bit identical due to e.g. delta encoding
         differences.
 
         The ``deltareuse`` argument control how deltas from the existing revlog
         are preserved in the destination revlog. The argument can have the
         following values:
 
         DELTAREUSEALWAYS
            Deltas will always be reused (if possible), even if the destination
            revlog would not select the same revisions for the delta. This is the
            fastest mode of operation.
         DELTAREUSESAMEREVS
            Deltas will be reused if the destination revlog would pick the same
            revisions for the delta. This mode strikes a balance between speed
            and optimization.
         DELTAREUSENEVER
            Deltas will never be reused. This is the slowest mode of execution.
            This mode can be used to recompute deltas (e.g. if the diff/delta
            algorithm changes).
         DELTAREUSEFULLADD
            Revision will be re-added as if their were new content. This is
            slower than DELTAREUSEALWAYS but allow more mechanism to kicks in.
            eg: large file detection and handling.
 
         Delta computation can be slow, so the choice of delta reuse policy can
         significantly affect run time.
 
         The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
         two extremes. Deltas will be reused if they are appropriate. But if the
         delta could choose a better revision, it will do so. This means if you
         are converting a non-generaldelta revlog to a generaldelta revlog,
         deltas will be recomputed if the delta's parent isn't a parent of the
         revision.
 
         In addition to the delta policy, the ``forcedeltabothparents``
         argument controls whether to force compute deltas against both parents
         for merges. By default, the current default is used.
 
         If not None, the `sidedatacompanion` is callable that accept two
         arguments:
 
             (srcrevlog, rev)
 
         and return a quintet that control changes to sidedata content from the
         old revision to the new clone result:
 
             (dropall, filterout, update, new_flags, dropped_flags)
 
         * if `dropall` is True, all sidedata should be dropped
         * `filterout` is a set of sidedata keys that should be dropped
         * `update` is a mapping of additionnal/new key -> value
         * new_flags is a bitfields of new flags that the revision should get
         * dropped_flags is a bitfields of new flags that the revision shoudl not longer have
         """
         if deltareuse not in self.DELTAREUSEALL:
             raise ValueError(
                 _(b'value for deltareuse invalid: %s') % deltareuse
             )
 
         if len(destrevlog):
             raise ValueError(_(b'destination revlog is not empty'))
 
         if getattr(self, 'filteredrevs', None):
             raise ValueError(_(b'source revlog has filtered revisions'))
         if getattr(destrevlog, 'filteredrevs', None):
             raise ValueError(_(b'destination revlog has filtered revisions'))
 
         # lazydelta and lazydeltabase controls whether to reuse a cached delta,
         # if possible.
         oldlazydelta = destrevlog._lazydelta
         oldlazydeltabase = destrevlog._lazydeltabase
         oldamd = destrevlog._deltabothparents
 
         try:
             if deltareuse == self.DELTAREUSEALWAYS:
                 destrevlog._lazydeltabase = True
                 destrevlog._lazydelta = True
             elif deltareuse == self.DELTAREUSESAMEREVS:
                 destrevlog._lazydeltabase = False
                 destrevlog._lazydelta = True
             elif deltareuse == self.DELTAREUSENEVER:
                 destrevlog._lazydeltabase = False
                 destrevlog._lazydelta = False
 
             destrevlog._deltabothparents = forcedeltabothparents or oldamd
 
             self._clone(
                 tr,
                 destrevlog,
                 addrevisioncb,
                 deltareuse,
                 forcedeltabothparents,
                 sidedatacompanion,
             )
 
         finally:
             destrevlog._lazydelta = oldlazydelta
             destrevlog._lazydeltabase = oldlazydeltabase
             destrevlog._deltabothparents = oldamd
 
     def _clone(
         self,
         tr,
         destrevlog,
         addrevisioncb,
         deltareuse,
         forcedeltabothparents,
         sidedatacompanion,
     ):
         """perform the core duty of `revlog.clone` after parameter processing"""
         deltacomputer = deltautil.deltacomputer(destrevlog)
         index = self.index
         for rev in self:
             entry = index[rev]
 
             # Some classes override linkrev to take filtered revs into
             # account. Use raw entry from index.
             flags = entry[0] & 0xFFFF
             linkrev = entry[4]
             p1 = index[entry[5]][7]
             p2 = index[entry[6]][7]
             node = entry[7]
 
             sidedataactions = (False, [], {}, 0, 0)
             if sidedatacompanion is not None:
                 sidedataactions = sidedatacompanion(self, rev)
 
             # (Possibly) reuse the delta from the revlog if allowed and
             # the revlog chunk is a delta.
             cachedelta = None
             rawtext = None
             if any(sidedataactions) or deltareuse == self.DELTAREUSEFULLADD:
                 dropall = sidedataactions[0]
                 filterout = sidedataactions[1]
                 update = sidedataactions[2]
                 new_flags = sidedataactions[3]
                 dropped_flags = sidedataactions[4]
                 text, sidedata = self._revisiondata(rev)
                 if dropall:
                     sidedata = {}
                 for key in filterout:
                     sidedata.pop(key, None)
                 sidedata.update(update)
                 if not sidedata:
                     sidedata = None
 
                 flags |= new_flags
                 flags &= ~dropped_flags
 
                 destrevlog.addrevision(
                     text,
                     tr,
                     linkrev,
                     p1,
                     p2,
                     cachedelta=cachedelta,
                     node=node,
                     flags=flags,
                     deltacomputer=deltacomputer,
                     sidedata=sidedata,
                 )
             else:
                 if destrevlog._lazydelta:
                     dp = self.deltaparent(rev)
                     if dp != nullrev:
                         cachedelta = (dp, bytes(self._chunk(rev)))
 
                 if not cachedelta:
                     rawtext = self.rawdata(rev)
 
                 ifh = destrevlog.opener(
                     destrevlog.indexfile, b'a+', checkambig=False
                 )
                 dfh = None
                 if not destrevlog._inline:
                     dfh = destrevlog.opener(destrevlog.datafile, b'a+')
                 try:
                     destrevlog._addrevision(
                         node,
                         rawtext,
                         tr,
                         linkrev,
                         p1,
                         p2,
                         flags,
                         cachedelta,
                         ifh,
                         dfh,
                         deltacomputer=deltacomputer,
                     )
                 finally:
                     if dfh:
                         dfh.close()
                     ifh.close()
 
             if addrevisioncb:
                 addrevisioncb(self, rev, node)
 
     def censorrevision(self, tr, censornode, tombstone=b''):
         if (self.version & 0xFFFF) == REVLOGV0:
             raise error.RevlogError(
                 _(b'cannot censor with version %d revlogs') % self.version
             )
 
         censorrev = self.rev(censornode)
         tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
 
         if len(tombstone) > self.rawsize(censorrev):
             raise error.Abort(
                 _(b'censor tombstone must be no longer than censored data')
             )
 
         # Rewriting the revlog in place is hard. Our strategy for censoring is
         # to create a new revlog, copy all revisions to it, then replace the
         # revlogs on transaction close.
 
         newindexfile = self.indexfile + b'.tmpcensored'
         newdatafile = self.datafile + b'.tmpcensored'
 
         # This is a bit dangerous. We could easily have a mismatch of state.
         newrl = revlog(self.opener, newindexfile, newdatafile, censorable=True)
         newrl.version = self.version
         newrl._generaldelta = self._generaldelta
         newrl._io = self._io
 
         for rev in self.revs():
             node = self.node(rev)
             p1, p2 = self.parents(node)
 
             if rev == censorrev:
                 newrl.addrawrevision(
                     tombstone,
                     tr,
                     self.linkrev(censorrev),
                     p1,
                     p2,
                     censornode,
                     REVIDX_ISCENSORED,
                 )
 
                 if newrl.deltaparent(rev) != nullrev:
                     raise error.Abort(
                         _(
                             b'censored revision stored as delta; '
                             b'cannot censor'
                         ),
                         hint=_(
                             b'censoring of revlogs is not '
                             b'fully implemented; please report '
                             b'this bug'
                         ),
                     )
                 continue
 
             if self.iscensored(rev):
                 if self.deltaparent(rev) != nullrev:
                     raise error.Abort(
                         _(
                             b'cannot censor due to censored '
                             b'revision having delta stored'
                         )
                     )
                 rawtext = self._chunk(rev)
             else:
                 rawtext = self.rawdata(rev)
 
             newrl.addrawrevision(
                 rawtext, tr, self.linkrev(rev), p1, p2, node, self.flags(rev)
             )
 
         tr.addbackup(self.indexfile, location=b'store')
         if not self._inline:
             tr.addbackup(self.datafile, location=b'store')
 
         self.opener.rename(newrl.indexfile, self.indexfile)
         if not self._inline:
             self.opener.rename(newrl.datafile, self.datafile)
 
         self.clearcaches()
         self._loadindex()
 
     def verifyintegrity(self, state):
         """Verifies the integrity of the revlog.
 
         Yields ``revlogproblem`` instances describing problems that are
         found.
         """
         dd, di = self.checksize()
         if dd:
             yield revlogproblem(error=_(b'data length off by %d bytes') % dd)
         if di:
             yield revlogproblem(error=_(b'index contains %d extra bytes') % di)
 
         version = self.version & 0xFFFF
 
         # The verifier tells us what version revlog we should be.
         if version != state[b'expectedversion']:
             yield revlogproblem(
                 warning=_(b"warning: '%s' uses revlog format %d; expected %d")
                 % (self.indexfile, version, state[b'expectedversion'])
             )
 
         state[b'skipread'] = set()
         state[b'safe_renamed'] = set()
 
         for rev in self:
             node = self.node(rev)
 
             # Verify contents. 4 cases to care about:
             #
             #   common: the most common case
             #   rename: with a rename
             #   meta: file content starts with b'\1\n', the metadata
             #         header defined in filelog.py, but without a rename
             #   ext: content stored externally
             #
             # More formally, their differences are shown below:
             #
             #                       | common | rename | meta  | ext
             #  -------------------------------------------------------
             #   flags()             | 0      | 0      | 0     | not 0
             #   renamed()           | False  | True   | False | ?
             #   rawtext[0:2]=='\1\n'| False  | True   | True  | ?
             #
             # "rawtext" means the raw text stored in revlog data, which
             # could be retrieved by "rawdata(rev)". "text"
             # mentioned below is "revision(rev)".
             #
             # There are 3 different lengths stored physically:
             #  1. L1: rawsize, stored in revlog index
             #  2. L2: len(rawtext), stored in revlog data
             #  3. L3: len(text), stored in revlog data if flags==0, or
             #     possibly somewhere else if flags!=0
             #
             # L1 should be equal to L2. L3 could be different from them.
             # "text" may or may not affect commit hash depending on flag
             # processors (see flagutil.addflagprocessor).
             #
             #              | common  | rename | meta  | ext
             # -------------------------------------------------
             #    rawsize() | L1      | L1     | L1    | L1
             #       size() | L1      | L2-LM  | L1(*) | L1 (?)
             # len(rawtext) | L2      | L2     | L2    | L2
             #    len(text) | L2      | L2     | L2    | L3
             #  len(read()) | L2      | L2-LM  | L2-LM | L3 (?)
             #
             # LM:  length of metadata, depending on rawtext
             # (*): not ideal, see comment in filelog.size
             # (?): could be "- len(meta)" if the resolved content has
             #      rename metadata
             #
             # Checks needed to be done:
             #  1. length check: L1 == L2, in all cases.
             #  2. hash check: depending on flag processor, we may need to
             #     use either "text" (external), or "rawtext" (in revlog).
 
             try:
                 skipflags = state.get(b'skipflags', 0)
                 if skipflags:
                     skipflags &= self.flags(rev)
 
                 _verify_revision(self, skipflags, state, node)
 
                 l1 = self.rawsize(rev)
                 l2 = len(self.rawdata(node))
 
                 if l1 != l2:
                     yield revlogproblem(
                         error=_(b'unpacked size is %d, %d expected') % (l2, l1),
                         node=node,
                     )
 
             except error.CensoredNodeError:
                 if state[b'erroroncensored']:
                     yield revlogproblem(
                         error=_(b'censored file data'), node=node
                     )
                     state[b'skipread'].add(node)
             except Exception as e:
                 yield revlogproblem(
                     error=_(b'unpacking %s: %s')
                     % (short(node), stringutil.forcebytestr(e)),
                     node=node,
                 )
                 state[b'skipread'].add(node)
 
     def storageinfo(
         self,
         exclusivefiles=False,
         sharedfiles=False,
         revisionscount=False,
         trackedsize=False,
         storedsize=False,
     ):
         d = {}
 
         if exclusivefiles:
             d[b'exclusivefiles'] = [(self.opener, self.indexfile)]
             if not self._inline:
                 d[b'exclusivefiles'].append((self.opener, self.datafile))
 
         if sharedfiles:
             d[b'sharedfiles'] = []
 
         if revisionscount:
             d[b'revisionscount'] = len(self)
 
         if trackedsize:
             d[b'trackedsize'] = sum(map(self.rawsize, iter(self)))
 
         if storedsize:
             d[b'storedsize'] = sum(
                 self.opener.stat(path).st_size for path in self.files()
             )
 
         return d

tests/test-check-interfaces.py

@@ -1,287 +1,288 b''
 # Test that certain objects conform to well-defined interfaces.
 
 from __future__ import absolute_import, print_function
 
 from mercurial import encoding
 
 encoding.environ[b'HGREALINTERFACES'] = b'1'
 
 import os
 import subprocess
 import sys
 
 # Only run if tests are run in a repo
 if subprocess.call(
     [sys.executable, '%s/hghave' % os.environ['TESTDIR'], 'test-repo']
 ):
     sys.exit(80)
 
 from mercurial.interfaces import (
     dirstate as intdirstate,
     repository,
 )
 from mercurial.thirdparty.zope import interface as zi
 from mercurial.thirdparty.zope.interface import verify as ziverify
 from mercurial import (
     bundlerepo,
     dirstate,
     filelog,
     httppeer,
     localrepo,
     manifest,
     pycompat,
     revlog,
     sshpeer,
     statichttprepo,
     ui as uimod,
     unionrepo,
     vfs as vfsmod,
     wireprotoserver,
     wireprototypes,
     wireprotov1peer,
     wireprotov2server,
 )
 
 testdir = os.path.dirname(__file__)
 rootdir = pycompat.fsencode(os.path.normpath(os.path.join(testdir, '..')))
 
 sys.path[0:0] = [testdir]
 import simplestorerepo
 
 del sys.path[0]
 
 
 def checkzobject(o, allowextra=False):
     """Verify an object with a zope interface."""
     ifaces = zi.providedBy(o)
     if not ifaces:
         print('%r does not provide any zope interfaces' % o)
         return
 
     # Run zope.interface's built-in verification routine. This verifies that
     # everything that is supposed to be present is present.
     for iface in ifaces:
         ziverify.verifyObject(iface, o)
 
     if allowextra:
         return
 
     # Now verify that the object provides no extra public attributes that
     # aren't declared as part of interfaces.
     allowed = set()
     for iface in ifaces:
         allowed |= set(iface.names(all=True))
 
     public = {a for a in dir(o) if not a.startswith('_')}
 
     for attr in sorted(public - allowed):
         print(
             'public attribute not declared in interfaces: %s.%s'
             % (o.__class__.__name__, attr)
         )
 
 
 # Facilitates testing localpeer.
 class dummyrepo(object):
     def __init__(self):
         self.ui = uimod.ui()
 
     def filtered(self, name):
         pass
 
     def _restrictcapabilities(self, caps):
         pass
 
 
 class dummyopener(object):
     handlers = []
 
 
 # Facilitates testing sshpeer without requiring a server.
 class badpeer(httppeer.httppeer):
     def __init__(self):
         super(badpeer, self).__init__(
             None, None, None, dummyopener(), None, None
         )
         self.badattribute = True
 
     def badmethod(self):
         pass
 
 
 class dummypipe(object):
     def close(self):
         pass
 
     @property
     def closed(self):
         pass
 
 
 def main():
     ui = uimod.ui()
     # Needed so we can open a local repo with obsstore without a warning.
     ui.setconfig(b'experimental', b'evolution.createmarkers', True)
 
     checkzobject(badpeer())
 
     ziverify.verifyClass(repository.ipeerbase, httppeer.httppeer)
     checkzobject(httppeer.httppeer(None, None, None, dummyopener(), None, None))
 
     ziverify.verifyClass(repository.ipeerv2, httppeer.httpv2peer)
     checkzobject(httppeer.httpv2peer(None, b'', b'', None, None, None))
 
     ziverify.verifyClass(repository.ipeerbase, localrepo.localpeer)
     checkzobject(localrepo.localpeer(dummyrepo()))
 
     ziverify.verifyClass(
         repository.ipeercommandexecutor, localrepo.localcommandexecutor
     )
     checkzobject(localrepo.localcommandexecutor(None))
 
     ziverify.verifyClass(
         repository.ipeercommandexecutor, wireprotov1peer.peerexecutor
     )
     checkzobject(wireprotov1peer.peerexecutor(None))
 
     ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv1peer)
     checkzobject(
         sshpeer.sshv1peer(
             ui,
             b'ssh://localhost/foo',
             b'',
             dummypipe(),
             dummypipe(),
             None,
             None,
         )
     )
 
     ziverify.verifyClass(repository.ipeerbase, sshpeer.sshv2peer)
     checkzobject(
         sshpeer.sshv2peer(
             ui,
             b'ssh://localhost/foo',
             b'',
             dummypipe(),
             dummypipe(),
             None,
             None,
         )
     )
 
     ziverify.verifyClass(repository.ipeerbase, bundlerepo.bundlepeer)
     checkzobject(bundlerepo.bundlepeer(dummyrepo()))
 
     ziverify.verifyClass(repository.ipeerbase, statichttprepo.statichttppeer)
     checkzobject(statichttprepo.statichttppeer(dummyrepo()))
 
     ziverify.verifyClass(repository.ipeerbase, unionrepo.unionpeer)
     checkzobject(unionrepo.unionpeer(dummyrepo()))
 
     ziverify.verifyClass(
         repository.ilocalrepositorymain, localrepo.localrepository
     )
     ziverify.verifyClass(
         repository.ilocalrepositoryfilestorage, localrepo.revlogfilestorage
     )
     repo = localrepo.makelocalrepository(ui, rootdir)
     checkzobject(repo)
 
     ziverify.verifyClass(
         wireprototypes.baseprotocolhandler, wireprotoserver.sshv1protocolhandler
     )
     ziverify.verifyClass(
         wireprototypes.baseprotocolhandler, wireprotoserver.sshv2protocolhandler
     )
     ziverify.verifyClass(
         wireprototypes.baseprotocolhandler,
         wireprotoserver.httpv1protocolhandler,
     )
     ziverify.verifyClass(
         wireprototypes.baseprotocolhandler,
         wireprotov2server.httpv2protocolhandler,
     )
 
     sshv1 = wireprotoserver.sshv1protocolhandler(None, None, None)
     checkzobject(sshv1)
     sshv2 = wireprotoserver.sshv2protocolhandler(None, None, None)
     checkzobject(sshv2)
 
     httpv1 = wireprotoserver.httpv1protocolhandler(None, None, None)
     checkzobject(httpv1)
     httpv2 = wireprotov2server.httpv2protocolhandler(None, None)
     checkzobject(httpv2)
 
     ziverify.verifyClass(repository.ifilestorage, filelog.filelog)
     ziverify.verifyClass(repository.imanifestdict, manifest.manifestdict)
     ziverify.verifyClass(repository.imanifestdict, manifest.treemanifest)
     ziverify.verifyClass(
         repository.imanifestrevisionstored, manifest.manifestctx
     )
     ziverify.verifyClass(
         repository.imanifestrevisionwritable, manifest.memmanifestctx
     )
     ziverify.verifyClass(
         repository.imanifestrevisionstored, manifest.treemanifestctx
     )
     ziverify.verifyClass(
         repository.imanifestrevisionwritable, manifest.memtreemanifestctx
     )
     ziverify.verifyClass(repository.imanifestlog, manifest.manifestlog)
     ziverify.verifyClass(repository.imanifeststorage, manifest.manifestrevlog)
 
     ziverify.verifyClass(
         repository.irevisiondelta, simplestorerepo.simplestorerevisiondelta
     )
     ziverify.verifyClass(repository.ifilestorage, simplestorerepo.filestorage)
     ziverify.verifyClass(
         repository.iverifyproblem, simplestorerepo.simplefilestoreproblem
     )
 
     ziverify.verifyClass(intdirstate.idirstate, dirstate.dirstate)
 
     vfs = vfsmod.vfs(b'.')
     fl = filelog.filelog(vfs, b'dummy.i')
     checkzobject(fl, allowextra=True)
 
     # Conforms to imanifestlog.
     ml = manifest.manifestlog(
         vfs, repo, manifest.manifestrevlog(repo.svfs), repo.narrowmatch()
     )
     checkzobject(ml)
     checkzobject(repo.manifestlog)
 
     # Conforms to imanifestrevision.
     mctx = ml[repo[0].manifestnode()]
     checkzobject(mctx)
 
     # Conforms to imanifestrevisionwritable.
     checkzobject(mctx.copy())
 
     # Conforms to imanifestdict.
     checkzobject(mctx.read())
 
     mrl = manifest.manifestrevlog(vfs)
     checkzobject(mrl)
 
     ziverify.verifyClass(repository.irevisiondelta, revlog.revlogrevisiondelta)
 
     rd = revlog.revlogrevisiondelta(
         node=b'',
         p1node=b'',
         p2node=b'',
         basenode=b'',
         linknode=b'',
         flags=b'',
         baserevisionsize=None,
         revision=b'',
+        sidedata=b'',
         delta=None,
     )
     checkzobject(rd)
 
     ziverify.verifyClass(repository.iverifyproblem, revlog.revlogproblem)
     checkzobject(revlog.revlogproblem())
 
 
 main()

tests/test-revlog-raw.py

@@ -1,502 +1,504 b''
 # test revlog interaction about raw data (flagprocessor)
 
 from __future__ import absolute_import, print_function
 
 import collections
 import hashlib
 import sys
 
 from mercurial.node import nullid
 from mercurial import (
     encoding,
     revlog,
     transaction,
     vfs,
 )
 
 from mercurial.revlogutils import (
     deltas,
     flagutil,
 )
 
 # TESTTMP is optional. This makes it convenient to run without run-tests.py
 tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp'))
 
 # Enable generaldelta otherwise revlog won't use delta as expected by the test
 tvfs.options = {
     b'generaldelta': True,
     b'revlogv1': True,
     b'sparse-revlog': True,
 }
 
 # The test wants to control whether to use delta explicitly, based on
 # "storedeltachains".
 revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self._storedeltachains
 
 
 def abort(msg):
     print('abort: %s' % msg)
     # Return 0 so run-tests.py could compare the output.
     sys.exit()
 
 
 # Register a revlog processor for flag EXTSTORED.
 #
 # It simply prepends a fixed header, and replaces '1' to 'i'. So it has
 # insertion and replacement, and may be interesting to test revlog's line-based
 # deltas.
 _extheader = b'E\n'
 
 
 def readprocessor(self, rawtext):
     # True: the returned text could be used to verify hash
     text = rawtext[len(_extheader) :].replace(b'i', b'1')
     return text, True
 
 
 def writeprocessor(self, text):
     # False: the returned rawtext shouldn't be used to verify hash
     rawtext = _extheader + text.replace(b'1', b'i')
     return rawtext, False
 
 
 def rawprocessor(self, rawtext):
     # False: do not verify hash. Only the content returned by "readprocessor"
     # can be used to verify hash.
     return False
 
 
 flagutil.addflagprocessor(
     revlog.REVIDX_EXTSTORED, (readprocessor, writeprocessor, rawprocessor)
 )
 
 # Utilities about reading and appending revlog
 
 
 def newtransaction():
     # A transaction is required to write revlogs
     report = lambda msg: None
     return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
 
 
 def newrevlog(name=b'_testrevlog.i', recreate=False):
     if recreate:
         tvfs.tryunlink(name)
     rlog = revlog.revlog(tvfs, name)
     return rlog
 
 
 def appendrev(rlog, text, tr, isext=False, isdelta=True):
     """Append a revision. If isext is True, set the EXTSTORED flag so flag
     processor will be used (and rawtext is different from text). If isdelta is
     True, force the revision to be a delta, otherwise it's full text.
     """
     nextrev = len(rlog)
     p1 = rlog.node(nextrev - 1)
     p2 = nullid
     if isext:
         flags = revlog.REVIDX_EXTSTORED
     else:
         flags = revlog.REVIDX_DEFAULT_FLAGS
     # Change storedeltachains temporarily, to override revlog's delta decision
     rlog._storedeltachains = isdelta
     try:
         rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags)
         return nextrev
     except Exception as ex:
         abort('rev %d: failed to append: %s' % (nextrev, ex))
     finally:
         # Restore storedeltachains. It is always True, see revlog.__init__
         rlog._storedeltachains = True
 
 
 def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True):
     """Copy revlog to destname using revlog.addgroup. Return the copied revlog.
 
     This emulates push or pull. They use changegroup. Changegroup requires
     repo to work. We don't have a repo, so a dummy changegroup is used.
 
     If optimaldelta is True, use optimized delta parent, so the destination
     revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
     the destination revlog needs more work to use it.
 
     This exercises some revlog.addgroup (and revlog._addrevision(text=None))
     code path, which is not covered by "appendrev" alone.
     """
 
     class dummychangegroup(object):
         @staticmethod
         def deltachunk(pnode):
             pnode = pnode or nullid
             parentrev = rlog.rev(pnode)
             r = parentrev + 1
             if r >= len(rlog):
                 return {}
             if optimaldelta:
                 deltaparent = parentrev
             else:
                 # suboptimal deltaparent
                 deltaparent = min(0, parentrev)
             if not rlog.candelta(deltaparent, r):
                 deltaparent = -1
             return {
                 b'node': rlog.node(r),
                 b'p1': pnode,
                 b'p2': nullid,
                 b'cs': rlog.node(rlog.linkrev(r)),
                 b'flags': rlog.flags(r),
                 b'deltabase': rlog.node(deltaparent),
                 b'delta': rlog.revdiff(deltaparent, r),
+                b'sidedata': rlog.sidedata(r),
             }
 
         def deltaiter(self):
             chain = None
             for chunkdata in iter(lambda: self.deltachunk(chain), {}):
                 node = chunkdata[b'node']
                 p1 = chunkdata[b'p1']
                 p2 = chunkdata[b'p2']
                 cs = chunkdata[b'cs']
                 deltabase = chunkdata[b'deltabase']
                 delta = chunkdata[b'delta']
                 flags = chunkdata[b'flags']
+                sidedata = chunkdata[b'sidedata']
 
                 chain = node
 
-                yield (node, p1, p2, cs, deltabase, delta, flags)
+                yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def linkmap(lnode):
         return rlog.rev(lnode)
 
     dlog = newrevlog(destname, recreate=True)
     dummydeltas = dummychangegroup().deltaiter()
     dlog.addgroup(dummydeltas, linkmap, tr)
     return dlog
 
 
 def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'):
     """Like addgroupcopy, but use the low level revlog._addrevision directly.
 
     It exercises some code paths that are hard to reach easily otherwise.
     """
     dlog = newrevlog(destname, recreate=True)
     for r in rlog:
         p1 = rlog.node(r - 1)
         p2 = nullid
         if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED):
             text = rlog.rawdata(r)
             cachedelta = None
         else:
             # deltaparent cannot have EXTSTORED flag.
             deltaparent = max(
                 [-1]
                 + [
                     p
                     for p in range(r)
                     if rlog.flags(p) & revlog.REVIDX_EXTSTORED == 0
                 ]
             )
             text = None
             cachedelta = (deltaparent, rlog.revdiff(deltaparent, r))
         flags = rlog.flags(r)
         ifh = dfh = None
         try:
             ifh = dlog.opener(dlog.indexfile, b'a+')
             if not dlog._inline:
                 dfh = dlog.opener(dlog.datafile, b'a+')
             dlog._addrevision(
                 rlog.node(r), text, tr, r, p1, p2, flags, cachedelta, ifh, dfh
             )
         finally:
             if dfh is not None:
                 dfh.close()
             if ifh is not None:
                 ifh.close()
     return dlog
 
 
 # Utilities to generate revisions for testing
 
 
 def genbits(n):
     """Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n).
     i.e. the generated numbers have a width of n bits.
 
     The combination of two adjacent numbers will cover all possible cases.
     That is to say, given any x, y where both x, and y are in range(2 ** n),
     there is an x followed immediately by y in the generated sequence.
     """
     m = 2 ** n
 
     # Gray Code. See https://en.wikipedia.org/wiki/Gray_code
     gray = lambda x: x ^ (x >> 1)
     reversegray = {gray(i): i for i in range(m)}
 
     # Generate (n * 2) bit gray code, yield lower n bits as X, and look for
     # the next unused gray code where higher n bits equal to X.
 
     # For gray codes whose higher bits are X, a[X] of them have been used.
     a = [0] * m
 
     # Iterate from 0.
     x = 0
     yield x
     for i in range(m * m):
         x = reversegray[x]
         y = gray(a[x] + x * m) & (m - 1)
         assert a[x] < m
         a[x] += 1
         x = y
         yield x
 
 
 def gentext(rev):
     '''Given a revision number, generate dummy text'''
     return b''.join(b'%d\n' % j for j in range(-1, rev % 5))
 
 
 def writecases(rlog, tr):
     """Write some revisions interested to the test.
 
     The test is interested in 3 properties of a revision:
 
         - Is it a delta or a full text? (isdelta)
           This is to catch some delta application issues.
         - Does it have a flag of EXTSTORED? (isext)
           This is to catch some flag processor issues. Especially when
           interacted with revlog deltas.
         - Is its text empty? (isempty)
           This is less important. It is intended to try to catch some careless
           checks like "if text" instead of "if text is None". Note: if flag
           processor is involved, raw text may be not empty.
 
     Write 65 revisions. So that all combinations of the above flags for
     adjacent revisions are covered. That is to say,
 
         len(set(
             (r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty)
             for r in range(len(rlog) - 1)
            )) is 64.
 
     Where "r.delta", "r.ext", and "r.empty" are booleans matching properties
     mentioned above.
 
     Return expected [(text, rawtext)].
     """
     result = []
     for i, x in enumerate(genbits(3)):
         isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4)
         if isempty:
             text = b''
         else:
             text = gentext(i)
         rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
 
         # Verify text, rawtext, and rawsize
         if isext:
             rawtext = writeprocessor(None, text)[0]
         else:
             rawtext = text
         if rlog.rawsize(rev) != len(rawtext):
             abort('rev %d: wrong rawsize' % rev)
         if rlog.revision(rev, raw=False) != text:
             abort('rev %d: wrong text' % rev)
         if rlog.rawdata(rev) != rawtext:
             abort('rev %d: wrong rawtext' % rev)
         result.append((text, rawtext))
 
         # Verify flags like isdelta, isext work as expected
         # isdelta can be overridden to False if this or p1 has isext set
         if bool(rlog.deltaparent(rev) > -1) and not isdelta:
             abort('rev %d: isdelta is unexpected' % rev)
         if bool(rlog.flags(rev)) != isext:
             abort('rev %d: isext is ineffective' % rev)
     return result
 
 
 # Main test and checking
 
 
 def checkrevlog(rlog, expected):
     '''Check if revlog has expected contents. expected is [(text, rawtext)]'''
     # Test using different access orders. This could expose some issues
     # depending on revlog caching (see revlog._cache).
     for r0 in range(len(rlog) - 1):
         r1 = r0 + 1
         for revorder in [[r0, r1], [r1, r0]]:
             for raworder in [[True], [False], [True, False], [False, True]]:
                 nlog = newrevlog()
                 for rev in revorder:
                     for raw in raworder:
                         if raw:
                             t = nlog.rawdata(rev)
                         else:
                             t = nlog.revision(rev)
                         if t != expected[rev][int(raw)]:
                             abort(
                                 'rev %d: corrupted %stext'
                                 % (rev, raw and 'raw' or '')
                             )
 
 
 slicingdata = [
     ([0, 1, 2, 3, 55, 56, 58, 59, 60], [[0, 1], [2], [58], [59, 60]], 10),
     ([0, 1, 2, 3, 55, 56, 58, 59, 60], [[0, 1], [2], [58], [59, 60]], 10),
     (
         [-1, 0, 1, 2, 3, 55, 56, 58, 59, 60],
         [[-1, 0, 1], [2], [58], [59, 60]],
         10,
     ),
 ]
 
 
 def slicingtest(rlog):
     oldmin = rlog._srmingapsize
     try:
         # the test revlog is small, we remove the floor under which we
         # slicing is diregarded.
         rlog._srmingapsize = 0
         for item in slicingdata:
             chain, expected, target = item
             result = deltas.slicechunk(rlog, chain, targetsize=target)
             result = list(result)
             if result != expected:
                 print('slicing differ:')
                 print('  chain: %s' % chain)
                 print('  target: %s' % target)
                 print('  expected: %s' % expected)
                 print('  result:   %s' % result)
     finally:
         rlog._srmingapsize = oldmin
 
 
 def md5sum(s):
     return hashlib.md5(s).digest()
 
 
 def _maketext(*coord):
     """create piece of text according to range of integers
 
     The test returned use a md5sum of the integer to make it less
     compressible"""
     pieces = []
     for start, size in coord:
         num = range(start, start + size)
         p = [md5sum(b'%d' % r) for r in num]
         pieces.append(b'\n'.join(p))
     return b'\n'.join(pieces) + b'\n'
 
 
 data = [
     _maketext((0, 120), (456, 60)),
     _maketext((0, 120), (345, 60)),
     _maketext((0, 120), (734, 60)),
     _maketext((0, 120), (734, 60), (923, 45)),
     _maketext((0, 120), (734, 60), (234, 45)),
     _maketext((0, 120), (734, 60), (564, 45)),
     _maketext((0, 120), (734, 60), (361, 45)),
     _maketext((0, 120), (734, 60), (489, 45)),
     _maketext((0, 120), (123, 60)),
     _maketext((0, 120), (145, 60)),
     _maketext((0, 120), (104, 60)),
     _maketext((0, 120), (430, 60)),
     _maketext((0, 120), (430, 60), (923, 45)),
     _maketext((0, 120), (430, 60), (234, 45)),
     _maketext((0, 120), (430, 60), (564, 45)),
     _maketext((0, 120), (430, 60), (361, 45)),
     _maketext((0, 120), (430, 60), (489, 45)),
     _maketext((0, 120), (249, 60)),
     _maketext((0, 120), (832, 60)),
     _maketext((0, 120), (891, 60)),
     _maketext((0, 120), (543, 60)),
     _maketext((0, 120), (120, 60)),
     _maketext((0, 120), (60, 60), (768, 30)),
     _maketext((0, 120), (60, 60), (260, 30)),
     _maketext((0, 120), (60, 60), (450, 30)),
     _maketext((0, 120), (60, 60), (361, 30)),
     _maketext((0, 120), (60, 60), (886, 30)),
     _maketext((0, 120), (60, 60), (116, 30)),
     _maketext((0, 120), (60, 60), (567, 30), (629, 40)),
     _maketext((0, 120), (60, 60), (569, 30), (745, 40)),
     _maketext((0, 120), (60, 60), (777, 30), (700, 40)),
     _maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)),
 ]
 
 
 def makesnapshot(tr):
     rl = newrevlog(name=b'_snaprevlog3.i', recreate=True)
     for i in data:
         appendrev(rl, i, tr)
     return rl
 
 
 snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30]
 
 
 def issnapshottest(rlog):
     result = []
     if rlog.issnapshot(-1):
         result.append(-1)
     for rev in rlog:
         if rlog.issnapshot(rev):
             result.append(rev)
     if snapshots != result:
         print('snapshot differ:')
         print('  expected: %s' % snapshots)
         print('  got:      %s' % result)
 
 
 snapshotmapall = {0: [6, 8, 11, 17, 19, 25], 8: [21], -1: [0, 30]}
 snapshotmap15 = {0: [17, 19, 25], 8: [21], -1: [30]}
 
 
 def findsnapshottest(rlog):
     resultall = collections.defaultdict(list)
     deltas._findsnapshots(rlog, resultall, 0)
     resultall = dict(resultall.items())
     if resultall != snapshotmapall:
         print('snapshot map  differ:')
         print('  expected: %s' % snapshotmapall)
         print('  got:      %s' % resultall)
     result15 = collections.defaultdict(list)
     deltas._findsnapshots(rlog, result15, 15)
     result15 = dict(result15.items())
     if result15 != snapshotmap15:
         print('snapshot map  differ:')
         print('  expected: %s' % snapshotmap15)
         print('  got:      %s' % result15)
 
 
 def maintest():
     with newtransaction() as tr:
         rl = newrevlog(recreate=True)
         expected = writecases(rl, tr)
         checkrevlog(rl, expected)
         print('local test passed')
         # Copy via revlog.addgroup
         rl1 = addgroupcopy(rl, tr)
         checkrevlog(rl1, expected)
         rl2 = addgroupcopy(rl, tr, optimaldelta=False)
         checkrevlog(rl2, expected)
         print('addgroupcopy test passed')
         # Copy via revlog.clone
         rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True)
         rl.clone(tr, rl3)
         checkrevlog(rl3, expected)
         print('clone test passed')
         # Copy via low-level revlog._addrevision
         rl4 = lowlevelcopy(rl, tr)
         checkrevlog(rl4, expected)
         print('lowlevelcopy test passed')
         slicingtest(rl)
         print('slicing test passed')
         rl5 = makesnapshot(tr)
         issnapshottest(rl5)
         print('issnapshot test passed')
         findsnapshottest(rl5)
         print('findsnapshot test passed')
 
 
 try:
     maintest()
 except Exception as ex:
     abort('crashed: %s' % ex)