sidedata-exchange: add `wanted_sidedata` and `sidedata_computers` to repos

Each repo will advertise the sidedata categories it requires (categories being
unique and canonical), and have a set of "computers", functions to generate
sidedata from `(repo, revlog, rev, previous_sidedata)`, for a given category.
The set of computers can be a superset of the set of the wanted categories, but
not smaller: repos are expected to be coherent in their handling of sidedata.

Differential Revision: https://phab.mercurial-scm.org/D10028

mercurial/bundle2.py

@@ -1812,6 +1812,28 b' def _formatrequirementsparams(requiremen'
     return params
 
 
+def format_remote_wanted_sidedata(repo):
+    """Formats a repo's wanted sidedata categories into a bytestring for
+    capabilities exchange."""
+    wanted = b""
+    if repo._wanted_sidedata:
+        wanted = b','.join(
+            pycompat.bytestr(c) for c in sorted(repo._wanted_sidedata)
+        )
+    return wanted
+
+
+def read_remote_wanted_sidedata(remote):
+    sidedata_categories = remote.capable(b'exp-wanted-sidedata')
+    return read_wanted_sidedata(sidedata_categories)
+
+
+def read_wanted_sidedata(formatted):
+    if formatted:
+        return set(formatted.split(b','))
+    return set()
+
+
 def addpartbundlestream2(bundler, repo, **kwargs):
     if not kwargs.get('stream', False):
         return
@@ -1957,6 +1979,7 b' def combinechangegroupresults(op):'
         b'version',
         b'nbchanges',
         b'exp-sidedata',
+        b'exp-wanted-sidedata',
         b'treemanifest',
         b'targetphase',
     ),
@@ -1999,6 +2022,10 b' def handlechangegroup(op, inpart):'
     targetphase = inpart.params.get(b'targetphase')
     if targetphase is not None:
         extrakwargs['targetphase'] = int(targetphase)
+
+    remote_sidedata = inpart.params.get(b'exp-wanted-sidedata')
+    extrakwargs['sidedata_categories'] = read_wanted_sidedata(remote_sidedata)
+
     ret = _processchangegroup(
         op,
         cg,
@@ -2559,5 +2586,7 b' def widen_bundle('
             part.addparam(b'treemanifest', b'1')
         if b'exp-sidedata-flag' in repo.requirements:
             part.addparam(b'exp-sidedata', b'1')
+            wanted = format_remote_wanted_sidedata(repo)
+            part.addparam(b'exp-wanted-sidedata', wanted)
 
     return bundler

mercurial/changegroup.py

@@ -945,6 +945,9 b' class cgpacker(object):'
         if bundlecaps is None:
             bundlecaps = set()
         self._bundlecaps = bundlecaps
+        if remote_sidedata is None:
+            remote_sidedata = set()
+        self._remote_sidedata = remote_sidedata
         self._isshallow = shallow
         self._fullclnodes = fullnodes
 

mercurial/exchange.py

@@ -420,7 +420,20 b' def push('
                 b'unbundle wire protocol command'
             )
         )
-
+    for category in sorted(bundle2.read_remote_wanted_sidedata(pushop.remote)):
+        # Check that a computer is registered for that category for at least
+        # one revlog kind.
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            raise error.Abort(
+                _(
+                    b'cannot push: required sidedata category not supported'
+                    b" by this client: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
     # get lock as we might write phase data
     wlock = lock = None
     try:
@@ -865,8 +878,15 b' def _pushb2ctx(pushop, bundler):'
         if not cgversions:
             raise error.Abort(_(b'no common changegroup version'))
         version = max(cgversions)
+
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pushop.remote)
     cgstream = changegroup.makestream(
-        pushop.repo, pushop.outgoing, version, b'push'
+        pushop.repo,
+        pushop.outgoing,
+        version,
+        b'push',
+        bundlecaps=b2caps,
+        remote_sidedata=remote_sidedata,
     )
     cgpart = bundler.newpart(b'changegroup', data=cgstream)
     if cgversions:
@@ -1607,6 +1627,23 b' def pull('
             ) % (b', '.join(sorted(missing)))
             raise error.Abort(msg)
 
+    for category in repo._wanted_sidedata:
+        # Check that a computer is registered for that category for at least
+        # one revlog kind.
+        for kind, computers in repo._sidedata_computers.items():
+            if computers.get(category):
+                break
+        else:
+            # This should never happen since repos are supposed to be able to
+            # generate the sidedata they require.
+            raise error.ProgrammingError(
+                _(
+                    b'sidedata category requested by local side without local'
+                    b"support: '%s'"
+                )
+                % pycompat.bytestr(category)
+            )
+
     pullop.trmanager = transactionmanager(repo, b'pull', remote.url())
     wlock = util.nullcontextmanager()
     if not bookmod.bookmarksinstore(repo):
@@ -1820,6 +1857,10 b' def _pullbundle2(pullop):'
             pullop.stepsdone.add(b'obsmarkers')
     _pullbundle2extraprepare(pullop, kwargs)
 
+    remote_sidedata = bundle2.read_remote_wanted_sidedata(pullop.remote)
+    if remote_sidedata:
+        kwargs[b'remote_sidedata'] = remote_sidedata
+
     with pullop.remote.commandexecutor() as e:
         args = dict(kwargs)
         args[b'source'] = b'pull'
@@ -2388,6 +2429,8 b' def _getbundlechangegrouppart('
 
     if b'exp-sidedata-flag' in repo.requirements:
         part.addparam(b'exp-sidedata', b'1')
+        sidedata = bundle2.format_remote_wanted_sidedata(repo)
+        part.addparam(b'exp-wanted-sidedata', sidedata)
 
     if (
         kwargs.get('narrow', False)

mercurial/interfaces/repository.py

@@ -1832,6 +1832,12 b' class ilocalrepositorymain(interfaceutil'
     def savecommitmessage(text):
         pass
 
+    def register_sidedata_computer(kind, category, keys, computer):
+        pass
+
+    def register_wanted_sidedata(category):
+        pass
+
 
 class completelocalrepository(
     ilocalrepositorymain, ilocalrepositoryfilestorage

mercurial/localrepo.py

@@ -49,6 +49,7 b' from . import ('
     match as matchmod,
     mergestate as mergestatemod,
     mergeutil,
+    metadata as metadatamod,
     namespaces,
     narrowspec,
     obsolete,
@@ -273,6 +274,11 b' class localpeer(repository.peer):'
             caps = moderncaps.copy()
         self._repo = repo.filtered(b'served')
         self.ui = repo.ui
+
+        if repo._wanted_sidedata:
+            formatted = bundle2.format_remote_wanted_sidedata(repo)
+            caps.add(b'exp-wanted-sidedata=' + formatted)
+
         self._caps = repo._restrictcapabilities(caps)
 
     # Begin of _basepeer interface.
@@ -1395,6 +1401,10 b' class localrepository(object):'
         if requirementsmod.COPIESSDC_REQUIREMENT in self.requirements:
             self.filecopiesmode = b'changeset-sidedata'
 
+        self._wanted_sidedata = set()
+        self._sidedata_computers = {}
+        metadatamod.set_sidedata_spec_for_repo(self)
+
     def _getvfsward(self, origfunc):
         """build a ward for self.vfs"""
         rref = weakref.ref(self)
@@ -3332,6 +3342,22 b' class localrepository(object):'
             fp.close()
         return self.pathto(fp.name[len(self.root) + 1 :])
 
+    def register_wanted_sidedata(self, category):
+        self._wanted_sidedata.add(pycompat.bytestr(category))
+
+    def register_sidedata_computer(self, kind, category, keys, computer):
+        if kind not in (b"changelog", b"manifest", b"filelog"):
+            msg = _(b"unexpected revlog kind '%s'.")
+            raise error.ProgrammingError(msg % kind)
+        category = pycompat.bytestr(category)
+        if category in self._sidedata_computers.get(kind, []):
+            msg = _(
+                b"cannot register a sidedata computer twice for category '%s'."
+            )
+            raise error.ProgrammingError(msg % category)
+        self._sidedata_computers.setdefault(kind, {})
+        self._sidedata_computers[kind][category] = (keys, computer)
+
 
 # used to avoid circular references so destructors work
 def aftertrans(files):

mercurial/metadata.py

@@ -18,6 +18,7 b' from .node import ('
 from . import (
     error,
     pycompat,
+    requirements as requirementsmod,
     util,
 )
 
@@ -804,6 +805,21 b' def _getsidedata(srcrepo, rev):'
     return encode_files_sidedata(files), files.has_copies_info
 
 
+def copies_sidedata_computer(repo, revlog, rev, existing_sidedata):
+    return _getsidedata(repo, rev)[0]
+
+
+def set_sidedata_spec_for_repo(repo):
+    if requirementsmod.COPIESSDC_REQUIREMENT in repo.requirements:
+        repo.register_wanted_sidedata(sidedatamod.SD_FILES)
+        repo.register_sidedata_computer(
+            b"changelog",
+            sidedatamod.SD_FILES,
+            (sidedatamod.SD_FILES,),
+            copies_sidedata_computer,
+        )
+
+
 def getsidedataadder(srcrepo, destrepo):
     use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade')
     if pycompat.iswindows or not use_w:

mercurial/statichttprepo.py

@@ -172,6 +172,7 b' class statichttprepository('
         self.names = namespaces.namespaces()
         self.filtername = None
         self._extrafilterid = None
+        self._wanted_sidedata = set()
 
         try:
             requirements = set(self.vfs.read(b'requires').splitlines())

tests/test-check-interfaces.py

@@ -85,6 +85,7 b' def checkzobject(o, allowextra=False):'
 class dummyrepo(object):
     def __init__(self):
         self.ui = uimod.ui()
+        self._wanted_sidedata = set()
 
     def filtered(self, name):
         pass