changegroupv4: add sidedata helpers

These helpers carry the information and computers needed to rewrite sidedata
when generating/applying patches. We will be making use of them soon.

Differential Revision: https://phab.mercurial-scm.org/D10029

hgext/sqlitestore.py

@@ -587,6 +587,7 b' class sqlitefilestore(object):'
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
+        sidedata_helpers=None,
     ):
         if nodesorder not in (b'nodes', b'storage', b'linear', None):
             raise error.ProgrammingError(
@@ -625,6 +626,7 b' class sqlitefilestore(object):'
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
+            sidedata_helpers=sidedata_helpers,
         ):
 
             yield delta

mercurial/changegroup.py

@@ -272,6 +272,7 b' class cg1unpacker(object):'
         url,
         targetphase=phases.draft,
         expectedtotal=None,
+        sidedata_categories=None,
     ):
         """Add the changegroup returned by source.read() to this repo.
         srctype is a string like 'push', 'pull', or 'unbundle'.  url is
@@ -282,9 +283,23 b' class cg1unpacker(object):'
         - more heads than before: 1+added heads (2..n)
         - fewer heads than before: -1-removed heads (-2..-n)
         - number of heads stays the same: 1
+
+        `sidedata_categories` is an optional set of the remote's sidedata wanted
+        categories.
         """
         repo = repo.unfiltered()
 
+        # Only useful if we're adding sidedata categories. If both peers have
+        # the same categories, then we simply don't do anything.
+        if self.version == b'04' and srctype == b'pull':
+            sidedata_helpers = get_sidedata_helpers(
+                repo,
+                sidedata_categories or set(),
+                pull=True,
+            )
+        else:
+            sidedata_helpers = None
+
         def csmap(x):
             repo.ui.debug(b"add changeset %s\n" % short(x))
             return len(cl)
@@ -749,6 +764,7 b' def deltagroup('
     clrevtolocalrev=None,
     fullclnodes=None,
     precomputedellipsis=None,
+    sidedata_helpers=None,
 ):
     """Calculate deltas for a set of revisions.
 
@@ -756,6 +772,8 b' def deltagroup('
 
     If topic is not None, progress detail will be generated using this
     topic name (e.g. changesets, manifests, etc).
+
+    See `storageutil.emitrevisions` for the doc on `sidedata_helpers`.
     """
     if not nodes:
         return
@@ -854,6 +872,7 b' def deltagroup('
         revisiondata=True,
         assumehaveparentrevisions=not ellipses,
         deltamode=deltamode,
+        sidedata_helpers=sidedata_helpers,
     )
 
     for i, revision in enumerate(revisions):
@@ -974,8 +993,21 b' class cgpacker(object):'
         self._verbosenote(_(b'uncompressed size of bundle content:\n'))
         size = 0
 
+        sidedata_helpers = None
+        if self.version == b'04':
+            remote_sidedata = self._remote_sidedata
+            if source == b'strip':
+                # We're our own remote when stripping, get the no-op helpers
+                # TODO a better approach would be for the strip bundle to
+                # correctly advertise its sidedata categories directly.
+                remote_sidedata = repo._wanted_sidedata
+            sidedata_helpers = get_sidedata_helpers(repo, remote_sidedata)
+
         clstate, deltas = self._generatechangelog(
-            cl, clnodes, generate=changelog
+            cl,
+            clnodes,
+            generate=changelog,
+            sidedata_helpers=sidedata_helpers,
         )
         for delta in deltas:
             for chunk in _revisiondeltatochunks(delta, self._builddeltaheader):
@@ -1023,6 +1055,7 b' class cgpacker(object):'
             fnodes,
             source,
             clstate[b'clrevtomanifestrev'],
+            sidedata_helpers=sidedata_helpers,
         )
 
         for tree, deltas in it:
@@ -1063,6 +1096,7 b' class cgpacker(object):'
             fastpathlinkrev,
             fnodes,
             clrevs,
+            sidedata_helpers=sidedata_helpers,
         )
 
         for path, deltas in it:
@@ -1087,7 +1121,9 b' class cgpacker(object):'
         if clnodes:
             repo.hook(b'outgoing', node=hex(clnodes[0]), source=source)
 
-    def _generatechangelog(self, cl, nodes, generate=True):
+    def _generatechangelog(
+        self, cl, nodes, generate=True, sidedata_helpers=None
+    ):
         """Generate data for changelog chunks.
 
         Returns a 2-tuple of a dict containing state and an iterable of
@@ -1096,6 +1132,8 b' class cgpacker(object):'
 
         if generate is False, the state will be fully populated and no chunk
         stream will be yielded
+
+        See `storageutil.emitrevisions` for the doc on `sidedata_helpers`.
         """
         clrevorder = {}
         manifests = {}
@@ -1179,6 +1217,7 b' class cgpacker(object):'
             clrevtolocalrev={},
             fullclnodes=self._fullclnodes,
             precomputedellipsis=self._precomputedellipsis,
+            sidedata_helpers=sidedata_helpers,
         )
 
         return state, gen
@@ -1192,11 +1231,14 b' class cgpacker(object):'
         fnodes,
         source,
         clrevtolocalrev,
+        sidedata_helpers=None,
     ):
         """Returns an iterator of changegroup chunks containing manifests.
 
         `source` is unused here, but is used by extensions like remotefilelog to
         change what is sent based in pulls vs pushes, etc.
+
+        See `storageutil.emitrevisions` for the doc on `sidedata_helpers`.
         """
         repo = self._repo
         mfl = repo.manifestlog
@@ -1285,6 +1327,7 b' class cgpacker(object):'
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis,
+                sidedata_helpers=sidedata_helpers,
             )
 
             if not self._oldmatcher.visitdir(store.tree[:-1]):
@@ -1323,6 +1366,7 b' class cgpacker(object):'
         fastpathlinkrev,
         fnodes,
         clrevs,
+        sidedata_helpers=None,
     ):
         changedfiles = [
             f
@@ -1417,6 +1461,7 b' class cgpacker(object):'
                 clrevtolocalrev=clrevtolocalrev,
                 fullclnodes=self._fullclnodes,
                 precomputedellipsis=self._precomputedellipsis,
+                sidedata_helpers=sidedata_helpers,
             )
 
             yield fname, deltas
@@ -1792,3 +1837,25 b' def _addchangegroupfiles(repo, source, r'
                 )
 
     return revisions, files
+
+
+def get_sidedata_helpers(repo, remote_sd_categories, pull=False):
+    # Computers for computing sidedata on-the-fly
+    sd_computers = collections.defaultdict(list)
+    # Computers for categories to remove from sidedata
+    sd_removers = collections.defaultdict(list)
+
+    to_generate = remote_sd_categories - repo._wanted_sidedata
+    to_remove = repo._wanted_sidedata - remote_sd_categories
+    if pull:
+        to_generate, to_remove = to_remove, to_generate
+
+    for revlog_kind, computers in repo._sidedata_computers.items():
+        for category, computer in computers.items():
+            if category in to_generate:
+                sd_computers[revlog_kind].append(computer)
+            if category in to_remove:
+                sd_removers[revlog_kind].append(computer)
+
+    sidedata_helpers = (repo, sd_computers, sd_removers)
+    return sidedata_helpers

mercurial/filelog.py

@@ -103,6 +103,7 b' class filelog(object):'
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
+        sidedata_helpers=None,
     ):
         return self._revlog.emitrevisions(
             nodes,
@@ -110,6 +111,7 b' class filelog(object):'
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
+            sidedata_helpers=sidedata_helpers,
         )
 
     def addrevision(

mercurial/manifest.py

@@ -1826,6 +1826,7 b' class manifestrevlog(object):'
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
+        sidedata_helpers=None,
     ):
         return self._revlog.emitrevisions(
             nodes,
@@ -1833,6 +1834,7 b' class manifestrevlog(object):'
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
+            sidedata_helpers=sidedata_helpers,
         )
 
     def addgroup(

mercurial/revlog.py

@@ -2733,6 +2733,7 b' class revlog(object):'
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
+        sidedata_helpers=None,
     ):
         if nodesorder not in (b'nodes', b'storage', b'linear', None):
             raise error.ProgrammingError(
@@ -2761,6 +2762,7 b' class revlog(object):'
             deltamode=deltamode,
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
+            sidedata_helpers=sidedata_helpers,
         )
 
     DELTAREUSEALWAYS = b'always'

mercurial/utils/storageutil.py

@@ -23,6 +23,7 b' from .. import ('
     pycompat,
 )
 from ..interfaces import repository
+from ..revlogutils import sidedata as sidedatamod
 from ..utils import hashutil
 
 _nullhash = hashutil.sha1(nullid)
@@ -294,6 +295,7 b' def emitrevisions('
     deltamode=repository.CG_DELTAMODE_STD,
     revisiondata=False,
     assumehaveparentrevisions=False,
+    sidedata_helpers=None,
 ):
     """Generic implementation of ifiledata.emitrevisions().
 
@@ -356,6 +358,21 b' def emitrevisions('
     ``nodesorder``
     ``revisiondata``
     ``assumehaveparentrevisions``
+    ``sidedata_helpers`` (optional)
+        If not None, means that sidedata should be included.
+        A dictionary of revlog type to tuples of `(repo, computers, removers)`:
+            * `repo` is used as an argument for computers
+            * `computers` is a list of `(category, (keys, computer)` that
+               compute the missing sidedata categories that were asked:
+               * `category` is the sidedata category
+               * `keys` are the sidedata keys to be affected
+               * `computer` is the function `(repo, store, rev, sidedata)` that
+                 returns a new sidedata dict.
+            * `removers` will remove the keys corresponding to the categories
+              that are present, but not needed.
+        If both `computers` and `removers` are empty, sidedata are simply not
+        transformed.
+        Revlog types are `changelog`, `manifest` or `filelog`.
     """
 
     fnode = store.node
@@ -469,6 +486,17 b' def emitrevisions('
 
                 available.add(rev)
 
+        sidedata = None
+        if sidedata_helpers:
+            sidedata = store.sidedata(rev)
+            sidedata = run_sidedata_helpers(
+                store=store,
+                sidedata_helpers=sidedata_helpers,
+                sidedata=sidedata,
+                rev=rev,
+            )
+            sidedata = sidedatamod.serialize_sidedata(sidedata)
+
         yield resultcls(
             node=node,
             p1node=fnode(p1rev),
@@ -484,6 +512,25 b' def emitrevisions('
         prevrev = rev
 
 
+def run_sidedata_helpers(store, sidedata_helpers, sidedata, rev):
+    """Returns the sidedata for the given revision after running through
+    the given helpers.
+    - `store`: the revlog this applies to (changelog, manifest, or filelog
+      instance)
+    - `sidedata_helpers`: see `storageutil.emitrevisions`
+    - `sidedata`: previous sidedata at the given rev, if any
+    - `rev`: affected rev of `store`
+    """
+    repo, sd_computers, sd_removers = sidedata_helpers
+    kind = store.revlog_kind
+    for _keys, sd_computer in sd_computers.get(kind, []):
+        sidedata = sd_computer(repo, store, rev, sidedata)
+    for keys, _computer in sd_removers.get(kind, []):
+        for key in keys:
+            sidedata.pop(key, None)
+    return sidedata
+
+
 def deltaiscensored(delta, baserev, baselenfn):
     """Determine if a delta represents censored revision data.
 

tests/simplestorerepo.py

@@ -446,6 +446,7 b' class filestorage(object):'
         revisiondata=False,
         assumehaveparentrevisions=False,
         deltamode=repository.CG_DELTAMODE_STD,
+        sidedata_helpers=None,
     ):
         # TODO this will probably break on some ordering options.
         nodes = [n for n in nodes if n != nullid]
@@ -459,6 +460,7 b' class filestorage(object):'
             revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
             deltamode=deltamode,
+            sidedata_helpers=sidedata_helpers,
         ):
             yield delta