clonebundles: allow manifest to specify sha256 digest of bundles

mercurial/bundlecaches.py

@@ -6,6 +6,7
 from __future__ import annotations
 
 import collections
+import re
 import typing
 
 from typing import (
@@ -27,6 +28,7 from . import (
     error,
     requirements as requirementsmod,
     sslutil,
+    url as urlmod,
     util,
 )
 from .utils import stringutil
@@ -406,6 +408,9 def isstreamclonespec(bundlespec):
     return False
 
 
+digest_regex = re.compile(b'^[a-z0-9]+:[0-9a-f]+(,[a-z0-9]+:[0-9a-f]+)*$')
+
+
 def filterclonebundleentries(
     repo, entries, streamclonerequested=False, pullbundles=False
 ):
@@ -481,6 +486,43 def filterclonebundleentries(
                 )
                 continue
 
+        if b'DIGEST' in entry:
+            if not digest_regex.match(entry[b'DIGEST']):
+                repo.ui.debug(
+                    b'filtering %s due to a bad DIGEST attribute\n' % url
+                )
+                continue
+            supported = 0
+            seen = {}
+            for digest_entry in entry[b'DIGEST'].split(b','):
+                algo, digest = digest_entry.split(b':')
+                if algo not in seen:
+                    seen[algo] = digest
+                elif seen[algo] != digest:
+                    repo.ui.debug(
+                        b'filtering %s due to conflicting %s digests\n'
+                        % (url, algo)
+                    )
+                    supported = 0
+                    break
+                digester = urlmod.digesthandler.digest_algorithms.get(algo)
+                if digester is None:
+                    continue
+                if len(digest) != digester().digest_size * 2:
+                    repo.ui.debug(
+                        b'filtering %s due to a bad %s digest\n' % (url, algo)
+                    )
+                    supported = 0
+                    break
+                supported += 1
+            else:
+                if supported == 0:
+                    repo.ui.debug(
+                        b'filtering %s due to lack of supported digest\n' % url
+                    )
+            if supported == 0:
+                continue
+
         newentries.append(entry)
 
     return newentries

mercurial/exchange.py

@@ -2900,8 +2900,23 def _maybeapplyclonebundle(pullop):
     entries = bundlecaches.sortclonebundleentries(repo.ui, entries)
 
     url = entries[0][b'URL']
+    digest = entries[0].get(b'DIGEST')
+    if digest:
+        algorithms = urlmod.digesthandler.digest_algorithms.keys()
+        preference = dict(zip(algorithms, range(len(algorithms))))
+        best_entry = None
+        best_preference = len(preference)
+        for digest_entry in digest.split(b','):
+            cur_algo, cur_digest = digest_entry.split(b':')
+            if cur_algo not in preference:
+                continue
+            if preference[cur_algo] < best_preference:
+                best_entry = digest_entry
+                best_preference = preference[cur_algo]
+        digest = best_entry
+
     repo.ui.status(_(b'applying clone bundle from %s\n') % url)
-    if trypullbundlefromurl(repo.ui, repo, url, remote):
+    if trypullbundlefromurl(repo.ui, repo, url, remote, digest):
         repo.ui.status(_(b'finished applying clone bundle\n'))
     # Bundle failed.
     #
@@ -2930,14 +2945,14 def inline_clone_bundle_open(ui, url, pe
     return util.chunkbuffer(peerclonebundle)
 
 
-def trypullbundlefromurl(ui, repo, url, peer):
+def trypullbundlefromurl(ui, repo, url, peer, digest):
     """Attempt to apply a bundle from a URL."""
     with repo.lock(), repo.transaction(b'bundleurl') as tr:
         try:
             if url.startswith(bundlecaches.CLONEBUNDLESCHEME):
                 fh = inline_clone_bundle_open(ui, url, peer)
             else:
-                fh = urlmod.open(ui, url)
+                fh = urlmod.open(ui, url, digest=digest)
             cg = readbundle(ui, fh, b'stream')
 
             if isinstance(cg, streamclone.streamcloneapplier):

mercurial/url.py

@@ -10,9 +10,11
 from __future__ import annotations
 
 import base64
+import hashlib
 import socket
 
 from .i18n import _
+from .node import hex
 from . import (
     encoding,
     error,
@@ -499,6 +501,71 class readlinehandler(urlreq.basehandler
     https_response = http_response
 
 
+class digesthandler(urlreq.basehandler):
+    # exchange.py assumes the algorithms are listed in order of preference,
+    # earlier entries are prefered.
+    digest_algorithms = {
+        b'sha256': hashlib.sha256,
+        b'sha512': hashlib.sha512,
+    }
+
+    def __init__(self, digest):
+        if b':' not in digest:
+            raise error.Abort(_(b'invalid digest specification'))
+        algo, checksum = digest.split(b':')
+        if algo not in self.digest_algorithms:
+            raise error.Abort(_(b'unsupported digest algorithm: %s') % algo)
+        self._digest = checksum
+        self._hasher = self.digest_algorithms[algo]()
+
+    def http_response(self, request, response):
+        class digestresponse(response.__class__):
+            def _digest_input(self, data):
+                self._hasher.update(data)
+                self._digest_consumed += len(data)
+                if self._digest_finished:
+                    digest = hex(self._hasher.digest())
+                    if digest != self._digest:
+                        raise error.SecurityError(
+                            _(
+                                b'file with digest %s expected, but %s found for %d bytes'
+                            )
+                            % (
+                                pycompat.bytestr(self._digest),
+                                pycompat.bytestr(digest),
+                                self._digest_consumed,
+                            )
+                        )
+
+            def read(self, amt=None):
+                data = super().read(amt)
+                self._digest_input(data)
+                return data
+
+            def readline(self):
+                data = super().readline()
+                self._digest_input(data)
+                return data
+
+            def readinto(self, dest):
+                got = super().readinto(dest)
+                self._digest_input(dest[:got])
+                return got
+
+            def _close_conn(self):
+                self._digest_finished = True
+                return super().close()
+
+        response.__class__ = digestresponse
+        response._digest = self._digest
+        response._digest_consumed = 0
+        response._hasher = self._hasher.copy()
+        response._digest_finished = False
+        return response
+
+    https_response = http_response
+
+
 handlerfuncs = []
 
 
@@ -510,6 +577,7 def opener(
     loggingname=b's',
     loggingopts=None,
     sendaccept=True,
+    digest=None,
 ):
     """
     construct an opener suitable for urllib2
@@ -562,6 +630,8 def opener(
     handlers.extend([h(ui, passmgr) for h in handlerfuncs])
     handlers.append(urlreq.httpcookieprocessor(cookiejar=load_cookiejar(ui)))
     handlers.append(readlinehandler())
+    if digest:
+        handlers.append(digesthandler(digest))
     opener = urlreq.buildopener(*handlers)
 
     # keepalive.py's handlers will populate these attributes if they exist.
@@ -600,7 +670,7 def opener(
     return opener
 
 
-def open(ui, url_, data=None, sendaccept=True):
+def open(ui, url_, data=None, sendaccept=True, digest=None):
     u = urlutil.url(url_)
     if u.scheme:
         u.scheme = u.scheme.lower()
@@ -611,7 +681,7 def open(ui, url_, data=None, sendaccept
             urlreq.pathname2url(pycompat.fsdecode(path))
         )
         authinfo = None
-    return opener(ui, authinfo, sendaccept=sendaccept).open(
+    return opener(ui, authinfo, sendaccept=sendaccept, digest=digest).open(
         pycompat.strurl(url_), data
     )
 

tests/test-clonebundles.t

@@ -743,6 +743,66 on a 32MB system.
   (sent 4 HTTP requests and * bytes; received * bytes in responses) (glob)
   $ killdaemons.py
 
+Testing a clone bundle with digest
+==================================
+
+  $ "$PYTHON" $TESTDIR/dumbhttp.py -p $HGPORT1 --pid http.pid
+  $ cat http.pid >> $DAEMON_PIDS
+  $ hg -R server serve -d -p $HGPORT --pid-file hg.pid --accesslog access.log
+  $ cat hg.pid >> $DAEMON_PIDS
+
+  $ digest=$("$PYTHON" -c "import hashlib; print (hashlib.sha256(open('gz-a.hg', 'rb').read()).hexdigest())")
+  $ cat > server/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/gz-a.hg BUNDLESPEC=gzip-v2 DIGEST=sha256:${digest}
+  > EOF
+  $ hg clone -U http://localhost:$HGPORT digest-valid
+  applying clone bundle from http://localhost:$HGPORT1/gz-a.hg
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 2 files
+  finished applying clone bundle
+  searching for changes
+  no changes found
+  2 local changesets published
+  $ digest_bad=$("$PYTHON" -c "import hashlib; print (hashlib.sha256(open('gz-a.hg', 'rb').read()+b'.').hexdigest())")
+  $ cat > server/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/gz-a.hg BUNDLESPEC=gzip-v2 DIGEST=sha256:${digest_bad}
+  > EOF
+  $ hg clone -U  http://localhost:$HGPORT digest-invalid
+  applying clone bundle from http://localhost:$HGPORT1/gz-a.hg
+  abort: file with digest [0-9a-f]* expected, but [0-9a-f]* found for [0-9]* bytes (re)
+  [150]
+  $ cat > server/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/bad-a.hg BUNDLESPEC=gzip-v2 DIGEST=sha256:xx
+  > http://localhost:$HGPORT1/bad-b.hg BUNDLESPEC=gzip-v2 DIGEST=xxx:0000
+  > http://localhost:$HGPORT1/bad-c.hg BUNDLESPEC=gzip-v2 DIGEST=sha256:0000
+  > http://localhost:$HGPORT1/bad-d.hg BUNDLESPEC=gzip-v2 DIGEST=xxx:00,xxx:01
+  > http://localhost:$HGPORT1/gz-a.hg BUNDLESPEC=gzip-v2 DIGEST=sha256:${digest_bad}
+  > EOF
+  $ hg clone --debug -U  http://localhost:$HGPORT digest-malformed
+  using http://localhost:$HGPORT/
+  sending capabilities command
+  sending clonebundles_manifest command
+  filtering http://localhost:$HGPORT1/bad-a.hg due to a bad DIGEST attribute
+  filtering http://localhost:$HGPORT1/bad-b.hg due to lack of supported digest
+  filtering http://localhost:$HGPORT1/bad-c.hg due to a bad sha256 digest
+  filtering http://localhost:$HGPORT1/bad-d.hg due to conflicting xxx digests
+  applying clone bundle from http://localhost:$HGPORT1/gz-a.hg
+  bundle2-input-bundle: 1 params with-transaction
+  bundle2-input-bundle: 0 parts total
+  \(sent [0-9]* HTTP requests and [0-9]* bytes; received [0-9]* bytes in responses\) (re)
+  abort: file with digest [0-9a-f]* expected, but [0-9a-f]* found for [0-9]* bytes (re)
+  [150]
+  $ cat > server/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/gz-a.hg BUNDLESPEC=gzip-v2 DIGEST=sha512:00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,sha256:0000000000000000000000000000000000000000000000000000000000000000
+  > EOF
+  $ hg clone -U  http://localhost:$HGPORT digest-preference
+  applying clone bundle from http://localhost:$HGPORT1/gz-a.hg
+  abort: file with digest 0{64} expected, but [0-9a-f]+ found for [0-9]+ bytes (re)
+  [150]
+  $ killdaemons.py
+
 Testing a clone bundles that involves revlog splitting (issue6811)
 ==================================================================