bundle: optional multithreaded compression, ATM zstd-only

Compression type can be a huge chunk of "hg bundle", especially when
using the higher compression levels. With level=22 and threads=7, the
NetBSD test repository took 28:39 wall time and 157:47 user time.
Before, level=22 would take 129:20 wall time and 129:07 user time.

Differential Revision: https://phab.mercurial-scm.org/D9283

mercurial/commands.py

@@ -1648,6 +1648,14 b' def bundle(ui, repo, fname, dest=None, *'
     if complevel is not None:
         compopts[b'level'] = complevel
 
+    compthreads = ui.configint(
+        b'experimental', b'bundlecompthreads.' + bundlespec.compression
+    )
+    if compthreads is None:
+        compthreads = ui.configint(b'experimental', b'bundlecompthreads')
+    if compthreads is not None:
+        compopts[b'threads'] = compthreads
+
     # Bundling of obsmarker and phases is optional as not all clients
     # support the necessary features.
     cfg = ui.configbool

mercurial/configitems.py

@@ -866,6 +866,31 b' coreconfigitem('
 )
 coreconfigitem(
     b'experimental',
+    b'bundlecompthreads',
+    default=None,
+)
+coreconfigitem(
+    b'experimental',
+    b'bundlecompthreads.bzip2',
+    default=None,
+)
+coreconfigitem(
+    b'experimental',
+    b'bundlecompthreads.gzip',
+    default=None,
+)
+coreconfigitem(
+    b'experimental',
+    b'bundlecompthreads.none',
+    default=None,
+)
+coreconfigitem(
+    b'experimental',
+    b'bundlecompthreads.zstd',
+    default=None,
+)
+coreconfigitem(
+    b'experimental',
     b'changegroup3',
     default=False,
 )

mercurial/utils/compression.py

@@ -685,9 +685,11 b' class _zstdengine(compressionengine):'
         # while providing no worse compression. It strikes a good balance
         # between speed and compression.
         level = opts.get(b'level', 3)
+        # default to single-threaded compression
+        threads = opts.get(b'threads', 0)
 
         zstd = self._module
-        z = zstd.ZstdCompressor(level=level).compressobj()
+        z = zstd.ZstdCompressor(level=level, threads=threads).compressobj()
         for chunk in it:
             data = z.compress(chunk)
             if data:

relnotes/next

@@ -5,6 +5,11 b''
  * The `rev-branch-cache` is now updated incrementally whenever changesets
    are added.
 
+ * The new options `experimental.bundlecompthreads` and
+   `experimental.bundlecompthreads.<engine>` can be used to instruct
+   the compression engines for bundle operations to use multiple threads
+   for compression. The default is single threaded operation. Currently
+   only supported for zstd.
 
 == New Experimental Features ==
 

tests/test-bundle-type.t

@@ -201,6 +201,15 b' zstd-v1 always fails'
   (see 'hg help bundlespec' for supported values for --type)
   [10]
 
+zstd supports threading
+
+  $ hg init test-compthreads
+  $ cd test-compthreads
+  $ hg debugbuilddag +3
+  $ hg --config experimental.bundlecompthreads=1 bundle -a -t zstd-v2 zstd-v2-threaded.hg
+  3 changesets found
+  $ cd ..
+
 #else
 
 zstd is a valid engine but isn't available