merge default into stable for 5.4 release

contrib/packaging/pyoxidizer.bzl

@@ -0,0 +1,60 b''
+# Instructions:
+#
+# 1. cargo install --version 0.5.0 pyoxidizer
+# 2. cd /path/to/hg
+# 3. pyoxidizer build --path contrib/packaging [--release]
+# 4. Run build/pyoxidizer/<arch>/<debug|release>/app/hg
+#
+# If you need to build again, you need to remove the build/lib.* and
+# build/temp.* directories, otherwise PyOxidizer fails to pick up C
+# extensions. This is a bug in PyOxidizer.
+
+ROOT = CWD + "/../.."
+
+set_build_path(ROOT + "/build/pyoxidizer")
+
+def make_exe():
+    dist = default_python_distribution()
+
+    code = "import hgdemandimport; hgdemandimport.enable(); from mercurial import dispatch; dispatch.run()"
+
+    config = PythonInterpreterConfig(
+        raw_allocator = "system",
+        run_eval = code,
+        # We want to let the user load extensions from the file system
+        filesystem_importer = True,
+        # We need this to make resourceutil happy, since it looks for sys.frozen.
+        sys_frozen = True,
+        legacy_windows_stdio = True,
+    )
+
+    exe = dist.to_python_executable(
+        name = "hg",
+        config = config,
+    )
+
+    # Use setup.py install to build Mercurial and collect Python resources to
+    # embed in the executable.
+    resources = dist.setup_py_install(ROOT)
+    exe.add_python_resources(resources)
+
+    return exe
+
+def make_install(exe):
+    m = FileManifest()
+
+    # `hg` goes in root directory.
+    m.add_python_resource(".", exe)
+
+    templates = glob(
+        include=[ROOT + "/mercurial/templates/**/*"],
+        strip_prefix = ROOT + "/mercurial/",
+    )
+    m.add_manifest(templates)
+
+    return m
+
+register_target("exe", make_exe)
+register_target("app", make_install, depends = ["exe"], default = True)
+
+resolve_targets()

contrib/phab-clean.py

@@ -0,0 +1,93 b''
+#!/usr/bin/env python
+#
+# A small script to automatically reject idle Diffs
+#
+# you need to set the PHABBOT_USER and PHABBOT_TOKEN environment variable for authentication
+from __future__ import absolute_import, print_function
+
+import datetime
+import os
+import sys
+
+import phabricator
+
+MESSAGE = """There seems to have been no activities on this Diff for the past 3 Months.
+
+By policy, we are automatically moving it out of the `need-review` state.
+
+Please, move it back to `need-review` without hesitation if this diff should still be discussed.
+
+:baymax:need-review-idle:
+"""
+
+
+PHAB_URL = "https://phab.mercurial-scm.org/api/"
+USER = os.environ.get("PHABBOT_USER", "baymax")
+TOKEN = os.environ.get("PHABBOT_TOKEN")
+
+
+NOW = datetime.datetime.now()
+
+# 3 months in seconds
+DELAY = 60 * 60 * 24 * 30 * 3
+
+
+def get_all_diff(phab):
+    """Fetch all the diff that the need review"""
+    return phab.differential.query(
+        status="status-needs-review",
+        order="order-modified",
+        paths=[('HG', None)],
+    )
+
+
+def filter_diffs(diffs, older_than):
+    """filter diffs to only keep the one unmodified sin <older_than> seconds"""
+    olds = []
+    for d in diffs:
+        modified = int(d['dateModified'])
+        modified = datetime.datetime.fromtimestamp(modified)
+        d["idleFor"] = idle_for = NOW - modified
+        if idle_for.total_seconds() > older_than:
+            olds.append(d)
+    return olds
+
+
+def nudge_diff(phab, diff):
+    """Comment on the idle diff and reject it"""
+    diff_id = int(d['id'])
+    phab.differential.createcomment(
+        revision_id=diff_id, message=MESSAGE, action="reject"
+    )
+
+
+if not USER:
+    print(
+        "not user specified please set PHABBOT_USER and PHABBOT_TOKEN",
+        file=sys.stderr,
+    )
+elif not TOKEN:
+    print(
+        "not api-token specified please set PHABBOT_USER and PHABBOT_TOKEN",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+phab = phabricator.Phabricator(USER, host=PHAB_URL, token=TOKEN)
+phab.connect()
+phab.update_interfaces()
+print('Hello "%s".' % phab.user.whoami()['realName'])
+
+diffs = get_all_diff(phab)
+print("Found %d Diffs" % len(diffs))
+olds = filter_diffs(diffs, DELAY)
+print("Found %d old Diffs" % len(olds))
+for d in olds:
+    diff_id = d['id']
+    status = d['statusName']
+    modified = int(d['dateModified'])
+    idle_for = d["idleFor"]
+    msg = 'nudging D%s in "%s" state for %s'
+    print(msg % (diff_id, status, idle_for))
+    # uncomment to actually affect phab
+    nudge_diff(phab, d)

Makefile

@@ -1,258 +1,259 b''
 # If you want to change PREFIX, do not just edit it below. The changed
 # value wont get passed on to recursive make calls. You should instead
 # override the variable on the command like:
 #
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
 PYTHON?=python
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=
 PYFILESCMD=find mercurial hgext doc -name '*.py'
 PYFILES:=$(shell $(PYFILESCMD))
 DOCFILES=mercurial/helptext/*.txt
 export LANGUAGE=C
 export LC_ALL=C
 TESTFLAGS ?= $(shell echo $$HGTESTFLAGS)
 OSXVERSIONFLAGS ?= $(shell echo $$OSXVERSIONFLAGS)
 CARGO = cargo
 
 # Set this to e.g. "mingw32" to use a non-default compiler.
 COMPILER=
 
 COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
 help:
 	@echo 'Commonly used make targets:'
 	@echo '  all          - build program and documentation'
 	@echo '  install      - install program and man pages to $$PREFIX ($(PREFIX))'
 	@echo '  install-home - install with setup.py install --home=$$HOME ($(HOME))'
 	@echo '  local        - build for inplace usage'
 	@echo '  tests        - run all tests in the automatic test suite'
 	@echo '  test-foo     - run only specified tests (e.g. test-merge1.t)'
 	@echo '  dist         - run all tests and create a source tarball in dist/'
 	@echo '  clean        - remove files created by other targets'
 	@echo '                 (except installed files or dist source tarball)'
 	@echo '  update-pot   - update i18n/hg.pot'
 	@echo
 	@echo 'Example for a system-wide installation under /usr/local:'
 	@echo '  make all && su -c "make install" && hg version'
 	@echo
 	@echo 'Example for a local installation (usable in this directory):'
 	@echo '  make local && ./hg version'
 
 all: build doc
 
 local:
 	$(PYTHON) setup.py $(PURE) \
 	  build_py -c -d . \
 	  build_ext $(COMPILERFLAG) -i \
 	  build_hgexe $(COMPILERFLAG) -i \
 	  build_mo
 	env HGRCPATH= $(PYTHON) hg version
 
 build:
 	$(PYTHON) setup.py $(PURE) build $(COMPILERFLAG)
 
 wheel:
 	FORCE_SETUPTOOLS=1 $(PYTHON) setup.py $(PURE) bdist_wheel $(COMPILERFLAG)
 
 doc:
 	$(MAKE) -C doc
 
 cleanbutpackages:
+	rm -f hg.exe
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests hgdemandimport \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
 	rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
 	rm -f mercurial/__modulepolicy__.py
 	if test -d .hg; then rm -f mercurial/__version__.py; fi
 	rm -rf build mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
 	rm -rf rust/target
 	rm -f mercurial/rustext.so
 
 clean: cleanbutpackages
 	rm -rf packages
 
 install: install-bin install-doc
 
 install-bin: build
 	$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force
 
 install-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) install
 
 install-home: install-home-bin install-home-doc
 
 install-home-bin: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
 install-home-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) PREFIX="$(HOME)" install
 
 MANIFEST-doc:
 	$(MAKE) -C doc MANIFEST
 
 MANIFEST.in: MANIFEST-doc
 	hg manifest | sed -e 's/^/include /' > MANIFEST.in
 	echo include mercurial/__version__.py >> MANIFEST.in
 	sed -e 's/^/include /' < doc/MANIFEST >> MANIFEST.in
 
 dist:	tests dist-notests
 
 dist-notests:	doc MANIFEST.in
 	TAR_OPTIONS="--owner=root --group=root --mode=u+w,go-w,a+rX-s" $(PYTHON) setup.py -q sdist
 
 check: tests
 
 tests:
         # Run Rust tests if cargo is installed
 	if command -v $(CARGO) >/dev/null 2>&1; then \
 		$(MAKE) rust-tests; \
 	fi
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
 
 test-%:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS) $@
 
 testpy-%:
 	@echo Looking for Python $* in $(HGPYTHONS)
 	[ -e $(HGPYTHONS)/$*/bin/python ] || ( \
 	cd $$(mktemp --directory --tmpdir) && \
         $(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
 	cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
 
 rust-tests: py_feature = $(shell $(PYTHON) -c \
  'import sys; print(["python27-bin", "python3-bin"][sys.version_info[0] >= 3])')
 rust-tests:
 	cd $(HGROOT)/rust/hg-cpython \
 		&& $(CARGO) test --quiet --all \
 			--no-default-features --features "$(py_feature)"
 
 check-code:
 	hg manifest | xargs python contrib/check-code.py
 
 format-c:
 	clang-format --style file -i \
 	  `hg files 'set:(**.c or **.cc or **.h) and not "listfile:contrib/clang-format-ignorelist"'`
 
 update-pot: i18n/hg.pot
 
 i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n/posplit i18n/hggettext
 	$(PYTHON) i18n/hggettext mercurial/commands.py \
 	  hgext/*.py hgext/*/__init__.py \
 	  mercurial/fileset.py mercurial/revset.py \
 	  mercurial/templatefilters.py \
 	  mercurial/templatefuncs.py \
 	  mercurial/templatekw.py \
 	  mercurial/filemerge.py \
 	  mercurial/hgweb/webcommands.py \
 	  mercurial/util.py \
 	  $(DOCFILES) > i18n/hg.pot.tmp
         # All strings marked for translation in Mercurial contain
         # ASCII characters only. But some files contain string
         # literals like this '\037\213'. xgettext thinks it has to
         # parse them even though they are not marked for translation.
         # Extracting with an explicit encoding of ISO-8859-1 will make
         # xgettext "parse" and ignore them.
 	$(PYFILESCMD) | xargs \
 	  xgettext --package-name "Mercurial" \
 	  --msgid-bugs-address "<[e-mail]>" \
 	  --copyright-holder "Matt Mackall <[e-mail]> and others" \
 	  --from-code ISO-8859-1 --join --sort-by-file --add-comments=i18n: \
 	  -d hg -p i18n -o hg.pot.tmp
 	$(PYTHON) i18n/posplit i18n/hg.pot.tmp
         # The target file is not created before the last step. So it never is in
         # an intermediate state.
 	mv -f i18n/hg.pot.tmp i18n/hg.pot
 
 %.po: i18n/hg.pot
         # work on a temporary copy for never having a half completed target
 	cp $@ $@.tmp
 	msgmerge --no-location --update $@.tmp $^
 	mv -f $@.tmp $@
 
 # Packaging targets
 
 packaging_targets := \
   centos5 \
   centos6 \
   centos7 \
   centos8 \
   deb \
   docker-centos5 \
   docker-centos6 \
   docker-centos7 \
   docker-centos8 \
   docker-debian-bullseye \
   docker-debian-buster \
   docker-debian-stretch \
   docker-fedora \
   docker-ubuntu-trusty \
   docker-ubuntu-trusty-ppa \
   docker-ubuntu-xenial \
   docker-ubuntu-xenial-ppa \
   docker-ubuntu-artful \
   docker-ubuntu-artful-ppa \
   docker-ubuntu-bionic \
   docker-ubuntu-bionic-ppa \
   fedora \
   linux-wheels \
   linux-wheels-x86_64 \
   linux-wheels-i686 \
   ppa
 
 # Forward packaging targets for convenience.
 $(packaging_targets):
 	$(MAKE) -C contrib/packaging $@
 
 osx:
 	rm -rf build/mercurial
 	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"
         # Place a bogon .DS_Store file in the target dir so we can be
         # sure it doesn't get included in the final package.
 	touch build/mercurial/.DS_Store
         # install zsh completions - this location appears to be
         # searched by default as of macOS Sierra.
 	install -d build/mercurial/usr/local/share/zsh/site-functions/
 	install -m 0644 contrib/zsh_completion build/mercurial/usr/local/share/zsh/site-functions/_hg
         # install bash completions - there doesn't appear to be a
         # place that's searched by default for bash, so we'll follow
         # the lead of Apple's git install and just put it in a
         # location of our own.
 	install -d build/mercurial/usr/local/hg/contrib/
 	install -m 0644 contrib/bash_completion build/mercurial/usr/local/hg/contrib/hg-completion.bash
 	make -C contrib/chg \
 	  HGPATH=/usr/local/bin/hg \
 	  PYTHON=/usr/bin/python2.7 \
 	  HGEXTDIR=/Library/Python/2.7/site-packages/hgext \
 	  DESTDIR=../../build/mercurial \
 	  PREFIX=/usr/local \
 	  clean install
 	mkdir -p $${OUTPUTDIR:-dist}
 	HGVER=$$(python contrib/genosxversion.py $(OSXVERSIONFLAGS) build/mercurial/Library/Python/2.7/site-packages/mercurial/__version__.py) && \
 	OSXVER=$$(sw_vers -productVersion | cut -d. -f1,2) && \
 	pkgbuild --filter \\.DS_Store --root build/mercurial/ \
 	  --identifier org.mercurial-scm.mercurial \
 	  --version "$${HGVER}" \
 	  build/mercurial.pkg && \
 	productbuild --distribution contrib/packaging/macosx/distribution.xml \
 	  --package-path build/ \
 	  --version "$${HGVER}" \
 	  --resources contrib/packaging/macosx/ \
 	  "$${OUTPUTDIR:-dist/}"/Mercurial-"$${HGVER}"-macosx"$${OSXVER}".pkg
 
 .PHONY: help all local build doc cleanbutpackages clean install install-bin \
 	install-doc install-home install-home-bin install-home-doc \
 	dist dist-notests check tests rust-tests check-code format-c \
 	update-pot \
 	$(packaging_targets) \
 	osx

black.toml

@@ -1,15 +1,14 b''
 [tool.black]
 line-length = 80
 exclude = '''
 build/
 | wheelhouse/
 | dist/
 | packages/
 | \.hg/
 | \.mypy_cache/
 | \.venv/
 | mercurial/thirdparty/
-| contrib/python-zstandard/
 '''
 skip-string-normalization = true
 quiet = true

contrib/benchmarks/__init__.py

@@ -1,126 +1,126 b''
 # __init__.py - asv benchmark suite
 #
 # Copyright 2016 Logilab SA <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # "historical portability" policy of contrib/benchmarks:
 #
 # We have to make this code work correctly with current mercurial stable branch
 # and if possible with reasonable cost with early Mercurial versions.
 
 '''ASV (https://asv.readthedocs.io) benchmark suite
 
 Benchmark are parameterized against reference repositories found in the
 directory pointed by the REPOS_DIR environment variable.
 
 Invocation example:
 
     $ export REPOS_DIR=~/hgperf/repos
     # run suite on given revision
     $ asv --config contrib/asv.conf.json run REV
     # run suite on new changesets found in stable and default branch
     $ asv --config contrib/asv.conf.json run NEW
     # display a comparative result table of benchmark results between two given
     # revisions
     $ asv --config contrib/asv.conf.json compare REV1 REV2
     # compute regression detection and generate ASV static website
     $ asv --config contrib/asv.conf.json publish
     # serve the static website
     $ asv --config contrib/asv.conf.json preview
 '''
 
 from __future__ import absolute_import
 
 import functools
 import os
 import re
 
 from mercurial import (
     extensions,
     hg,
     ui as uimod,
     util,
 )
 
 basedir = os.path.abspath(
     os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
 )
 reposdir = os.environ['REPOS_DIR']
 reposnames = [
     name
     for name in os.listdir(reposdir)
     if os.path.isdir(os.path.join(reposdir, name, ".hg"))
 ]
 if not reposnames:
     raise ValueError("No repositories found in $REPO_DIR")
 outputre = re.compile(
     (
         r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
         r'\d+.\d+ \(best of \d+\)'
     )
 )
 
 
 def runperfcommand(reponame, command, *args, **kwargs):
     os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "")
     # for "historical portability"
     # ui.load() has been available since d83ca85
     if util.safehasattr(uimod.ui, "load"):
         ui = uimod.ui.load()
     else:
         ui = uimod.ui()
     repo = hg.repository(ui, os.path.join(reposdir, reponame))
     perfext = extensions.load(
         ui, 'perfext', os.path.join(basedir, 'contrib', 'perf.py')
     )
     cmd = getattr(perfext, command)
     ui.pushbuffer()
     cmd(ui, repo, *args, **kwargs)
     output = ui.popbuffer()
     match = outputre.search(output)
     if not match:
-        raise ValueError("Invalid output {0}".format(output))
+        raise ValueError("Invalid output {}".format(output))
     return float(match.group(1))
 
 
 def perfbench(repos=reposnames, name=None, params=None):
     """decorator to declare ASV benchmark based on contrib/perf.py extension
 
     An ASV benchmark is a python function with the given attributes:
 
     __name__: should start with track_, time_ or mem_ to be collected by ASV
     params and param_name: parameter matrix to display multiple graphs on the
     same page.
     pretty_name: If defined it's displayed in web-ui instead of __name__
     (useful for revsets)
     the module name is prepended to the benchmark name and displayed as
     "category" in webui.
 
     Benchmarks are automatically parameterized with repositories found in the
     REPOS_DIR environment variable.
 
     `params` is the param matrix in the form of a list of tuple
     (param_name, [value0, value1])
 
     For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
     (a, c), (a, d), (b, c) and (b, d).
     """
     params = list(params or [])
     params.insert(0, ("repo", repos))
 
     def decorator(func):
         @functools.wraps(func)
         def wrapped(repo, *args):
             def perf(command, *a, **kw):
                 return runperfcommand(repo, command, *a, **kw)
 
             return func(perf, *args)
 
         wrapped.params = [p[1] for p in params]
         wrapped.param_names = [p[0] for p in params]
         wrapped.pretty_name = name
         return wrapped
 
     return decorator

contrib/check-py3-compat.py

@@ -1,113 +1,113 b''
 #!/usr/bin/env python
 #
 # check-py3-compat - check Python 3 compatibility of Mercurial files
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import ast
 import importlib
 import os
 import sys
 import traceback
 import warnings
 
 
 def check_compat_py2(f):
     """Check Python 3 compatibility for a file with Python 2"""
     with open(f, 'rb') as fh:
         content = fh.read()
     root = ast.parse(content)
 
     # Ignore empty files.
     if not root.body:
         return
 
     futures = set()
     haveprint = False
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
-                futures |= set(n.name for n in node.names)
+                futures |= {n.name for n in node.names}
         elif isinstance(node, ast.Print):
             haveprint = True
 
     if 'absolute_import' not in futures:
         print('%s not using absolute_import' % f)
     if haveprint and 'print_function' not in futures:
         print('%s requires print_function' % f)
 
 
 def check_compat_py3(f):
     """Check Python 3 compatibility of a file with Python 3."""
     with open(f, 'rb') as fh:
         content = fh.read()
 
     try:
         ast.parse(content, filename=f)
     except SyntaxError as e:
         print('%s: invalid syntax: %s' % (f, e))
         return
 
     # Try to import the module.
     # For now we only support modules in packages because figuring out module
     # paths for things not in a package can be confusing.
     if f.startswith(
         ('hgdemandimport/', 'hgext/', 'mercurial/')
     ) and not f.endswith('__init__.py'):
         assert f.endswith('.py')
         name = f.replace('/', '.')[:-3]
         try:
             importlib.import_module(name)
         except Exception as e:
             exc_type, exc_value, tb = sys.exc_info()
             # We walk the stack and ignore frames from our custom importer,
             # import mechanisms, and stdlib modules. This kinda/sorta
             # emulates CPython behavior in import.c while also attempting
             # to pin blame on a Mercurial file.
             for frame in reversed(traceback.extract_tb(tb)):
                 if frame.name == '_call_with_frames_removed':
                     continue
                 if 'importlib' in frame.filename:
                     continue
                 if 'mercurial/__init__.py' in frame.filename:
                     continue
                 if frame.filename.startswith(sys.prefix):
                     continue
                 break
 
             if frame.filename:
                 filename = os.path.basename(frame.filename)
                 print(
                     '%s: error importing: <%s> %s (error at %s:%d)'
                     % (f, type(e).__name__, e, filename, frame.lineno)
                 )
             else:
                 print(
                     '%s: error importing module: <%s> %s (line %d)'
                     % (f, type(e).__name__, e, frame.lineno)
                 )
 
 
 if __name__ == '__main__':
     if sys.version_info[0] == 2:
         fn = check_compat_py2
     else:
         fn = check_compat_py3
 
     for f in sys.argv[1:]:
         with warnings.catch_warnings(record=True) as warns:
             fn(f)
 
         for w in warns:
             print(
                 warnings.formatwarning(
                     w.message, w.category, w.filename, w.lineno
                 ).rstrip()
             )
 
     sys.exit(0)

contrib/chg/chg.c

@@ -1,456 +1,470 b''
 /*
  * A fast client for Mercurial command server
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/file.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/un.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
 
 #include "hgclient.h"
 #include "procutil.h"
 #include "util.h"
 
 #ifndef PATH_MAX
 #define PATH_MAX 4096
 #endif
 
 struct cmdserveropts {
 	char sockname[PATH_MAX];
 	char initsockname[PATH_MAX];
 	char redirectsockname[PATH_MAX];
 	size_t argsize;
 	const char **args;
 };
 
 static void initcmdserveropts(struct cmdserveropts *opts)
 {
 	memset(opts, 0, sizeof(struct cmdserveropts));
 }
 
 static void freecmdserveropts(struct cmdserveropts *opts)
 {
 	free(opts->args);
 	opts->args = NULL;
 	opts->argsize = 0;
 }
 
 /*
  * Test if an argument is a sensitive flag that should be passed to the server.
  * Return 0 if not, otherwise the number of arguments starting from the current
  * one that should be passed to the server.
  */
 static size_t testsensitiveflag(const char *arg)
 {
 	static const struct {
 		const char *name;
 		size_t narg;
 	} flags[] = {
 	    {"--config", 1},     {"--cwd", 1},       {"--repo", 1},
 	    {"--repository", 1}, {"--traceback", 0}, {"-R", 1},
 	};
 	size_t i;
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); ++i) {
 		size_t len = strlen(flags[i].name);
 		size_t narg = flags[i].narg;
 		if (memcmp(arg, flags[i].name, len) == 0) {
 			if (arg[len] == '\0') {
 				/* --flag (value) */
 				return narg + 1;
 			} else if (arg[len] == '=' && narg > 0) {
 				/* --flag=value */
 				return 1;
 			} else if (flags[i].name[1] != '-') {
 				/* short flag */
 				return 1;
 			}
 		}
 	}
 	return 0;
 }
 
 /*
  * Parse argv[] and put sensitive flags to opts->args
  */
 static void setcmdserverargs(struct cmdserveropts *opts, int argc,
                              const char *argv[])
 {
 	size_t i, step;
 	opts->argsize = 0;
 	for (i = 0, step = 1; i < (size_t)argc; i += step, step = 1) {
 		if (!argv[i])
 			continue; /* pass clang-analyse */
 		if (strcmp(argv[i], "--") == 0)
 			break;
 		size_t n = testsensitiveflag(argv[i]);
 		if (n == 0 || i + n > (size_t)argc)
 			continue;
 		opts->args =
 		    reallocx(opts->args, (n + opts->argsize) * sizeof(char *));
 		memcpy(opts->args + opts->argsize, argv + i,
 		       sizeof(char *) * n);
 		opts->argsize += n;
 		step = n;
 	}
 }
 
 static void preparesockdir(const char *sockdir)
 {
 	int r;
 	r = mkdir(sockdir, 0700);
 	if (r < 0 && errno != EEXIST)
 		abortmsgerrno("cannot create sockdir %s", sockdir);
 
 	struct stat st;
 	r = lstat(sockdir, &st);
 	if (r < 0)
 		abortmsgerrno("cannot stat %s", sockdir);
 	if (!S_ISDIR(st.st_mode))
 		abortmsg("cannot create sockdir %s (file exists)", sockdir);
 	if (st.st_uid != geteuid() || st.st_mode & 0077)
 		abortmsg("insecure sockdir %s", sockdir);
 }
 
 /*
  * Check if a socket directory exists and is only owned by the current user.
  * Return 1 if so, 0 if not. This is used to check if XDG_RUNTIME_DIR can be
  * used or not. According to the specification [1], XDG_RUNTIME_DIR should be
  * ignored if the directory is not owned by the user with mode 0700.
  * [1]: https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
  */
 static int checkruntimedir(const char *sockdir)
 {
 	struct stat st;
 	int r = lstat(sockdir, &st);
 	if (r < 0) /* ex. does not exist */
 		return 0;
 	if (!S_ISDIR(st.st_mode)) /* ex. is a file, not a directory */
 		return 0;
 	return st.st_uid == geteuid() && (st.st_mode & 0777) == 0700;
 }
 
 static void getdefaultsockdir(char sockdir[], size_t size)
 {
 	/* by default, put socket file in secure directory
 	 * (${XDG_RUNTIME_DIR}/chg, or /${TMPDIR:-tmp}/chg$UID)
 	 * (permission of socket file may be ignored on some Unices) */
 	const char *runtimedir = getenv("XDG_RUNTIME_DIR");
 	int r;
 	if (runtimedir && checkruntimedir(runtimedir)) {
 		r = snprintf(sockdir, size, "%s/chg", runtimedir);
 	} else {
 		const char *tmpdir = getenv("TMPDIR");
 		if (!tmpdir)
 			tmpdir = "/tmp";
 		r = snprintf(sockdir, size, "%s/chg%d", tmpdir, geteuid());
 	}
 	if (r < 0 || (size_t)r >= size)
 		abortmsg("too long TMPDIR (r = %d)", r);
 }
 
 static void setcmdserveropts(struct cmdserveropts *opts)
 {
 	int r;
 	char sockdir[PATH_MAX];
 	const char *envsockname = getenv("CHGSOCKNAME");
 	if (!envsockname) {
 		getdefaultsockdir(sockdir, sizeof(sockdir));
 		preparesockdir(sockdir);
 	}
 
 	const char *basename = (envsockname) ? envsockname : sockdir;
 	const char *sockfmt = (envsockname) ? "%s" : "%s/server";
 	r = snprintf(opts->sockname, sizeof(opts->sockname), sockfmt, basename);
 	if (r < 0 || (size_t)r >= sizeof(opts->sockname))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 	r = snprintf(opts->initsockname, sizeof(opts->initsockname), "%s.%u",
 	             opts->sockname, (unsigned)getpid());
 	if (r < 0 || (size_t)r >= sizeof(opts->initsockname))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 }
 
 static const char *gethgcmd(void)
 {
 	static const char *hgcmd = NULL;
 	if (!hgcmd) {
 		hgcmd = getenv("CHGHG");
 		if (!hgcmd || hgcmd[0] == '\0')
 			hgcmd = getenv("HG");
 		if (!hgcmd || hgcmd[0] == '\0')
 #ifdef HGPATH
 			hgcmd = (HGPATH);
 #else
 			hgcmd = "hg";
 #endif
 	}
 	return hgcmd;
 }
 
 static void execcmdserver(const struct cmdserveropts *opts)
 {
 	const char *hgcmd = gethgcmd();
 
 	const char *baseargv[] = {
 	    hgcmd,
 	    "serve",
 	    "--cmdserver",
 	    "chgunix",
 	    "--address",
 	    opts->initsockname,
 	    "--daemon-postexec",
 	    "chdir:/",
 	};
 	size_t baseargvsize = sizeof(baseargv) / sizeof(baseargv[0]);
 	size_t argsize = baseargvsize + opts->argsize + 1;
 
 	const char **argv = mallocx(sizeof(char *) * argsize);
 	memcpy(argv, baseargv, sizeof(baseargv));
 	if (opts->args) {
 		size_t size = sizeof(char *) * opts->argsize;
 		memcpy(argv + baseargvsize, opts->args, size);
 	}
 	argv[argsize - 1] = NULL;
 
+	const char *lc_ctype_env = getenv("LC_CTYPE");
+	if (lc_ctype_env == NULL) {
+		if (putenv("CHG_CLEAR_LC_CTYPE=") != 0)
+			abortmsgerrno("failed to putenv CHG_CLEAR_LC_CTYPE");
+	} else {
+		if (setenv("CHGORIG_LC_CTYPE", lc_ctype_env, 1) != 0) {
+			abortmsgerrno("failed to setenv CHGORIG_LC_CTYYPE");
+		}
+	}
+
 	if (putenv("CHGINTERNALMARK=") != 0)
 		abortmsgerrno("failed to putenv");
 	if (execvp(hgcmd, (char **)argv) < 0)
 		abortmsgerrno("failed to exec cmdserver");
 	free(argv);
 }
 
 /* Retry until we can connect to the server. Give up after some time. */
 static hgclient_t *retryconnectcmdserver(struct cmdserveropts *opts, pid_t pid)
 {
 	static const struct timespec sleepreq = {0, 10 * 1000000};
 	int pst = 0;
 
 	debugmsg("try connect to %s repeatedly", opts->initsockname);
 
 	unsigned int timeoutsec = 60; /* default: 60 seconds */
 	const char *timeoutenv = getenv("CHGTIMEOUT");
 	if (timeoutenv)
 		sscanf(timeoutenv, "%u", &timeoutsec);
 
 	for (unsigned int i = 0; !timeoutsec || i < timeoutsec * 100; i++) {
 		hgclient_t *hgc = hgc_open(opts->initsockname);
 		if (hgc) {
 			debugmsg("rename %s to %s", opts->initsockname,
 			         opts->sockname);
 			int r = rename(opts->initsockname, opts->sockname);
 			if (r != 0)
 				abortmsgerrno("cannot rename");
 			return hgc;
 		}
 
 		if (pid > 0) {
 			/* collect zombie if child process fails to start */
 			int r = waitpid(pid, &pst, WNOHANG);
 			if (r != 0)
 				goto cleanup;
 		}
 
 		nanosleep(&sleepreq, NULL);
 	}
 
 	abortmsg("timed out waiting for cmdserver %s", opts->initsockname);
 	return NULL;
 
 cleanup:
 	if (WIFEXITED(pst)) {
 		if (WEXITSTATUS(pst) == 0)
 			abortmsg("could not connect to cmdserver "
 			         "(exited with status 0)");
 		debugmsg("cmdserver exited with status %d", WEXITSTATUS(pst));
 		exit(WEXITSTATUS(pst));
 	} else if (WIFSIGNALED(pst)) {
 		abortmsg("cmdserver killed by signal %d", WTERMSIG(pst));
 	} else {
 		abortmsg("error while waiting for cmdserver");
 	}
 	return NULL;
 }
 
 /* Connect to a cmdserver. Will start a new server on demand. */
 static hgclient_t *connectcmdserver(struct cmdserveropts *opts)
 {
 	const char *sockname =
 	    opts->redirectsockname[0] ? opts->redirectsockname : opts->sockname;
 	debugmsg("try connect to %s", sockname);
 	hgclient_t *hgc = hgc_open(sockname);
 	if (hgc)
 		return hgc;
 
 	/* prevent us from being connected to an outdated server: we were
 	 * told by a server to redirect to opts->redirectsockname and that
 	 * address does not work. we do not want to connect to the server
 	 * again because it will probably tell us the same thing. */
 	if (sockname == opts->redirectsockname)
 		unlink(opts->sockname);
 
 	debugmsg("start cmdserver at %s", opts->initsockname);
 
 	pid_t pid = fork();
 	if (pid < 0)
 		abortmsg("failed to fork cmdserver process");
 	if (pid == 0) {
 		execcmdserver(opts);
 	} else {
 		hgc = retryconnectcmdserver(opts, pid);
 	}
 
 	return hgc;
 }
 
 static void killcmdserver(const struct cmdserveropts *opts)
 {
 	/* resolve config hash */
 	char *resolvedpath = realpath(opts->sockname, NULL);
 	if (resolvedpath) {
 		unlink(resolvedpath);
 		free(resolvedpath);
 	}
 }
 
 /* Run instructions sent from the server like unlink and set redirect path
  * Return 1 if reconnect is needed, otherwise 0 */
 static int runinstructions(struct cmdserveropts *opts, const char **insts)
 {
 	int needreconnect = 0;
 	if (!insts)
 		return needreconnect;
 
 	assert(insts);
 	opts->redirectsockname[0] = '\0';
 	const char **pinst;
 	for (pinst = insts; *pinst; pinst++) {
 		debugmsg("instruction: %s", *pinst);
 		if (strncmp(*pinst, "unlink ", 7) == 0) {
 			unlink(*pinst + 7);
 		} else if (strncmp(*pinst, "redirect ", 9) == 0) {
 			int r = snprintf(opts->redirectsockname,
 			                 sizeof(opts->redirectsockname), "%s",
 			                 *pinst + 9);
 			if (r < 0 || r >= (int)sizeof(opts->redirectsockname))
 				abortmsg("redirect path is too long (%d)", r);
 			needreconnect = 1;
 		} else if (strncmp(*pinst, "exit ", 5) == 0) {
 			int n = 0;
 			if (sscanf(*pinst + 5, "%d", &n) != 1)
 				abortmsg("cannot read the exit code");
 			exit(n);
 		} else if (strcmp(*pinst, "reconnect") == 0) {
 			needreconnect = 1;
 		} else {
 			abortmsg("unknown instruction: %s", *pinst);
 		}
 	}
 	return needreconnect;
 }
 
 /*
  * Test whether the command is unsupported or not. This is not designed to
- * cover all cases. But it's fast, does not depend on the server and does
- * not return false positives.
+ * cover all cases. But it's fast, does not depend on the server.
  */
 static int isunsupported(int argc, const char *argv[])
 {
 	enum { SERVE = 1,
 	       DAEMON = 2,
 	       SERVEDAEMON = SERVE | DAEMON,
 	};
 	unsigned int state = 0;
 	int i;
 	for (i = 0; i < argc; ++i) {
 		if (strcmp(argv[i], "--") == 0)
 			break;
-		if (i == 0 && strcmp("serve", argv[i]) == 0)
+		/*
+		 * there can be false positives but no false negative
+		 * we cannot assume `serve` will always be first argument
+		 * because global options can be passed before the command name
+		 */
+		if (strcmp("serve", argv[i]) == 0)
 			state |= SERVE;
 		else if (strcmp("-d", argv[i]) == 0 ||
 		         strcmp("--daemon", argv[i]) == 0)
 			state |= DAEMON;
 	}
 	return (state & SERVEDAEMON) == SERVEDAEMON;
 }
 
 static void execoriginalhg(const char *argv[])
 {
 	debugmsg("execute original hg");
 	if (execvp(gethgcmd(), (char **)argv) < 0)
 		abortmsgerrno("failed to exec original hg");
 }
 
 int main(int argc, const char *argv[], const char *envp[])
 {
 	if (getenv("CHGDEBUG"))
 		enabledebugmsg();
 
 	if (!getenv("HGPLAIN") && isatty(fileno(stderr)))
 		enablecolor();
 
 	if (getenv("CHGINTERNALMARK"))
 		abortmsg("chg started by chg detected.\n"
 		         "Please make sure ${HG:-hg} is not a symlink or "
 		         "wrapper to chg. Alternatively, set $CHGHG to the "
 		         "path of real hg.");
 
 	if (isunsupported(argc - 1, argv + 1))
 		execoriginalhg(argv);
 
 	struct cmdserveropts opts;
 	initcmdserveropts(&opts);
 	setcmdserveropts(&opts);
 	setcmdserverargs(&opts, argc, argv);
 
 	if (argc == 2) {
 		if (strcmp(argv[1], "--kill-chg-daemon") == 0) {
 			killcmdserver(&opts);
 			return 0;
 		}
 	}
 
 	hgclient_t *hgc;
 	size_t retry = 0;
 	while (1) {
 		hgc = connectcmdserver(&opts);
 		if (!hgc)
 			abortmsg("cannot open hg client");
 		hgc_setenv(hgc, envp);
 		const char **insts = hgc_validate(hgc, argv + 1, argc - 1);
 		int needreconnect = runinstructions(&opts, insts);
 		free(insts);
 		if (!needreconnect)
 			break;
 		hgc_close(hgc);
 		if (++retry > 10)
 			abortmsg("too many redirections.\n"
 			         "Please make sure %s is not a wrapper which "
 			         "changes sensitive environment variables "
 			         "before executing hg. If you have to use a "
 			         "wrapper, wrap chg instead of hg.",
 			         gethgcmd());
 	}
 
 	setupsignalhandler(hgc_peerpid(hgc), hgc_peerpgid(hgc));
 	atexit(waitpager);
 	int exitcode = hgc_runcommand(hgc, argv + 1, argc - 1);
 	restoresignalhandler();
 	hgc_close(hgc);
 	freecmdserveropts(&opts);
 
 	return exitcode;
 }

contrib/examples/fix.hgrc

@@ -1,14 +1,14 b''
 [fix]
 clang-format:command = clang-format --style file
 clang-format:pattern = set:(**.c or **.cc or **.h) and not "include:contrib/clang-format-ignorelist"
 
 rustfmt:command = rustfmt +nightly
 rustfmt:pattern = set:**.rs
 
 black:command = black --config=black.toml -
-black:pattern = set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"
+black:pattern = set:**.py - mercurial/thirdparty/**
 
 # Mercurial doesn't have any Go code, but if we did this is how we
 # would configure `hg fix` for Go:
 go:command = gofmt
 go:pattern = set:**.go

contrib/heptapod-ci.yml

@@ -1,81 +1,81 b''
 image: octobus/ci-mercurial-core
 
 # The runner made a clone as root.
 # We make a new clone owned by user used to run the step.
 before_script:
     - hg clone . /tmp/mercurial-ci/ --noupdate
     - hg -R /tmp/mercurial-ci/ update `hg log --rev '.' --template '{node}'`
     - cd /tmp/mercurial-ci/
     - ls -1 tests/test-check-*.* > /tmp/check-tests.txt
 
 variables:
     PYTHON: python
     TEST_HGMODULEPOLICY: "allow"
 
 .runtests_template: &runtests
     script:
         - echo "python used, $PYTHON"
         - echo "$RUNTEST_ARGS"
         - HGMODULEPOLICY="$TEST_HGMODULEPOLICY" "$PYTHON" tests/run-tests.py --color=always $RUNTEST_ARGS
 
 checks-py2:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
 
 checks-py3:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--time --test-list /tmp/check-tests.txt"
         PYTHON: python3
 
 rust-cargo-test-py2: &rust_cargo_test
     script:
         - echo "python used, $PYTHON"
         - make rust-tests
 
 rust-cargo-test-py3:
     <<: *rust_cargo_test
     variables:
         PYTHON: python3
 
 test-py2:
     <<: *runtests
     variables:
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "c"
 
 test-py3:
     <<: *runtests
     variables:
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: " --no-rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "c"
 
 test-py2-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "py"
 
 test-py3-pure:
     <<: *runtests
     variables:
         RUNTEST_ARGS: "--pure --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "py"
 
 test-py2-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         TEST_HGMODULEPOLICY: "rust+c"
 
 test-py3-rust:
     <<: *runtests
     variables:
         HGWITHRUSTEXT: cpython
-        RUNTEST_ARGS: "--blacklist /tmp/check-tests.txt"
+        RUNTEST_ARGS: "--rust --blacklist /tmp/check-tests.txt"
         PYTHON: python3
         TEST_HGMODULEPOLICY: "rust+c"

contrib/import-checker.py

@@ -1,820 +1,821 b''
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import io
 import os
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True:  # disable lexical sorting checks
     try:
         import BaseHTTPServer as basehttpserver
     except ImportError:
         basehttpserver = None
     import zlib
 
 import testparseutil
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'bzrlib',
     'hgclient',
     'mercurial',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.interfaces',
     'mercurial.node',
     'mercurial.pycompat',
     # for revlog to re-export constant to extensions
     'mercurial.revlogutils.constants',
     'mercurial.revlogutils.flagutil',
     # for cffi modules to re-export pure functions
     'mercurial.pure.base85',
     'mercurial.pure.bdiff',
     'mercurial.pure.mpatch',
     'mercurial.pure.osutil',
     'mercurial.pure.parsers',
     # third-party imports should be directly imported
     'mercurial.thirdparty',
     'mercurial.thirdparty.attr',
     'mercurial.thirdparty.zope',
     'mercurial.thirdparty.zope.interface',
 )
 
 # Whitelist of symbols that can be directly imported.
 directsymbols = ('demandimport',)
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 
 def dotted_name_of_path(path):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0]  # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     return '.'.join(parts)
 
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
     paths of locally defined (= Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__', 'foo.foo1',
     ...              'foo.bar.__init__', 'foo.bar.bar1',
     ...              'baz.__init__', 'baz.baz1'}
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     if not isinstance(modulename, str):
         modulename = modulename.decode('ascii')
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
 
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = [
                     '.'.join(modulename.split('.')[:-level]) + '.' + name
                 ]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
 
     return fromlocal
 
 
 def populateextmods(localmods):
     """Populate C extension modules based on pure modules"""
     newlocalmods = set(localmods)
     for n in localmods:
         if n.startswith('mercurial.pure.'):
             m = n[len('mercurial.pure.') :]
             newlocalmods.add('mercurial.cext.' + m)
             newlocalmods.add('mercurial.cffi._' + m)
     return newlocalmods
 
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> py3 = sys.version_info[0] >= 3
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods or py3
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods or py3
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield '__builtin__'
     yield 'builtins'  # python3 only
     yield 'importlib.abc'  # python3 only
     yield 'importlib.machinery'  # python3 only
     yield 'importlib.util'  # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime':  # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = {sys.prefix, sys.exec_prefix}
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (basehttpserver, zlib):
         if mod is None:
             continue
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with something in
         # stdlib_prefixes, but not directories from the hg sources.
         if os.path.abspath(libpath).startswith(sourceroot) or not any(
             libpath.startswith(p) for p in stdlib_prefixes
         ):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (
                     not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath
                     and d in ('hgdemandimport', 'hgext', 'mercurial')
                 ):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1 :]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 
 stdlib_modules = set(list_stdlib_modules())
 
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: set of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
-            if modnotfound:
+            if modnotfound and dottedpath != modulename:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
+                # specifically allow "from . import foo" from __init__.py
                 yield dottedpath
 
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed, unless that
       library is also a local module.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     if not isinstance(module, str):
         module = module.decode('ascii')
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
 
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
 
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(
                 module, node, localmods, node.col_offset + 4
             ):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg(
                     'multiple imported names: %s',
                     ', '.join(n.name for n in node.names),
                 )
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg(
                         'imports not lexically sorted: %s < %s', name, lastname
                     )
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg(
                     'stdlib import "%s" follows local import: %s',
                     name,
                     seenlocal,
                 )
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg(
                     '%s module must be "as" aliased to %s',
                     name,
                     requirealias[name],
                 )
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[: -node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or (
                     fullname in stdlib_modules
                     # allow standard 'from typing import ...' style
                     and fullname.startswith('.')
                     and fullname not in localmods
                     and fullname + '.__init__' not in localmods
                 ):
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = (
                     n.name for n in node.names if not fromlocal(prefix + n.name)
                 )
             else:
                 symbols = (n.name for n in node.names)
             symbols = [sym for sym in symbols if sym not in directsymbols]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg(
                         'direct symbol import %s from %s',
                         ', '.join(symbols),
                         fullname,
                     )
 
                 if symbols and seennonsymbollocal:
                     yield msg(
                         'symbol import follows non-symbol import: %s', fullname
                     )
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (
                     node.level in seenlevels
                     and node.col_offset == root_col_offset
                 ):
                     yield msg(
                         'multiple "from %s import" statements', '.' * node.level
                     )
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg(
                         'higher-level import should come first: %s', fullname
                     )
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg(
                         'imports from %s not lexically sorted: %s < %s',
                         fullname,
                         n.name,
                         lastentryname,
                     )
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg(
                         '%s from %s must be "as" aliased to %s',
                         n.name,
                         fullname,
                         requirealias[n.name],
                     )
 
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield (
                     'mixed imports\n   stdlib:    %s\n   relative:  %s'
                     % (
                         ', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False])),
                     ),
                     node.lineno,
                 )
 
 
 class CircularImport(Exception):
     pass
 
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 
 def _cycle_sortkey(c):
     return len(c), c
 
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def _forcestr(thing):
     ...     if not isinstance(thing, str):
     ...         return thing.decode('ascii')
     ...     return thing
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, b"example", lines):
     ...         print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
     ...         print(repr(_forcestr(s)))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test(b"example.t", lines)
     example[2] doctest.py 1
     "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
     example[8] foo.py 7
     'from __future__ import print_function\\n'
     """
     errors = []
     for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
         if not name:
             # use 'doctest.py', in order to make already existing
             # doctest above pass instantly
             name = 'doctest.py'
         # "starts" is "line number" (1-origin), but embedded() is
         # expected to return "line offset" (0-origin). Therefore, this
         # yields "starts - 1".
         if not isinstance(modname, str):
             modname = modname.decode('utf8')
         yield code, "%s[%d]" % (modname, starts), name, starts - 1
 
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f, 'rb') as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         # Strictly speaking we should sniff for the magic header that denotes
         # Python source file encoding. But in reality we don't use anything
         # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
         # simplicity is fine.
         with io.open(f, 'r', encoding='utf-8') as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname.encode('utf8'), t, line
 
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmodpaths = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path)
         localmodpaths[modname] = source_path
     localmods = populateextmods(localmodpaths)
     for localmodname, source_path in sorted(localmodpaths.items()):
         if not isinstance(localmodname, bytes):
             # This is only safe because all hg's files are ascii
             localmodname = localmodname.encode('ascii')
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(
                         src, modname, name, localmods, ignore_nested=True
                     )
                 )
                 for error, lineno in verify_import_convention(
                     modname, src, localmods
                 ):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print(
                     '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
                 )
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

contrib/perf.py

@@ -1,3853 +1,3854 b''
 # perf.py - performance test routines
 '''helper extension to measure performance
 
 Configurations
 ==============
 
 ``perf``
 --------
 
 ``all-timing``
     When set, additional statistics will be reported for each benchmark: best,
     worst, median average. If not set only the best timing is reported
     (default: off).
 
 ``presleep``
   number of second to wait before any group of runs (default: 1)
 
 ``pre-run``
   number of run to perform before starting measurement.
 
 ``profile-benchmark``
   Enable profiling for the benchmarked section.
   (The first iteration is benchmarked)
 
 ``run-limits``
   Control the number of runs each benchmark will perform. The option value
   should be a list of `<time>-<numberofrun>` pairs. After each run the
   conditions are considered in order with the following logic:
 
       If benchmark has been running for <time> seconds, and we have performed
       <numberofrun> iterations, stop the benchmark,
 
   The default value is: `3.0-100, 10.0-3`
 
 ``stub``
     When set, benchmarks will only be run once, useful for testing
     (default: off)
 '''
 
 # "historical portability" policy of perf.py:
 #
 # We have to do:
 # - make perf.py "loadable" with as wide Mercurial version as possible
 #   This doesn't mean that perf commands work correctly with that Mercurial.
 #   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
 # - make historical perf command work correctly with as wide Mercurial
 #   version as possible
 #
 # We have to do, if possible with reasonable cost:
 # - make recent perf command for historical feature work correctly
 #   with early Mercurial
 #
 # We don't have to do:
 # - make perf command for recent feature work correctly with early
 #   Mercurial
 
 from __future__ import absolute_import
 import contextlib
 import functools
 import gc
 import os
 import random
 import shutil
 import struct
 import sys
 import tempfile
 import threading
 import time
 from mercurial import (
     changegroup,
     cmdutil,
     commands,
     copies,
     error,
     extensions,
     hg,
     mdiff,
     merge,
     revlog,
     util,
 )
 
 # for "historical portability":
 # try to import modules separately (in dict order), and ignore
 # failure, because these aren't available with early Mercurial
 try:
     from mercurial import branchmap  # since 2.5 (or bcee63733aad)
 except ImportError:
     pass
 try:
     from mercurial import obsolete  # since 2.3 (or ad0d6c2b3279)
 except ImportError:
     pass
 try:
     from mercurial import registrar  # since 3.7 (or 37d50250b696)
 
     dir(registrar)  # forcibly load it
 except ImportError:
     registrar = None
 try:
     from mercurial import repoview  # since 2.5 (or 3a6ddacb7198)
 except ImportError:
     pass
 try:
     from mercurial.utils import repoviewutil  # since 5.0
 except ImportError:
     repoviewutil = None
 try:
     from mercurial import scmutil  # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
 try:
     from mercurial import setdiscovery  # since 1.9 (or cb98fed52495)
 except ImportError:
     pass
 
 try:
     from mercurial import profiling
 except ImportError:
     profiling = None
 
 
 def identity(a):
     return a
 
 
 try:
     from mercurial import pycompat
 
     getargspec = pycompat.getargspec  # added to module after 4.5
     _byteskwargs = pycompat.byteskwargs  # since 4.1 (or fbc3f73dc802)
     _sysstr = pycompat.sysstr  # since 4.0 (or 2219f4f82ede)
     _bytestr = pycompat.bytestr  # since 4.2 (or b70407bd84d5)
     _xrange = pycompat.xrange  # since 4.8 (or 7eba8f83129b)
     fsencode = pycompat.fsencode  # since 3.9 (or f4a5e0e86a7e)
     if pycompat.ispy3:
         _maxint = sys.maxsize  # per py3 docs for replacing maxint
     else:
         _maxint = sys.maxint
 except (NameError, ImportError, AttributeError):
     import inspect
 
     getargspec = inspect.getargspec
     _byteskwargs = identity
     _bytestr = str
     fsencode = identity  # no py3 support
     _maxint = sys.maxint  # no py3 support
     _sysstr = lambda x: x  # no py3 support
     _xrange = xrange
 
 try:
     # 4.7+
     queue = pycompat.queue.Queue
 except (NameError, AttributeError, ImportError):
     # <4.7.
     try:
         queue = pycompat.queue
     except (NameError, AttributeError, ImportError):
         import Queue as queue
 
 try:
     from mercurial import logcmdutil
 
     makelogtemplater = logcmdutil.maketemplater
 except (AttributeError, ImportError):
     try:
         makelogtemplater = cmdutil.makelogtemplater
     except (AttributeError, ImportError):
         makelogtemplater = None
 
 # for "historical portability":
 # define util.safehasattr forcibly, because util.safehasattr has been
 # available since 1.9.3 (or 94b200a11cf7)
 _undefined = object()
 
 
 def safehasattr(thing, attr):
     return getattr(thing, _sysstr(attr), _undefined) is not _undefined
 
 
 setattr(util, 'safehasattr', safehasattr)
 
 # for "historical portability":
 # define util.timer forcibly, because util.timer has been available
 # since ae5d60bb70c9
 if safehasattr(time, 'perf_counter'):
     util.timer = time.perf_counter
 elif os.name == b'nt':
     util.timer = time.clock
 else:
     util.timer = time.time
 
 # for "historical portability":
 # use locally defined empty option list, if formatteropts isn't
 # available, because commands.formatteropts has been available since
 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
 # available since 2.2 (or ae5f92e154d3)
 formatteropts = getattr(
     cmdutil, "formatteropts", getattr(commands, "formatteropts", [])
 )
 
 # for "historical portability":
 # use locally defined option list, if debugrevlogopts isn't available,
 # because commands.debugrevlogopts has been available since 3.7 (or
 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
 # since 1.9 (or a79fea6b3e77).
 revlogopts = getattr(
     cmdutil,
     "debugrevlogopts",
     getattr(
         commands,
         "debugrevlogopts",
         [
             (b'c', b'changelog', False, b'open changelog'),
             (b'm', b'manifest', False, b'open manifest'),
             (b'', b'dir', False, b'open directory manifest'),
         ],
     ),
 )
 
 cmdtable = {}
 
 # for "historical portability":
 # define parsealiases locally, because cmdutil.parsealiases has been
 # available since 1.5 (or 6252852b4332)
 def parsealiases(cmd):
     return cmd.split(b"|")
 
 
 if safehasattr(registrar, 'command'):
     command = registrar.command(cmdtable)
 elif safehasattr(cmdutil, 'command'):
     command = cmdutil.command(cmdtable)
     if b'norepo' not in getargspec(command).args:
         # for "historical portability":
         # wrap original cmdutil.command, because "norepo" option has
         # been available since 3.1 (or 75a96326cecb)
         _command = command
 
         def command(name, options=(), synopsis=None, norepo=False):
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return _command(name, list(options), synopsis)
 
 
 else:
     # for "historical portability":
     # define "@command" annotation locally, because cmdutil.command
     # has been available since 1.9 (or 2daa5179e73f)
     def command(name, options=(), synopsis=None, norepo=False):
         def decorator(func):
             if synopsis:
                 cmdtable[name] = func, list(options), synopsis
             else:
                 cmdtable[name] = func, list(options)
             if norepo:
                 commands.norepo += b' %s' % b' '.join(parsealiases(name))
             return func
 
         return decorator
 
 
 try:
     import mercurial.registrar
     import mercurial.configitems
 
     configtable = {}
     configitem = mercurial.registrar.configitem(configtable)
     configitem(
         b'perf',
         b'presleep',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'stub',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'parentscount',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf',
         b'all-timing',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
     configitem(
         b'perf', b'pre-run', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'run-limits',
         default=mercurial.configitems.dynamicdefault,
         experimental=True,
     )
 except (ImportError, AttributeError):
     pass
 except TypeError:
     # compatibility fix for a11fd395e83f
     # hg version: 5.2
     configitem(
         b'perf', b'presleep', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf', b'stub', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf', b'parentscount', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf', b'all-timing', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf', b'pre-run', default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf',
         b'profile-benchmark',
         default=mercurial.configitems.dynamicdefault,
     )
     configitem(
         b'perf', b'run-limits', default=mercurial.configitems.dynamicdefault,
     )
 
 
 def getlen(ui):
     if ui.configbool(b"perf", b"stub", False):
         return lambda x: 1
     return len
 
 
 class noop(object):
     """dummy context manager"""
 
     def __enter__(self):
         pass
 
     def __exit__(self, *args):
         pass
 
 
 NOOPCTX = noop()
 
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(getint(ui, b"perf", b"presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr unless buffer api is in use
     if not ui._buffers:
         ui = ui.copy()
         uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
         if uifout:
             # for "historical portability":
             # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
             uifout.set(ui.ferr)
 
     # get a formatter
     uiformatter = getattr(ui, 'formatter', None)
     if uiformatter:
         fm = uiformatter(b'perf', opts)
     else:
         # for "historical portability":
         # define formatter locally, because ui.formatter has been
         # available since 2.2 (or ae5f92e154d3)
         from mercurial import node
 
         class defaultformatter(object):
             """Minimized composition of baseformatter and plainformatter
             """
 
             def __init__(self, ui, topic, opts):
                 self._ui = ui
                 if ui.debugflag:
                     self.hexfunc = node.hex
                 else:
                     self.hexfunc = node.short
 
             def __nonzero__(self):
                 return False
 
             __bool__ = __nonzero__
 
             def startitem(self):
                 pass
 
             def data(self, **data):
                 pass
 
             def write(self, fields, deftext, *fielddata, **opts):
                 self._ui.write(deftext % fielddata, **opts)
 
             def condwrite(self, cond, fields, deftext, *fielddata, **opts):
                 if cond:
                     self._ui.write(deftext % fielddata, **opts)
 
             def plain(self, text, **opts):
                 self._ui.write(text, **opts)
 
             def end(self):
                 pass
 
         fm = defaultformatter(ui, b'perf', opts)
 
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool(b"perf", b"stub", False):
         return functools.partial(stub_timer, fm), fm
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # experimental config: perf.run-limits
     limitspec = ui.configlist(b"perf", b"run-limits", [])
     limits = []
     for item in limitspec:
         parts = item.split(b'-', 1)
         if len(parts) < 2:
             ui.warn((b'malformatted run limit entry, missing "-": %s\n' % item))
             continue
         try:
             time_limit = float(_sysstr(parts[0]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         try:
             run_limit = int(_sysstr(parts[1]))
         except ValueError as e:
             ui.warn(
                 (
                     b'malformatted run limit entry, %s: %s\n'
                     % (_bytestr(e), item)
                 )
             )
             continue
         limits.append((time_limit, run_limit))
     if not limits:
         limits = DEFAULTLIMITS
 
     profiler = None
     if profiling is not None:
         if ui.configbool(b"perf", b"profile-benchmark", False):
             profiler = profiling.profile(ui)
 
     prerun = getint(ui, b"perf", b"pre-run", 0)
     t = functools.partial(
         _timer,
         fm,
         displayall=displayall,
         limits=limits,
         prerun=prerun,
         profiler=profiler,
     )
     return t, fm
 
 
 def stub_timer(fm, func, setup=None, title=None):
     if setup is not None:
         setup()
     func()
 
 
 @contextlib.contextmanager
 def timeone():
     r = []
     ostart = os.times()
     cstart = util.timer()
     yield r
     cstop = util.timer()
     ostop = os.times()
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1] - a[1]))
 
 
 # list of stop condition (elapsed time, minimal run count)
 DEFAULTLIMITS = (
     (3.0, 100),
     (10.0, 3),
 )
 
 
 def _timer(
     fm,
     func,
     setup=None,
     title=None,
     displayall=False,
     limits=DEFAULTLIMITS,
     prerun=0,
     profiler=None,
 ):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
     if profiler is None:
         profiler = NOOPCTX
     for i in range(prerun):
         if setup is not None:
             setup()
         func()
     keepgoing = True
     while keepgoing:
         if setup is not None:
             setup()
         with profiler:
             with timeone() as item:
                 r = func()
         profiler = NOOPCTX
         count += 1
         results.append(item[0])
         cstop = util.timer()
         # Look for a stop condition.
         elapsed = cstop - begin
         for t, mincount in limits:
             if elapsed >= t and count >= mincount:
                 keepgoing = False
                 break
 
     formatone(fm, results, title=title, result=r, displayall=displayall)
 
 
 def formatone(fm, timings, title=None, result=None, displayall=False):
 
     count = len(timings)
 
     fm.startitem()
 
     if title:
         fm.write(b'title', b'! %s\n', title)
     if result:
         fm.write(b'result', b'! result: %s\n', result)
 
     def display(role, entry):
         prefix = b''
         if role != b'best':
             prefix = b'%s.' % role
         fm.plain(b'!')
         fm.write(prefix + b'wall', b' wall %f', entry[0])
         fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
         fm.write(prefix + b'user', b' user %f', entry[1])
         fm.write(prefix + b'sys', b' sys %f', entry[2])
         fm.write(prefix + b'count', b' (%s of %%d)' % role, count)
         fm.plain(b'\n')
 
     timings.sort()
     min_val = timings[0]
     display(b'best', min_val)
     if displayall:
         max_val = timings[-1]
         display(b'max', max_val)
         avg = tuple([sum(x) / count for x in zip(*timings)])
         display(b'avg', avg)
         median = timings[len(timings) // 2]
         display(b'median', median)
 
 
 # utilities for historical portability
 
 
 def getint(ui, section, name, default):
     # for "historical portability":
     # ui.configint has been available since 1.9 (or fa2b596db182)
     v = ui.config(section, name, None)
     if v is None:
         return default
     try:
         return int(v)
     except ValueError:
         raise error.ConfigError(
             b"%s.%s is not an integer ('%s')" % (section, name, v)
         )
 
 
 def safeattrsetter(obj, name, ignoremissing=False):
     """Ensure that 'obj' has 'name' attribute before subsequent setattr
 
     This function is aborted, if 'obj' doesn't have 'name' attribute
     at runtime. This avoids overlooking removal of an attribute, which
     breaks assumption of performance measurement, in the future.
 
     This function returns the object to (1) assign a new value, and
     (2) restore an original value to the attribute.
 
     If 'ignoremissing' is true, missing 'name' attribute doesn't cause
     abortion, and this function returns None. This is useful to
     examine an attribute, which isn't ensured in all Mercurial
     versions.
     """
     if not util.safehasattr(obj, name):
         if ignoremissing:
             return None
         raise error.Abort(
             (
                 b"missing attribute %s of %s might break assumption"
                 b" of performance measurement"
             )
             % (name, obj)
         )
 
     origvalue = getattr(obj, _sysstr(name))
 
     class attrutil(object):
         def set(self, newvalue):
             setattr(obj, _sysstr(name), newvalue)
 
         def restore(self):
             setattr(obj, _sysstr(name), origvalue)
 
     return attrutil()
 
 
 # utilities to examine each internal API changes
 
 
 def getbranchmapsubsettable():
     # for "historical portability":
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
     # - repoviewutil since 5.0
     for mod in (branchmap, repoview, repoviewutil):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
 
     # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
     # branchmap and repoview modules exist, but subsettable attribute
     # doesn't)
     raise error.Abort(
         b"perfbranchmap not available with this Mercurial",
         hint=b"use 2.5 or later",
     )
 
 
 def getsvfs(repo):
     """Return appropriate object to access files under .hg/store
     """
     # for "historical portability":
     # repo.svfs has been available since 2.3 (or 7034365089bf)
     svfs = getattr(repo, 'svfs', None)
     if svfs:
         return svfs
     else:
         return getattr(repo, 'sopener')
 
 
 def getvfs(repo):
     """Return appropriate object to access files under .hg
     """
     # for "historical portability":
     # repo.vfs has been available since 2.3 (or 7034365089bf)
     vfs = getattr(repo, 'vfs', None)
     if vfs:
         return vfs
     else:
         return getattr(repo, 'opener')
 
 
 def repocleartagscachefunc(repo):
     """Return the function to clear tags cache according to repo internal API
     """
     if util.safehasattr(repo, b'_tagscache'):  # since 2.0 (or 9dca7653b525)
         # in this case, setattr(repo, '_tagscache', None) or so isn't
         # correct way to clear tags cache, because existing code paths
         # expect _tagscache to be a structured object.
         def clearcache():
             # _tagscache has been filteredpropertycache since 2.5 (or
             # 98c867ac1330), and delattr() can't work in such case
             if '_tagscache' in vars(repo):
                 del repo.__dict__['_tagscache']
 
         return clearcache
 
     repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
     if repotags:  # since 1.4 (or 5614a628d173)
         return lambda: repotags.set(None)
 
     repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
     if repotagscache:  # since 0.6 (or d7df759d0e97)
         return lambda: repotagscache.set(None)
 
     # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
     # this point, but it isn't so problematic, because:
     # - repo.tags of such Mercurial isn't "callable", and repo.tags()
     #   in perftags() causes failure soon
     # - perf.py itself has been available since 1.1 (or eb240755386d)
     raise error.Abort(b"tags API of this hg command is unknown")
 
 
 # utilities to clear cache
 
 
 def clearfilecache(obj, attrname):
     unfiltered = getattr(obj, 'unfiltered', None)
     if unfiltered is not None:
         obj = obj.unfiltered()
     if attrname in vars(obj):
         delattr(obj, attrname)
     obj._filecache.pop(attrname, None)
 
 
 def clearchangelog(repo):
     if repo is not repo.unfiltered():
         object.__setattr__(repo, '_clcachekey', None)
         object.__setattr__(repo, '_clcache', None)
     clearfilecache(repo.unfiltered(), 'changelog')
 
 
 # perf commands
 
 
 @command(b'perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     m = scmutil.match(repo[None], pats, {})
     timer(
         lambda: len(
             list(
                 repo.dirstate.walk(m, subrepos=[], unknown=True, ignored=False)
             )
         )
     )
     fm.end()
 
 
 @command(b'perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     fc = repo[b'.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 
 @command(
     b'perfstatus',
     [
         (b'u', b'unknown', False, b'ask status to look for unknown files'),
         (b'', b'dirstate', False, b'benchmark the internal dirstate call'),
     ]
     + formatteropts,
 )
 def perfstatus(ui, repo, **opts):
     """benchmark the performance of a single status call
 
     The repository data are preserved between each call.
 
     By default, only the status of the tracked file are requested. If
     `--unknown` is passed, the "unknown" files are also tracked.
     """
     opts = _byteskwargs(opts)
     # m = match.always(repo.root, repo.getcwd())
     # timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     if opts[b'dirstate']:
         dirstate = repo.dirstate
         m = scmutil.matchall(repo)
         unknown = opts[b'unknown']
 
         def status_dirstate():
             s = dirstate.status(
                 m, subrepos=[], ignored=False, clean=False, unknown=unknown
             )
             sum(map(bool, s))
 
         timer(status_dirstate)
     else:
         timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
     fm.end()
 
 
 @command(b'perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
         if b'uipathfn' in getargspec(scmutil.addremove).args:
             uipathfn = scmutil.getuipathfn(repo)
             timer(lambda: scmutil.addremove(repo, matcher, b"", uipathfn, opts))
         else:
             timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, b'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, b'_nodecache'):
         # <= hg-5.2
         from mercurial.node import nullid, nullrev
 
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 
 @command(b'perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     """benchmark the computation of a changelog heads"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def s():
         clearcaches(cl)
 
     def d():
         len(cl.headrevs())
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perftags',
     formatteropts
     + [(b'', b'clear-revlogs', False, b'refresh changelog and manifest'),],
 )
 def perftags(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repocleartagscache = repocleartagscachefunc(repo)
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
             clearfilecache(repo.unfiltered(), 'manifest')
         repocleartagscache()
 
     def t():
         return len(repo.tags())
 
     timer(t, setup=s)
     fm.end()
 
 
 @command(b'perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
 
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
 
     timer(d)
     fm.end()
 
 
 @command(b'perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
 
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
 
     timer(d)
     fm.end()
 
 
 @command(b'perfdiscovery', formatteropts, b'PATH')
 def perfdiscovery(ui, repo, path, **opts):
     """benchmark discovery between local repo and the peer at given path
     """
     repos = [repo, None]
     timer, fm = gettimer(ui, opts)
     path = ui.expandpath(path)
 
     def s():
         repos[1] = hg.peer(ui, opts, path)
 
     def d():
         setdiscovery.findcommonheads(ui, *repos)
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(
     b'perfbookmarks',
     formatteropts
     + [(b'', b'clear-revlogs', False, b'refresh changelog and manifest'),],
 )
 def perfbookmarks(ui, repo, **opts):
     """benchmark parsing bookmarks from disk to memory"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     clearrevlogs = opts[b'clear_revlogs']
 
     def s():
         if clearrevlogs:
             clearchangelog(repo)
         clearfilecache(repo, b'_bookmarks')
 
     def d():
         repo._bookmarks
 
     timer(d, setup=s)
     fm.end()
 
 
 @command(b'perfbundleread', formatteropts, b'BUNDLE')
 def perfbundleread(ui, repo, bundlepath, **opts):
     """Benchmark reading of bundle files.
 
     This command is meant to isolate the I/O part of bundle reading as
     much as possible.
     """
     from mercurial import (
         bundle2,
         exchange,
         streamclone,
     )
 
     opts = _byteskwargs(opts)
 
     def makebench(fn):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 fn(bundle)
 
         return run
 
     def makereadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 while bundle.read(size):
                     pass
 
         return run
 
     def makestdioread(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 while fh.read(size):
                     pass
 
         return run
 
     # bundle1
 
     def deltaiter(bundle):
         for delta in bundle.deltaiter():
             pass
 
     def iterchunks(bundle):
         for chunk in bundle.getchunks():
             pass
 
     # bundle2
 
     def forwardchunks(bundle):
         for chunk in bundle._forwardchunks():
             pass
 
     def iterparts(bundle):
         for part in bundle.iterparts():
             pass
 
     def iterpartsseekable(bundle):
         for part in bundle.iterparts(seekable=True):
             pass
 
     def seek(bundle):
         for part in bundle.iterparts(seekable=True):
             part.seek(0, os.SEEK_END)
 
     def makepartreadnbytes(size):
         def run():
             with open(bundlepath, b'rb') as fh:
                 bundle = exchange.readbundle(ui, fh, bundlepath)
                 for part in bundle.iterparts():
                     while part.read(size):
                         pass
 
         return run
 
     benches = [
         (makestdioread(8192), b'read(8k)'),
         (makestdioread(16384), b'read(16k)'),
         (makestdioread(32768), b'read(32k)'),
         (makestdioread(131072), b'read(128k)'),
     ]
 
     with open(bundlepath, b'rb') as fh:
         bundle = exchange.readbundle(ui, fh, bundlepath)
 
         if isinstance(bundle, changegroup.cg1unpacker):
             benches.extend(
                 [
                     (makebench(deltaiter), b'cg1 deltaiter()'),
                     (makebench(iterchunks), b'cg1 getchunks()'),
                     (makereadnbytes(8192), b'cg1 read(8k)'),
                     (makereadnbytes(16384), b'cg1 read(16k)'),
                     (makereadnbytes(32768), b'cg1 read(32k)'),
                     (makereadnbytes(131072), b'cg1 read(128k)'),
                 ]
             )
         elif isinstance(bundle, bundle2.unbundle20):
             benches.extend(
                 [
                     (makebench(forwardchunks), b'bundle2 forwardchunks()'),
                     (makebench(iterparts), b'bundle2 iterparts()'),
                     (
                         makebench(iterpartsseekable),
                         b'bundle2 iterparts() seekable',
                     ),
                     (makebench(seek), b'bundle2 part seek()'),
                     (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
                     (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
                     (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
                     (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
                 ]
             )
         elif isinstance(bundle, streamclone.streamcloneapplier):
             raise error.Abort(b'stream clone bundles not supported')
         else:
             raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perfchangegroupchangelog',
     formatteropts
     + [
         (b'', b'cgversion', b'02', b'changegroup version'),
         (b'r', b'rev', b'', b'revisions to add to changegroup'),
     ],
 )
 def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
     """Benchmark producing a changelog group for a changegroup.
 
     This measures the time spent processing the changelog during a
     bundle operation. This occurs during `hg bundle` and on a server
     processing a `getbundle` wire protocol request (handles clones
     and pull requests).
 
     By default, all revisions are added to the changegroup.
     """
     opts = _byteskwargs(opts)
     cl = repo.changelog
     nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
     bundler = changegroup.getbundler(cgversion, repo)
 
     def d():
         state, chunks = bundler._generatechangelog(cl, nodes)
         for chunk in chunks:
             pass
 
     timer, fm = gettimer(ui, opts)
 
     # Terminal printing can interfere with timing. So disable it.
     with ui.configoverride({(b'progress', b'disable'): True}):
         timer(d)
 
     fm.end()
 
 
 @command(b'perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     b'a' in dirstate
 
     def d():
         dirstate.hasdir(b'a')
         del dirstate._map._dirs
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perfdirstate',
     [
         (
             b'',
             b'iteration',
             None,
             b'benchmark a full iteration for the dirstate',
         ),
         (
             b'',
             b'contains',
             None,
             b'benchmark a large amount of `nf in dirstate` calls',
         ),
     ]
     + formatteropts,
 )
 def perfdirstate(ui, repo, **opts):
     """benchmap the time of various distate operations
 
     By default benchmark the time necessary to load a dirstate from scratch.
     The dirstate is loaded to the point were a "contains" request can be
     answered.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     b"a" in repo.dirstate
 
     if opts[b'iteration'] and opts[b'contains']:
         msg = b'only specify one of --iteration or --contains'
         raise error.Abort(msg)
 
     if opts[b'iteration']:
         setup = None
         dirstate = repo.dirstate
 
         def d():
             for f in dirstate:
                 pass
 
     elif opts[b'contains']:
         setup = None
         dirstate = repo.dirstate
         allfiles = list(dirstate)
         # also add file path that will be "missing" from the dirstate
         allfiles.extend([f[::-1] for f in allfiles])
 
         def d():
             for f in allfiles:
                 f in dirstate
 
     else:
 
         def setup():
             repo.dirstate.invalidate()
 
         def d():
             b"a" in repo.dirstate
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     """benchmap a 'dirstate.hasdir' call from an empty `dirs` cache
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo.dirstate.hasdir(b"a")
 
     def setup():
         del repo.dirstate._map._dirs
 
     def d():
         repo.dirstate.hasdir(b"a")
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.filefoldmap.get()` request
 
     The dirstate filefoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.filefoldmap.get(b'a')
 
     def setup():
         del dirstate._map.filefoldmap
 
     def d():
         dirstate._map.filefoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     """benchmap a `dirstate._map.dirfoldmap.get()` request
 
     The dirstate dirfoldmap cache is dropped between every request.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     dirstate._map.dirfoldmap.get(b'a')
 
     def setup():
         del dirstate._map.dirfoldmap
         del dirstate._map._dirs
 
     def d():
         dirstate._map.dirfoldmap.get(b'a')
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     """benchmap the time it take to write a dirstate on disk
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     b"a" in ds
 
     def setup():
         ds._dirty = True
 
     def d():
         ds.write(repo.currenttransaction())
 
     timer(d, setup=setup)
     fm.end()
 
 
 def _getmergerevs(repo, opts):
     """parse command argument to return rev involved in merge
 
     input: options dictionnary with `rev`, `from` and `bse`
     output: (localctx, otherctx, basectx)
     """
     if opts[b'from']:
         fromrev = scmutil.revsingle(repo, opts[b'from'])
         wctx = repo[fromrev]
     else:
         wctx = repo[None]
         # we don't want working dir files to be stat'd in the benchmark, so
         # prime that cache
         wctx.dirty()
     rctx = scmutil.revsingle(repo, opts[b'rev'], opts[b'rev'])
     if opts[b'base']:
         fromrev = scmutil.revsingle(repo, opts[b'base'])
         ancestor = repo[fromrev]
     else:
         ancestor = wctx.ancestor(rctx)
     return (wctx, rctx, ancestor)
 
 
 @command(
     b'perfmergecalculate',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecalculate(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(
             repo,
             wctx,
             rctx,
             [ancestor],
             branchmerge=False,
             force=False,
             acceptremote=True,
             followcopies=True,
         )
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perfmergecopies',
     [
         (b'r', b'rev', b'.', b'rev to merge against'),
         (b'', b'from', b'', b'rev to merge from'),
         (b'', b'base', b'', b'the revision to use as base'),
     ]
     + formatteropts,
 )
 def perfmergecopies(ui, repo, **opts):
     """measure runtime of `copies.mergecopies`"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     wctx, rctx, ancestor = _getmergerevs(repo, opts)
 
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         copies.mergecopies(repo, wctx, rctx, ancestor)
 
     timer(d)
     fm.end()
 
 
 @command(b'perfpathcopies', [], b"REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     """benchmark the copy tracing logic"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
 
     def d():
         copies.pathcopies(ctx1, ctx2)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perfphases',
     [(b'', b'full', False, b'include file reading time too'),],
     b"",
 )
 def perfphases(ui, repo, **opts):
     """benchmark phasesets computation"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     _phases = repo._phasecache
     full = opts.get(b'full')
 
     def d():
         phases = _phases
         if full:
             clearfilecache(repo, b'_phasecache')
             phases = repo._phasecache
         phases.invalidate()
         phases.loadphaserevs(repo)
 
     timer(d)
     fm.end()
 
 
 @command(b'perfphasesremote', [], b"[DEST]")
 def perfphasesremote(ui, repo, dest=None, **opts):
     """benchmark time needed to analyse phases of the remote server"""
     from mercurial.node import bin
     from mercurial import (
         exchange,
         hg,
         phases,
     )
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
     if not path:
         raise error.Abort(
             b'default repository not configured!',
             hint=b"see 'hg help config.paths'",
         )
     dest = path.pushloc or path.loc
     ui.statusnoi18n(b'analysing phase of %s\n' % util.hidepassword(dest))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
     op = exchange.pushoperation(repo, other)
     exchange._pushdiscoverychangeset(op)
 
     remotesubset = op.fallbackheads
 
     with other.commandexecutor() as e:
         remotephases = e.callcommand(
             b'listkeys', {b'namespace': b'phases'}
         ).result()
     del other
     publishing = remotephases.get(b'publishing', False)
     if publishing:
         ui.statusnoi18n(b'publishing: yes\n')
     else:
         ui.statusnoi18n(b'publishing: no\n')
 
     has_node = getattr(repo.changelog.index, 'has_node', None)
     if has_node is None:
         has_node = repo.changelog.nodemap.__contains__
     nonpublishroots = 0
     for nhex, phase in remotephases.iteritems():
         if nhex == b'publishing':  # ignore data related to publish option
             continue
         node = bin(nhex)
         if has_node(node) and int(phase):
             nonpublishroots += 1
     ui.statusnoi18n(b'number of roots: %d\n' % len(remotephases))
     ui.statusnoi18n(b'number of known non public roots: %d\n' % nonpublishroots)
 
     def d():
         phases.remotephasessummary(repo, remotesubset, remotephases)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perfmanifest',
     [
         (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
         (b'', b'clear-disk', False, b'clear on-disk caches too'),
     ]
     + formatteropts,
     b'REV|NODE',
 )
 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
     """benchmark the time to read a manifest from disk and return a usable
     dict-like object
 
     Manifest caches are cleared before retrieval."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     if not manifest_rev:
         ctx = scmutil.revsingle(repo, rev, rev)
         t = ctx.manifestnode()
     else:
         from mercurial.node import bin
 
         if len(rev) == 40:
             t = bin(rev)
         else:
             try:
                 rev = int(rev)
 
                 if util.safehasattr(repo.manifestlog, b'getstorage'):
                     t = repo.manifestlog.getstorage(b'').node(rev)
                 else:
                     t = repo.manifestlog._revlog.lookup(rev)
             except ValueError:
                 raise error.Abort(
                     b'manifest revision must be integer or full node'
                 )
 
     def d():
         repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
         repo.manifestlog[t].read()
 
     timer(d)
     fm.end()
 
 
 @command(b'perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     n = scmutil.revsingle(repo, rev).node()
 
     def d():
         repo.changelog.read(n)
         # repo.changelog._cache = None
 
     timer(d)
     fm.end()
 
 
 @command(b'perfignore', formatteropts)
 def perfignore(ui, repo, **opts):
     """benchmark operation related to computing ignore"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
 
     def setupone():
         dirstate.invalidate()
         clearfilecache(dirstate, b'_ignore')
 
     def runone():
         dirstate._ignore
 
     timer(runone, setup=setupone, title=b"load")
     fm.end()
 
 
 @command(
     b'perfindex',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'no-lookup', None, b'do not revision lookup post creation'),
     ]
     + formatteropts,
 )
 def perfindex(ui, repo, **opts):
     """benchmark index creation time followed by a lookup
 
     The default is to look `tip` up. Depending on the index implementation,
     the revision looked up can matters. For example, an implementation
     scanning the index will have a faster lookup time for `--rev tip` than for
     `--rev 0`. The number of looked up revisions and their order can also
     matters.
 
     Example of useful set to test:
+
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     It is not currently possible to check for lookup of a missing node. For
     deeper lookup benchmarking, checkout the `perfnodemap` command."""
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     if opts[b'no_lookup']:
         if opts['rev']:
             raise error.Abort('--no-lookup and --rev are mutually exclusive')
         nodes = []
     elif not opts[b'rev']:
         nodes = [repo[b"tip"].node()]
     else:
         revs = scmutil.revrange(repo, opts[b'rev'])
         cl = repo.changelog
         nodes = [cl.node(r) for r in revs]
 
     unfi = repo.unfiltered()
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
 
     def setup():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
 
     def d():
         cl = makecl(unfi)
         for n in nodes:
             cl.rev(n)
 
     timer(d, setup=setup)
     fm.end()
 
 
 @command(
     b'perfnodemap',
     [
         (b'', b'rev', [], b'revision to be looked up (default tip)'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ]
     + formatteropts,
 )
 def perfnodemap(ui, repo, **opts):
     """benchmark the time necessary to look up revision from a cold nodemap
 
     Depending on the implementation, the amount and order of revision we look
     up can varies. Example of useful set to test:
     * tip
     * 0
     * -10:
     * :10
     * -10: + :10
     * :10: + -10:
     * -10000:
     * -10000: + 0
 
     The command currently focus on valid binary lookup. Benchmarking for
     hexlookup, prefix lookup and missing lookup would also be valuable.
     """
     import mercurial.revlog
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
 
     unfi = repo.unfiltered()
     clearcaches = opts['clear_caches']
     # find the filecache func directly
     # This avoid polluting the benchmark with the filecache logic
     makecl = unfi.__class__.changelog.func
     if not opts[b'rev']:
         raise error.Abort('use --rev to specify revisions to look up')
     revs = scmutil.revrange(repo, opts[b'rev'])
     cl = repo.changelog
     nodes = [cl.node(r) for r in revs]
 
     # use a list to pass reference to a nodemap from one closure to the next
     nodeget = [None]
 
     def setnodeget():
         # probably not necessary, but for good measure
         clearchangelog(unfi)
         cl = makecl(unfi)
         if util.safehasattr(cl.index, 'get_rev'):
             nodeget[0] = cl.index.get_rev
         else:
             nodeget[0] = cl.nodemap.get
 
     def d():
         get = nodeget[0]
         for n in nodes:
             get(n)
 
     setup = None
     if clearcaches:
 
         def setup():
             setnodeget()
 
     else:
         setnodeget()
         d()  # prewarm the data structure
     timer(d, setup=setup)
     fm.end()
 
 
 @command(b'perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def d():
         if os.name != 'nt':
             os.system(
                 b"HGRCPATH= %s version -q > /dev/null" % fsencode(sys.argv[0])
             )
         else:
             os.environ['HGRCPATH'] = r' '
             os.system("%s version -q > NUL" % sys.argv[0])
 
     timer(d)
     fm.end()
 
 
 @command(b'perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     """benchmark the time necessary to fetch one changeset's parents.
 
     The fetch is done using the `node identifier`, traversing all object layers
     from the repository object. The first N revisions will be used for this
     benchmark. N is controlled by the ``perf.parentscount`` config option
     (default: 1000).
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = getint(ui, b"perf", b"parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort(b"repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in _xrange(count)]
 
     def d():
         for n in nl:
             repo.changelog.parents(n)
 
     timer(d)
     fm.end()
 
 
 @command(b'perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
 
     def d():
         len(repo[x].files())
 
     timer(d)
     fm.end()
 
 
 @command(b'perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     opts = _byteskwargs(opts)
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
 
     def d():
         len(cl.read(x)[3])
 
     timer(d)
     fm.end()
 
 
 @command(b'perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 
 @command(
     b'perflinelogedits',
     [
         (b'n', b'edits', 10000, b'number of edits'),
         (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
     ],
     norepo=True,
 )
 def perflinelogedits(ui, **opts):
     from mercurial import linelog
 
     opts = _byteskwargs(opts)
 
     edits = opts[b'edits']
     maxhunklines = opts[b'max_hunk_lines']
 
     maxb1 = 100000
     random.seed(0)
     randint = random.randint
     currentlines = 0
     arglist = []
     for rev in _xrange(edits):
         a1 = randint(0, currentlines)
         a2 = randint(a1, min(currentlines, a1 + maxhunklines))
         b1 = randint(0, maxb1)
         b2 = randint(b1, b1 + maxhunklines)
         currentlines += (b2 - b1) - (a2 - a1)
         arglist.append((rev, a1, a2, b1, b2))
 
     def d():
         ll = linelog.linelog()
         for args in arglist:
             ll.replacelines(*args)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 
 @command(b'perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
 
     mercurial.revlog._prereadsize = 2 ** 24  # disable lazy parser in old hg
     n = scmutil.revsingle(repo, rev).node()
     cl = mercurial.revlog.revlog(getsvfs(repo), b"00changelog.i")
 
     def d():
         cl.rev(n)
         clearcaches(cl)
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perflog',
     [(b'', b'rename', False, b'ask log to follow renames')] + formatteropts,
 )
 def perflog(ui, repo, rev=None, **opts):
     opts = _byteskwargs(opts)
     if rev is None:
         rev = []
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(
         lambda: commands.log(
             ui, repo, rev=rev, date=b'', user=b'', copies=opts.get(b'rename')
         )
     )
     ui.popbuffer()
     fm.end()
 
 
 @command(b'perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
 
     def moonwalk():
         for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
             ctx = repo[i]
             ctx.branch()  # read changelog data (in addition to the index)
 
     timer(moonwalk)
     fm.end()
 
 
 @command(
     b'perftemplating',
     [(b'r', b'rev', [], b'revisions to run the template on'),] + formatteropts,
 )
 def perftemplating(ui, repo, testedtemplate=None, **opts):
     """test the rendering time of a given template"""
     if makelogtemplater is None:
         raise error.Abort(
             b"perftemplating not available with this Mercurial",
             hint=b"use 4.3 or later",
         )
 
     opts = _byteskwargs(opts)
 
     nullui = ui.copy()
     nullui.fout = open(os.devnull, 'wb')
     nullui.disablepager()
     revs = opts.get(b'rev')
     if not revs:
         revs = [b'all()']
     revs = list(scmutil.revrange(repo, revs))
 
     defaulttemplate = (
         b'{date|shortdate} [{rev}:{node|short}]'
         b' {author|person}: {desc|firstline}\n'
     )
     if testedtemplate is None:
         testedtemplate = defaulttemplate
     displayer = makelogtemplater(nullui, repo, testedtemplate)
 
     def format():
         for r in revs:
             ctx = repo[r]
             displayer.show(ctx)
             displayer.flush(ctx)
 
     timer, fm = gettimer(ui, opts)
     timer(format)
     fm.end()
 
 
 def _displaystats(ui, opts, entries, data):
     # use a second formatter because the data are quite different, not sure
     # how it flies with the templater.
     fm = ui.formatter(b'perf-stats', opts)
     for key, title in entries:
         values = data[key]
         nbvalues = len(data)
         values.sort()
         stats = {
             'key': key,
             'title': title,
             'nbitems': len(values),
             'min': values[0][0],
             '10%': values[(nbvalues * 10) // 100][0],
             '25%': values[(nbvalues * 25) // 100][0],
             '50%': values[(nbvalues * 50) // 100][0],
             '75%': values[(nbvalues * 75) // 100][0],
             '80%': values[(nbvalues * 80) // 100][0],
             '85%': values[(nbvalues * 85) // 100][0],
             '90%': values[(nbvalues * 90) // 100][0],
             '95%': values[(nbvalues * 95) // 100][0],
             '99%': values[(nbvalues * 99) // 100][0],
             'max': values[-1][0],
         }
         fm.startitem()
         fm.data(**stats)
         # make node pretty for the human output
         fm.plain('### %s (%d items)\n' % (title, len(values)))
         lines = [
             'min',
             '10%',
             '25%',
             '50%',
             '75%',
             '80%',
             '85%',
             '90%',
             '95%',
             '99%',
             'max',
         ]
         for l in lines:
             fm.plain('%s: %s\n' % (l, stats[l]))
     fm.end()
 
 
 @command(
     b'perfhelper-mergecopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelpermergecopies(ui, repo, revs=[], **opts):
     """find statistics about potential parameters for `perfmergecopies`
 
     This command find (base, p1, p2) triplet relevant for copytracing
     benchmarking in the context of a merge.  It reports values for some of the
     parameters that impact merge copy tracing time during merge.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details come at the cost of slower command
     execution.
 
     Since rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision triplets are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     output_template = [
         ("base", "%(base)12s"),
         ("p1", "%(p1.node)12s"),
         ("p2", "%(p2.node)12s"),
         ("p1.nb-revs", "%(p1.nbrevs)12d"),
         ("p1.nb-files", "%(p1.nbmissingfiles)12d"),
         ("p1.renames", "%(p1.renamedfiles)12d"),
         ("p1.time", "%(p1.time)12.3f"),
         ("p2.nb-revs", "%(p2.nbrevs)12d"),
         ("p2.nb-files", "%(p2.nbmissingfiles)12d"),
         ("p2.renames", "%(p2.renamedfiles)12d"),
         ("p2.time", "%(p2.time)12.3f"),
         ("renames", "%(nbrenamedfiles)12d"),
         ("total.time", "%(time)12.3f"),
     ]
     if not dotiming:
         output_template = [
             i
             for i in output_template
             if not ('time' in i[0] or 'renames' in i[0])
         ]
     header_names = [h for (h, v) in output_template]
     output = ' '.join([v for (h, v) in output_template]) + '\n'
     header = ' '.join(['%12s'] * len(header_names)) + '\n'
     fm.plain(header % tuple(header_names))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['parentnbrenames'] = []
             alldata['totalnbrenames'] = []
             alldata['parenttime'] = []
             alldata['totaltime'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1()
         p2 = ctx.p2()
         bases = repo.changelog._commonancestorsheads(p1.rev(), p2.rev())
         for b in bases:
             b = repo[b]
             p1missing = copies._computeforwardmissing(b, p1)
             p2missing = copies._computeforwardmissing(b, p2)
             data = {
                 b'base': b.hex(),
                 b'p1.node': p1.hex(),
                 b'p1.nbrevs': len(repo.revs('only(%d, %d)', p1.rev(), b.rev())),
                 b'p1.nbmissingfiles': len(p1missing),
                 b'p2.node': p2.hex(),
                 b'p2.nbrevs': len(repo.revs('only(%d, %d)', p2.rev(), b.rev())),
                 b'p2.nbmissingfiles': len(p2missing),
             }
             if dostats:
                 if p1missing:
                     alldata['nbrevs'].append(
                         (data['p1.nbrevs'], b.hex(), p1.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p1.nbmissingfiles'], b.hex(), p1.hex())
                     )
                 if p2missing:
                     alldata['nbrevs'].append(
                         (data['p2.nbrevs'], b.hex(), p2.hex())
                     )
                     alldata['nbmissingfiles'].append(
                         (data['p2.nbmissingfiles'], b.hex(), p2.hex())
                     )
             if dotiming:
                 begin = util.timer()
                 mergedata = copies.mergecopies(repo, p1, p2, b)
                 end = util.timer()
                 # not very stable timing since we did only one run
                 data['time'] = end - begin
                 # mergedata contains five dicts: "copy", "movewithdir",
                 # "diverge", "renamedelete" and "dirmove".
                 # The first 4 are about renamed file so lets count that.
                 renames = len(mergedata[0])
                 renames += len(mergedata[1])
                 renames += len(mergedata[2])
                 renames += len(mergedata[3])
                 data['nbrenamedfiles'] = renames
                 begin = util.timer()
                 p1renames = copies.pathcopies(b, p1)
                 end = util.timer()
                 data['p1.time'] = end - begin
                 begin = util.timer()
                 p2renames = copies.pathcopies(b, p2)
                 end = util.timer()
                 data['p2.time'] = end - begin
                 data['p1.renamedfiles'] = len(p1renames)
                 data['p2.renamedfiles'] = len(p2renames)
 
                 if dostats:
                     if p1missing:
                         alldata['parentnbrenames'].append(
                             (data['p1.renamedfiles'], b.hex(), p1.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p1.time'], b.hex(), p1.hex())
                         )
                     if p2missing:
                         alldata['parentnbrenames'].append(
                             (data['p2.renamedfiles'], b.hex(), p2.hex())
                         )
                         alldata['parenttime'].append(
                             (data['p2.time'], b.hex(), p2.hex())
                         )
                     if p1missing or p2missing:
                         alldata['totalnbrenames'].append(
                             (
                                 data['nbrenamedfiles'],
                                 b.hex(),
                                 p1.hex(),
                                 p2.hex(),
                             )
                         )
                         alldata['totaltime'].append(
                             (data['time'], b.hex(), p1.hex(), p2.hex())
                         )
             fm.startitem()
             fm.data(**data)
             # make node pretty for the human output
             out = data.copy()
             out['base'] = fm.hexfunc(b.node())
             out['p1.node'] = fm.hexfunc(p1.node())
             out['p2.node'] = fm.hexfunc(p2.node())
             fm.plain(output % out)
 
     fm.end()
     if dostats:
         # use a second formatter because the data are quite different, not sure
         # how it flies with the templater.
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(
                 ('parentnbrenames', 'rename from one parent to base')
             )
             entries.append(('totalnbrenames', 'total number of renames'))
             entries.append(('parenttime', 'time for one parent'))
             entries.append(('totaltime', 'time for both parents'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(
     b'perfhelper-pathcopies',
     formatteropts
     + [
         (b'r', b'revs', [], b'restrict search to these revisions'),
         (b'', b'timing', False, b'provides extra data (costly)'),
         (b'', b'stats', False, b'provides statistic about the measured data'),
     ],
 )
 def perfhelperpathcopies(ui, repo, revs=[], **opts):
     """find statistic about potential parameters for the `perftracecopies`
 
     This command find source-destination pair relevant for copytracing testing.
     It report value for some of the parameters that impact copy tracing time.
 
     If `--timing` is set, rename detection is run and the associated timing
     will be reported. The extra details comes at the cost of a slower command
     execution.
 
     Since the rename detection is only run once, other factors might easily
     affect the precision of the timing. However it should give a good
     approximation of which revision pairs are very costly.
     """
     opts = _byteskwargs(opts)
     fm = ui.formatter(b'perf', opts)
     dotiming = opts[b'timing']
     dostats = opts[b'stats']
 
     if dotiming:
         header = '%12s %12s %12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d "
             "%(nbrenamedfiles)12d %(time)18.5f\n"
         )
         header_names = (
             "source",
             "destination",
             "nb-revs",
             "nb-files",
             "nb-renames",
             "time",
         )
         fm.plain(header % header_names)
     else:
         header = '%12s %12s %12s %12s\n'
         output = (
             "%(source)12s %(destination)12s "
             "%(nbrevs)12d %(nbmissingfiles)12d\n"
         )
         fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
 
     if not revs:
         revs = ['all()']
     revs = scmutil.revrange(repo, revs)
 
     if dostats:
         alldata = {
             'nbrevs': [],
             'nbmissingfiles': [],
         }
         if dotiming:
             alldata['nbrenames'] = []
             alldata['time'] = []
 
     roi = repo.revs('merge() and %ld', revs)
     for r in roi:
         ctx = repo[r]
         p1 = ctx.p1().rev()
         p2 = ctx.p2().rev()
         bases = repo.changelog._commonancestorsheads(p1, p2)
         for p in (p1, p2):
             for b in bases:
                 base = repo[b]
                 parent = repo[p]
                 missing = copies._computeforwardmissing(base, parent)
                 if not missing:
                     continue
                 data = {
                     b'source': base.hex(),
                     b'destination': parent.hex(),
                     b'nbrevs': len(repo.revs('only(%d, %d)', p, b)),
                     b'nbmissingfiles': len(missing),
                 }
                 if dostats:
                     alldata['nbrevs'].append(
                         (data['nbrevs'], base.hex(), parent.hex(),)
                     )
                     alldata['nbmissingfiles'].append(
                         (data['nbmissingfiles'], base.hex(), parent.hex(),)
                     )
                 if dotiming:
                     begin = util.timer()
                     renames = copies.pathcopies(base, parent)
                     end = util.timer()
                     # not very stable timing since we did only one run
                     data['time'] = end - begin
                     data['nbrenamedfiles'] = len(renames)
                     if dostats:
                         alldata['time'].append(
                             (data['time'], base.hex(), parent.hex(),)
                         )
                         alldata['nbrenames'].append(
                             (data['nbrenamedfiles'], base.hex(), parent.hex(),)
                         )
                 fm.startitem()
                 fm.data(**data)
                 out = data.copy()
                 out['source'] = fm.hexfunc(base.node())
                 out['destination'] = fm.hexfunc(parent.node())
                 fm.plain(output % out)
 
     fm.end()
     if dostats:
         entries = [
             ('nbrevs', 'number of revision covered'),
             ('nbmissingfiles', 'number of missing files at head'),
         ]
         if dotiming:
             entries.append(('nbrenames', 'renamed files'))
             entries.append(('time', 'time'))
         _displaystats(ui, opts, entries, alldata)
 
 
 @command(b'perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 
 @command(b'perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
 
     def d():
         s.fncache._load()
 
     timer(d)
     fm.end()
 
 
 @command(b'perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     lock = repo.lock()
     s.fncache._load()
     tr = repo.transaction(b'perffncachewrite')
     tr.addbackup(b'fncache')
 
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
 
     timer(d)
     tr.close()
     lock.release()
     fm.end()
 
 
 @command(b'perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
 
     def d():
         for p in s.fncache.entries:
             s.encode(p)
 
     timer(d)
     fm.end()
 
 
 def _bdiffworker(q, blocks, xdiff, ready, done):
     while not done.is_set():
         pair = q.get()
         while pair is not None:
             if xdiff:
                 mdiff.bdiff.xdiffblocks(*pair)
             elif blocks:
                 mdiff.bdiff.blocks(*pair)
             else:
                 mdiff.textdiff(*pair)
             q.task_done()
             pair = q.get()
         q.task_done()  # for the None one
         with ready:
             ready.wait()
 
 
 def _manifestrevision(repo, mnode):
     ml = repo.manifestlog
 
     if util.safehasattr(ml, b'getstorage'):
         store = ml.getstorage(b'')
     else:
         store = ml._revlog
 
     return store.revision(mnode)
 
 
 @command(
     b'perfbdiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
         (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
         (b'', b'blocks', False, b'test computing diffs into blocks'),
         (b'', b'xdiff', False, b'use xdiff algorithm'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
     """benchmark a bdiff between revisions
 
     By default, benchmark a bdiff between its delta parent and itself.
 
     With ``--count``, benchmark bdiffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure bdiffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
 
     if opts[b'xdiff'] and not opts[b'blocks']:
         raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
 
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfbdiff', b'invalid arguments')
 
     blocks = opts[b'blocks']
     xdiff = opts[b'xdiff']
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     withthreads = threads > 0
     if not withthreads:
 
         def d():
             for pair in textpairs:
                 if xdiff:
                     mdiff.bdiff.xdiffblocks(*pair)
                 elif blocks:
                     mdiff.bdiff.blocks(*pair)
                 else:
                     mdiff.textdiff(*pair)
 
     else:
         q = queue()
         for i in _xrange(threads):
             q.put(None)
         ready = threading.Condition()
         done = threading.Event()
         for i in _xrange(threads):
             threading.Thread(
                 target=_bdiffworker, args=(q, blocks, xdiff, ready, done)
             ).start()
         q.join()
 
         def d():
             for pair in textpairs:
                 q.put(pair)
             for i in _xrange(threads):
                 q.put(None)
             with ready:
                 ready.notify_all()
             q.join()
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
     if withthreads:
         done.set()
         for i in _xrange(threads):
             q.put(None)
         with ready:
             ready.notify_all()
 
 
 @command(
     b'perfunidiff',
     revlogopts
     + formatteropts
     + [
         (
             b'',
             b'count',
             1,
             b'number of revisions to test (when using --startrev)',
         ),
         (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
     ],
     b'-c|-m|FILE REV',
 )
 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
     """benchmark a unified diff between revisions
 
     This doesn't include any copy tracing - it's just a unified diff
     of the texts.
 
     By default, benchmark a diff between its delta parent and itself.
 
     With ``--count``, benchmark diffs between delta parents and self for N
     revisions starting at the specified revision.
 
     With ``--alldata``, assume the requested revision is a changeset and
     measure diffs for all changes related to that changeset (manifest
     and filelogs).
     """
     opts = _byteskwargs(opts)
     if opts[b'alldata']:
         opts[b'changelog'] = True
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfunidiff', b'invalid arguments')
 
     textpairs = []
 
     r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
 
     startrev = r.rev(r.lookup(rev))
     for rev in range(startrev, min(startrev + count, len(r) - 1)):
         if opts[b'alldata']:
             # Load revisions associated with changeset.
             ctx = repo[rev]
             mtext = _manifestrevision(repo, ctx.manifestnode())
             for pctx in ctx.parents():
                 pman = _manifestrevision(repo, pctx.manifestnode())
                 textpairs.append((pman, mtext))
 
             # Load filelog revisions by iterating manifest delta.
             man = ctx.manifest()
             pman = ctx.p1().manifest()
             for filename, change in pman.diff(man).items():
                 fctx = repo.file(filename)
                 f1 = fctx.revision(change[0][0] or -1)
                 f2 = fctx.revision(change[1][0] or -1)
                 textpairs.append((f1, f2))
         else:
             dp = r.deltaparent(rev)
             textpairs.append((r.revision(dp), r.revision(rev)))
 
     def d():
         for left, right in textpairs:
             # The date strings don't matter, so we pass empty strings.
             headerlines, hunks = mdiff.unidiff(
                 left, b'', right, b'', b'left', b'right', binary=False
             )
             # consume iterators in roughly the way patch.py does
             b'\n'.join(headerlines)
             b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(b'perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
     }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
-        opts = dict((options[c], b'1') for c in diffopt)
+        opts = {options[c]: b'1' for c in diffopt}
 
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
 
         diffopt = diffopt.encode('ascii')
         title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
         timer(d, title=title)
     fm.end()
 
 
 @command(b'perfrevlogindex', revlogopts + formatteropts, b'-c|-m|FILE')
 def perfrevlogindex(ui, repo, file_=None, **opts):
     """Benchmark operations against a revlog index.
 
     This tests constructing a revlog instance, reading index data,
     parsing index data, and performing various operations related to
     index data.
     """
 
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
 
     opener = getattr(rl, 'opener')  # trick linter
     indexfile = rl.indexfile
     data = opener.read(indexfile)
 
     header = struct.unpack(b'>I', data[0:4])[0]
     version = header & 0xFFFF
     if version == 1:
         revlogio = revlog.revlogio()
         inline = header & (1 << 16)
     else:
         raise error.Abort(b'unsupported revlog version: %d' % version)
 
     rllen = len(rl)
 
     node0 = rl.node(0)
     node25 = rl.node(rllen // 4)
     node50 = rl.node(rllen // 2)
     node75 = rl.node(rllen // 4 * 3)
     node100 = rl.node(rllen - 1)
 
     allrevs = range(rllen)
     allrevsrev = list(reversed(allrevs))
     allnodes = [rl.node(rev) for rev in range(rllen)]
     allnodesrev = list(reversed(allnodes))
 
     def constructor():
         revlog.revlog(opener, indexfile)
 
     def read():
         with opener(indexfile) as fh:
             fh.read()
 
     def parseindex():
         revlogio.parseindex(data, inline)
 
     def getentry(revornode):
         index = revlogio.parseindex(data, inline)[0]
         index[revornode]
 
     def getentries(revs, count=1):
         index = revlogio.parseindex(data, inline)[0]
 
         for i in range(count):
             for rev in revs:
                 index[rev]
 
     def resolvenode(node):
         index = revlogio.parseindex(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(
                 revlogio.parseindex(data, inline)[0], 'nodemap', None
             )
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         try:
             rev(node)
         except error.RevlogError:
             pass
 
     def resolvenodes(nodes, count=1):
         index = revlogio.parseindex(data, inline)[0]
         rev = getattr(index, 'rev', None)
         if rev is None:
             nodemap = getattr(
                 revlogio.parseindex(data, inline)[0], 'nodemap', None
             )
             # This only works for the C code.
             if nodemap is None:
                 return
             rev = nodemap.__getitem__
 
         for i in range(count):
             for node in nodes:
                 try:
                     rev(node)
                 except error.RevlogError:
                     pass
 
     benches = [
         (constructor, b'revlog constructor'),
         (read, b'read'),
         (parseindex, b'create index object'),
         (lambda: getentry(0), b'retrieve index entry for rev 0'),
         (lambda: resolvenode(b'a' * 20), b'look up missing node'),
         (lambda: resolvenode(node0), b'look up node at rev 0'),
         (lambda: resolvenode(node25), b'look up node at 1/4 len'),
         (lambda: resolvenode(node50), b'look up node at 1/2 len'),
         (lambda: resolvenode(node75), b'look up node at 3/4 len'),
         (lambda: resolvenode(node100), b'look up node at tip'),
         # 2x variation is to measure caching impact.
         (lambda: resolvenodes(allnodes), b'look up all nodes (forward)'),
         (lambda: resolvenodes(allnodes, 2), b'look up all nodes 2x (forward)'),
         (lambda: resolvenodes(allnodesrev), b'look up all nodes (reverse)'),
         (
             lambda: resolvenodes(allnodesrev, 2),
             b'look up all nodes 2x (reverse)',
         ),
         (lambda: getentries(allrevs), b'retrieve all index entries (forward)'),
         (
             lambda: getentries(allrevs, 2),
             b'retrieve all index entries 2x (forward)',
         ),
         (
             lambda: getentries(allrevsrev),
             b'retrieve all index entries (reverse)',
         ),
         (
             lambda: getentries(allrevsrev, 2),
             b'retrieve all index entries 2x (reverse)',
         ),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perfrevlogrevisions',
     revlogopts
     + formatteropts
     + [
         (b'd', b'dist', 100, b'distance between the revisions'),
         (b's', b'startrev', 0, b'revision to start reading at'),
         (b'', b'reverse', False, b'read in reverse'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogrevisions(
     ui, repo, file_=None, startrev=0, reverse=False, **opts
 ):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
     rllen = getlen(ui)(rl)
 
     if startrev < 0:
         startrev = rllen + startrev
 
     def d():
         rl.clearcaches()
 
         beginrev = startrev
         endrev = rllen
         dist = opts[b'dist']
 
         if reverse:
             beginrev, endrev = endrev - 1, beginrev - 1
             dist = -1 * dist
 
         for x in _xrange(beginrev, endrev, dist):
             # Old revisions don't support passing int.
             n = rl.node(x)
             rl.revision(n)
 
     timer, fm = gettimer(ui, opts)
     timer(d)
     fm.end()
 
 
 @command(
     b'perfrevlogwrite',
     revlogopts
     + formatteropts
     + [
         (b's', b'startrev', 1000, b'revision to start writing at'),
         (b'', b'stoprev', -1, b'last revision to write'),
         (b'', b'count', 3, b'number of passes to perform'),
         (b'', b'details', False, b'print timing for every revisions tested'),
         (b'', b'source', b'full', b'the kind of data feed in the revlog'),
         (b'', b'lazydeltabase', True, b'try the provided delta first'),
         (b'', b'clear-caches', True, b'clear revlog cache between calls'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
     """Benchmark writing a series of revisions to a revlog.
 
     Possible source values are:
     * `full`: add from a full text (default).
     * `parent-1`: add from a delta to the first parent
     * `parent-2`: add from a delta to the second parent if it exists
                   (use a delta from the first parent otherwise)
     * `parent-smallest`: add from the smallest delta (either p1 or p2)
     * `storage`: add from the existing precomputed deltas
 
     Note: This performance command measures performance in a custom way. As a
     result some of the global configuration of the 'perf' command does not
     apply to it:
 
     * ``pre-run``: disabled
 
     * ``profile-benchmark``: disabled
 
     * ``run-limits``: disabled use --count instead
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
     rllen = getlen(ui)(rl)
     if startrev < 0:
         startrev = rllen + startrev
     if stoprev < 0:
         stoprev = rllen + stoprev
 
     lazydeltabase = opts['lazydeltabase']
     source = opts['source']
     clearcaches = opts['clear_caches']
     validsource = (
         b'full',
         b'parent-1',
         b'parent-2',
         b'parent-smallest',
         b'storage',
     )
     if source not in validsource:
         raise error.Abort('invalid source type: %s' % source)
 
     ### actually gather results
     count = opts['count']
     if count <= 0:
         raise error.Abort('invalide run count: %d' % count)
     allresults = []
     for c in range(count):
         timing = _timeonewrite(
             ui,
             rl,
             source,
             startrev,
             stoprev,
             c + 1,
             lazydeltabase=lazydeltabase,
             clearcaches=clearcaches,
         )
         allresults.append(timing)
 
     ### consolidate the results in a single list
     results = []
     for idx, (rev, t) in enumerate(allresults[0]):
         ts = [t]
         for other in allresults[1:]:
             orev, ot = other[idx]
             assert orev == rev
             ts.append(ot)
         results.append((rev, ts))
     resultcount = len(results)
 
     ### Compute and display relevant statistics
 
     # get a formatter
     fm = ui.formatter(b'perf', opts)
     displayall = ui.configbool(b"perf", b"all-timing", False)
 
     # print individual details if requested
     if opts['details']:
         for idx, item in enumerate(results, 1):
             rev, data = item
             title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
             formatone(fm, data, title=title, displayall=displayall)
 
     # sorts results by median time
     results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
     # list of (name, index) to display)
     relevants = [
         ("min", 0),
         ("10%", resultcount * 10 // 100),
         ("25%", resultcount * 25 // 100),
         ("50%", resultcount * 70 // 100),
         ("75%", resultcount * 75 // 100),
         ("90%", resultcount * 90 // 100),
         ("95%", resultcount * 95 // 100),
         ("99%", resultcount * 99 // 100),
         ("99.9%", resultcount * 999 // 1000),
         ("99.99%", resultcount * 9999 // 10000),
         ("99.999%", resultcount * 99999 // 100000),
         ("max", -1),
     ]
     if not ui.quiet:
         for name, idx in relevants:
             data = results[idx]
             title = '%s of %d, rev %d' % (name, resultcount, data[0])
             formatone(fm, data[1], title=title, displayall=displayall)
 
     # XXX summing that many float will not be very precise, we ignore this fact
     # for now
     totaltime = []
     for item in allresults:
         totaltime.append(
             (
                 sum(x[1][0] for x in item),
                 sum(x[1][1] for x in item),
                 sum(x[1][2] for x in item),
             )
         )
     formatone(
         fm,
         totaltime,
         title="total time (%d revs)" % resultcount,
         displayall=displayall,
     )
     fm.end()
 
 
 class _faketr(object):
     def add(s, x, y, z=None):
         return None
 
 
 def _timeonewrite(
     ui,
     orig,
     source,
     startrev,
     stoprev,
     runidx=None,
     lazydeltabase=True,
     clearcaches=True,
 ):
     timings = []
     tr = _faketr()
     with _temprevlog(ui, orig, startrev) as dest:
         dest._lazydeltabase = lazydeltabase
         revs = list(orig.revs(startrev, stoprev))
         total = len(revs)
         topic = 'adding'
         if runidx is not None:
             topic += ' (run #%d)' % runidx
         # Support both old and new progress API
         if util.safehasattr(ui, 'makeprogress'):
             progress = ui.makeprogress(topic, unit='revs', total=total)
 
             def updateprogress(pos):
                 progress.update(pos)
 
             def completeprogress():
                 progress.complete()
 
         else:
 
             def updateprogress(pos):
                 ui.progress(topic, pos, unit='revs', total=total)
 
             def completeprogress():
                 ui.progress(topic, None, unit='revs', total=total)
 
         for idx, rev in enumerate(revs):
             updateprogress(idx)
             addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
             if clearcaches:
                 dest.index.clearcaches()
                 dest.clearcaches()
             with timeone() as r:
                 dest.addrawrevision(*addargs, **addkwargs)
             timings.append((rev, r[0]))
         updateprogress(total)
         completeprogress()
     return timings
 
 
 def _getrevisionseed(orig, rev, tr, source):
     from mercurial.node import nullid
 
     linkrev = orig.linkrev(rev)
     node = orig.node(rev)
     p1, p2 = orig.parents(node)
     flags = orig.flags(rev)
     cachedelta = None
     text = None
 
     if source == b'full':
         text = orig.revision(rev)
     elif source == b'parent-1':
         baserev = orig.rev(p1)
         cachedelta = (baserev, orig.revdiff(p1, rev))
     elif source == b'parent-2':
         parent = p2
         if p2 == nullid:
             parent = p1
         baserev = orig.rev(parent)
         cachedelta = (baserev, orig.revdiff(parent, rev))
     elif source == b'parent-smallest':
         p1diff = orig.revdiff(p1, rev)
         parent = p1
         diff = p1diff
         if p2 != nullid:
             p2diff = orig.revdiff(p2, rev)
             if len(p1diff) > len(p2diff):
                 parent = p2
                 diff = p2diff
         baserev = orig.rev(parent)
         cachedelta = (baserev, diff)
     elif source == b'storage':
         baserev = orig.deltaparent(rev)
         cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
 
     return (
         (text, tr, linkrev, p1, p2),
         {'node': node, 'flags': flags, 'cachedelta': cachedelta},
     )
 
 
 @contextlib.contextmanager
 def _temprevlog(ui, orig, truncaterev):
     from mercurial import vfs as vfsmod
 
     if orig._inline:
         raise error.Abort('not supporting inline revlog (yet)')
     revlogkwargs = {}
     k = 'upperboundcomp'
     if util.safehasattr(orig, k):
         revlogkwargs[k] = getattr(orig, k)
 
     origindexpath = orig.opener.join(orig.indexfile)
     origdatapath = orig.opener.join(orig.datafile)
     indexname = 'revlog.i'
     dataname = 'revlog.d'
 
     tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
     try:
         # copy the data file in a temporary directory
         ui.debug('copying data in %s\n' % tmpdir)
         destindexpath = os.path.join(tmpdir, 'revlog.i')
         destdatapath = os.path.join(tmpdir, 'revlog.d')
         shutil.copyfile(origindexpath, destindexpath)
         shutil.copyfile(origdatapath, destdatapath)
 
         # remove the data we want to add again
         ui.debug('truncating data to be rewritten\n')
         with open(destindexpath, 'ab') as index:
             index.seek(0)
             index.truncate(truncaterev * orig._io.size)
         with open(destdatapath, 'ab') as data:
             data.seek(0)
             data.truncate(orig.start(truncaterev))
 
         # instantiate a new revlog from the temporary copy
         ui.debug('truncating adding to be rewritten\n')
         vfs = vfsmod.vfs(tmpdir)
         vfs.options = getattr(orig.opener, 'options', None)
 
         dest = revlog.revlog(
             vfs, indexfile=indexname, datafile=dataname, **revlogkwargs
         )
         if dest._inline:
             raise error.Abort('not supporting inline revlog (yet)')
         # make sure internals are initialized
         dest.revision(len(dest) - 1)
         yield dest
         del dest, vfs
     finally:
         shutil.rmtree(tmpdir, True)
 
 
 @command(
     b'perfrevlogchunks',
     revlogopts
     + formatteropts
     + [
         (b'e', b'engines', b'', b'compression engines to use'),
         (b's', b'startrev', 0, b'revision to start at'),
     ],
     b'-c|-m|FILE',
 )
 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
     """Benchmark operations on revlog chunks.
 
     Logically, each revlog is a collection of fulltext revisions. However,
     stored within each revlog are "chunks" of possibly compressed data. This
     data needs to be read and decompressed or compressed and written.
 
     This command measures the time it takes to read+decompress and recompress
     chunks in a revlog. It effectively isolates I/O and compression performance.
     For measurements of higher-level operations like resolving revisions,
     see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
     """
     opts = _byteskwargs(opts)
 
     rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = rl._getsegmentforrevs
     except AttributeError:
         segmentforrevs = rl._chunkraw
 
     # Verify engines argument.
     if engines:
-        engines = set(e.strip() for e in engines.split(b','))
+        engines = {e.strip() for e in engines.split(b',')}
         for engine in engines:
             try:
                 util.compressionengines[engine]
             except KeyError:
                 raise error.Abort(b'unknown compression engine: %s' % engine)
     else:
         engines = []
         for e in util.compengines:
             engine = util.compengines[e]
             try:
                 if engine.available():
                     engine.revlogcompressor().compress(b'dummy')
                     engines.append(e)
             except NotImplementedError:
                 pass
 
     revs = list(rl.revs(startrev, len(rl) - 1))
 
     def rlfh(rl):
         if rl._inline:
             return getsvfs(repo)(rl.indexfile)
         else:
             return getsvfs(repo)(rl.datafile)
 
     def doread():
         rl.clearcaches()
         for rev in revs:
             segmentforrevs(rev, rev)
 
     def doreadcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             segmentforrevs(rev, rev, df=fh)
 
     def doreadbatch():
         rl.clearcaches()
         segmentforrevs(revs[0], revs[-1])
 
     def doreadbatchcachedfh():
         rl.clearcaches()
         fh = rlfh(rl)
         segmentforrevs(revs[0], revs[-1], df=fh)
 
     def dochunk():
         rl.clearcaches()
         fh = rlfh(rl)
         for rev in revs:
             rl._chunk(rev, df=fh)
 
     chunks = [None]
 
     def dochunkbatch():
         rl.clearcaches()
         fh = rlfh(rl)
         # Save chunks as a side-effect.
         chunks[0] = rl._chunks(revs, df=fh)
 
     def docompress(compressor):
         rl.clearcaches()
 
         try:
             # Swap in the requested compression engine.
             oldcompressor = rl._compressor
             rl._compressor = compressor
             for chunk in chunks[0]:
                 rl.compress(chunk)
         finally:
             rl._compressor = oldcompressor
 
     benches = [
         (lambda: doread(), b'read'),
         (lambda: doreadcachedfh(), b'read w/ reused fd'),
         (lambda: doreadbatch(), b'read batch'),
         (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
         (lambda: dochunk(), b'chunk'),
         (lambda: dochunkbatch(), b'chunk batch'),
     ]
 
     for engine in sorted(engines):
         compressor = util.compengines[engine].revlogcompressor()
         benches.append(
             (
                 functools.partial(docompress, compressor),
                 b'compress w/ %s' % engine,
             )
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(
     b'perfrevlogrevision',
     revlogopts
     + formatteropts
     + [(b'', b'cache', False, b'use caches instead of clearing')],
     b'-c|-m|FILE REV',
 )
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
     2. Slice the delta chain if applicable
     3. Obtain the raw chunks for that delta chain
     4. Decompress each raw chunk
     5. Apply binary patches to obtain fulltext
     6. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     opts = _byteskwargs(opts)
 
     if opts.get(b'changelog') or opts.get(b'manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
 
     r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
 
     # _chunkraw was renamed to _getsegmentforrevs.
     try:
         segmentforrevs = r._getsegmentforrevs
     except AttributeError:
         segmentforrevs = r._chunkraw
 
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def getrawchunks(data, chain):
         start = r.start
         length = r.length
         inline = r._inline
         iosize = r._io.size
         buffer = util.buffer
 
         chunks = []
         ladd = chunks.append
         for idx, item in enumerate(chain):
             offset = start(item[0])
             bits = data[idx]
             for rev in item:
                 chunkstart = start(rev)
                 if inline:
                     chunkstart += (rev + 1) * iosize
                 chunklength = length(rev)
                 ladd(buffer(bits, chunkstart - offset, chunklength))
 
         return chunks
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
         for item in slicedchain:
             segmentforrevs(item[0], item[-1])
 
     def doslice(r, chain, size):
         for s in slicechunk(r, chain, targetsize=size):
             pass
 
     def dorawchunks(data, chain):
         if not cache:
             r.clearcaches()
         getrawchunks(data, chain)
 
     def dodecompress(chunks):
         decomp = r.decompress
         for chunk in chunks:
             decomp(chunk)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r.checkhash(text, node, rev=rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
     try:
         from mercurial.revlogutils.deltas import slicechunk
     except ImportError:
         slicechunk = getattr(revlog, '_slicechunk', None)
 
     size = r.length(rev)
     chain = r._deltachain(rev)[0]
     if not getattr(r, '_withsparseread', False):
         slicedchain = (chain,)
     else:
         slicedchain = tuple(slicechunk(r, chain, targetsize=size))
     data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
     rawchunks = getrawchunks(data, slicedchain)
     bins = r._chunks(chain)
     text = bytes(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), b'full'),
         (lambda: dodeltachain(rev), b'deltachain'),
         (lambda: doread(chain), b'read'),
     ]
 
     if getattr(r, '_withsparseread', False):
         slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
         benches.append(slicing)
 
     benches.extend(
         [
             (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
             (lambda: dodecompress(rawchunks), b'decompress'),
             (lambda: dopatch(text, bins), b'patch'),
             (lambda: dohash(text), b'hash'),
         ]
     )
 
     timer, fm = gettimer(ui, opts)
     for fn, title in benches:
         timer(fn, title=title)
     fm.end()
 
 
 @command(
     b'perfrevset',
     [
         (b'C', b'clear', False, b'clear volatile cache between each call.'),
         (b'', b'contexts', False, b'obtain changectx for each revision'),
     ]
     + formatteropts,
     b"REVSET",
 )
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr):
                 pass
         else:
             for r in repo.revs(expr):
                 pass
 
     timer(d)
     fm.end()
 
 
 @command(
     b'perfvolatilesets',
     [(b'', b'clear-obsstore', False, b'drop obsstore between each call.'),]
     + formatteropts,
 )
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             obsolete.getrevs(repo, name)
 
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             if opts[b'clear_obsstore']:
                 clearfilecache(repo, b'obsstore')
             repoview.filterrevs(repo, name)
 
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 
 @command(
     b'perfbranchmap',
     [
         (b'f', b'full', False, b'Includes build time of subset'),
         (
             b'',
             b'clear-revbranch',
             False,
             b'purge the revbranch cache between computation',
         ),
     ]
     + formatteropts,
 )
 def perfbranchmap(ui, repo, *filternames, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     opts = _byteskwargs(opts)
     full = opts.get(b"full", False)
     clear_revbranch = opts.get(b"clear_revbranch", False)
     timer, fm = gettimer(ui, opts)
 
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         if util.safehasattr(view._branchcaches, '_per_filter'):
             filtered = view._branchcaches._per_filter
         else:
             # older versions
             filtered = view._branchcaches
 
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
                 filtered.pop(filtername, None)
             view.branchmap()
 
         return d
 
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     if filternames:
         possiblefilters &= set(filternames)
     subsettable = getbranchmapsubsettable()
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, b'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     if not filternames or b'unfiltered' in filternames:
         # add unfiltered
         allfilters.append(None)
 
     if util.safehasattr(branchmap.branchcache, 'fromfile'):
         branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
         branchcacheread.set(classmethod(lambda *args: None))
     else:
         # older versions
         branchcacheread = safeattrsetter(branchmap, b'read')
         branchcacheread.set(lambda *args: None)
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
     branchcachewrite.set(lambda *args: None)
     try:
         for name in allfilters:
             printname = name
             if name is None:
                 printname = b'unfiltered'
             timer(getbranchmap(name), title=str(printname))
     finally:
         branchcacheread.restore()
         branchcachewrite.restore()
     fm.end()
 
 
 @command(
     b'perfbranchmapupdate',
     [
         (b'', b'base', [], b'subset of revision to start from'),
         (b'', b'target', [], b'subset of revision to end with'),
         (b'', b'clear-caches', False, b'clear cache between each runs'),
     ]
     + formatteropts,
 )
 def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
     """benchmark branchmap update from for <base> revs to <target> revs
 
     If `--clear-caches` is passed, the following items will be reset before
     each update:
         * the changelog instance and associated indexes
         * the rev-branch-cache instance
 
     Examples:
 
        # update for the one last revision
        $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
 
        $ update for change coming with a new branch
        $ hg perfbranchmapupdate --base 'stable' --target 'default'
     """
     from mercurial import branchmap
     from mercurial import repoview
 
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     clearcaches = opts[b'clear_caches']
     unfi = repo.unfiltered()
     x = [None]  # used to pass data between closure
 
     # we use a `list` here to avoid possible side effect from smartset
     baserevs = list(scmutil.revrange(repo, base))
     targetrevs = list(scmutil.revrange(repo, target))
     if not baserevs:
         raise error.Abort(b'no revisions selected for --base')
     if not targetrevs:
         raise error.Abort(b'no revisions selected for --target')
 
     # make sure the target branchmap also contains the one in the base
     targetrevs = list(set(baserevs) | set(targetrevs))
     targetrevs.sort()
 
     cl = repo.changelog
     allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
     allbaserevs.sort()
     alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
 
     newrevs = list(alltargetrevs.difference(allbaserevs))
     newrevs.sort()
 
     allrevs = frozenset(unfi.changelog.revs())
     basefilterrevs = frozenset(allrevs.difference(allbaserevs))
     targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
 
     def basefilter(repo, visibilityexceptions=None):
         return basefilterrevs
 
     def targetfilter(repo, visibilityexceptions=None):
         return targetfilterrevs
 
     msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
     ui.status(msg % (len(allbaserevs), len(newrevs)))
     if targetfilterrevs:
         msg = b'(%d revisions still filtered)\n'
         ui.status(msg % len(targetfilterrevs))
 
     try:
         repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
         repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
 
         baserepo = repo.filtered(b'__perf_branchmap_update_base')
         targetrepo = repo.filtered(b'__perf_branchmap_update_target')
 
         # try to find an existing branchmap to reuse
         subsettable = getbranchmapsubsettable()
         candidatefilter = subsettable.get(None)
         while candidatefilter is not None:
             candidatebm = repo.filtered(candidatefilter).branchmap()
             if candidatebm.validfor(baserepo):
                 filtered = repoview.filterrevs(repo, candidatefilter)
                 missing = [r for r in allbaserevs if r in filtered]
                 base = candidatebm.copy()
                 base.update(baserepo, missing)
                 break
             candidatefilter = subsettable.get(candidatefilter)
         else:
             # no suitable subset where found
             base = branchmap.branchcache()
             base.update(baserepo, allbaserevs)
 
         def setup():
             x[0] = base.copy()
             if clearcaches:
                 unfi._revbranchcache = None
                 clearchangelog(repo)
 
         def bench():
             x[0].update(targetrepo, newrevs)
 
         timer(bench, setup=setup)
         fm.end()
     finally:
         repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
         repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
 
 
 @command(
     b'perfbranchmapload',
     [
         (b'f', b'filter', b'', b'Specify repoview filter'),
         (b'', b'list', False, b'List brachmap filter caches'),
         (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
     ]
     + formatteropts,
 )
 def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
     """benchmark reading the branchmap"""
     opts = _byteskwargs(opts)
     clearrevlogs = opts[b'clear_revlogs']
 
     if list:
         for name, kind, st in repo.cachevfs.readdir(stat=True):
             if name.startswith(b'branch2'):
                 filtername = name.partition(b'-')[2] or b'unfiltered'
                 ui.status(
                     b'%s - %s\n' % (filtername, util.bytecount(st.st_size))
                 )
         return
     if not filter:
         filter = None
     subsettable = getbranchmapsubsettable()
     if filter is None:
         repo = repo.unfiltered()
     else:
         repo = repoview.repoview(repo, filter)
 
     repo.branchmap()  # make sure we have a relevant, up to date branchmap
 
     try:
         fromfile = branchmap.branchcache.fromfile
     except AttributeError:
         # older versions
         fromfile = branchmap.read
 
     currentfilter = filter
     # try once without timer, the filter may not be cached
     while fromfile(repo) is None:
         currentfilter = subsettable.get(currentfilter)
         if currentfilter is None:
             raise error.Abort(
                 b'No branchmap cached for %s repo' % (filter or b'unfiltered')
             )
         repo = repo.filtered(currentfilter)
     timer, fm = gettimer(ui, opts)
 
     def setup():
         if clearrevlogs:
             clearchangelog(repo)
 
     def bench():
         fromfile(repo)
 
     timer(bench, setup=setup)
     fm.end()
 
 
 @command(b'perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     svfs = getsvfs(repo)
     timer(lambda: len(obsolete.obsstore(svfs)))
     fm.end()
 
 
 @command(
     b'perflrucachedict',
     formatteropts
     + [
         (b'', b'costlimit', 0, b'maximum total cost of items in cache'),
         (b'', b'mincost', 0, b'smallest cost of items in cache'),
         (b'', b'maxcost', 100, b'maximum cost of items in cache'),
         (b'', b'size', 4, b'size of cache'),
         (b'', b'gets', 10000, b'number of key lookups'),
         (b'', b'sets', 10000, b'number of key sets'),
         (b'', b'mixed', 10000, b'number of mixed mode operations'),
         (
             b'',
             b'mixedgetfreq',
             50,
             b'frequency of get vs set ops in mixed mode',
         ),
     ],
     norepo=True,
 )
 def perflrucache(
     ui,
     mincost=0,
     maxcost=100,
     costlimit=0,
     size=4,
     gets=10000,
     sets=10000,
     mixed=10000,
     mixedgetfreq=50,
     **opts
 ):
     opts = _byteskwargs(opts)
 
     def doinit():
         for i in _xrange(10000):
             util.lrucachedict(size)
 
     costrange = list(range(mincost, maxcost + 1))
 
     values = []
     for i in _xrange(size):
         values.append(random.randint(0, _maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in _xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value  # silence pyflakes warning
 
     def dogetscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(values):
             d.insert(v, v, cost=costs[i])
         for key in getseq:
             try:
                 value = d[key]
                 value  # silence pyflakes warning
             except KeyError:
                 pass
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     costs = []
     for i in _xrange(sets):
         setseq.append(random.randint(0, _maxint))
         costs.append(random.choice(costrange))
 
     def doinserts():
         d = util.lrucachedict(size)
         for v in setseq:
             d.insert(v, v)
 
     def doinsertscost():
         d = util.lrucachedict(size, maxcost=costlimit)
         for i, v in enumerate(setseq):
             d.insert(v, v, cost=costs[i])
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in _xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append(
             (op, random.randint(0, size * 2), random.choice(costrange))
         )
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     def domixedcost():
         d = util.lrucachedict(size, maxcost=costlimit)
 
         for op, v, cost in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d.insert(v, v, cost=cost)
 
     benches = [
         (doinit, b'init'),
     ]
 
     if costlimit:
         benches.extend(
             [
                 (dogetscost, b'gets w/ cost limit'),
                 (doinsertscost, b'inserts w/ cost limit'),
                 (domixedcost, b'mixed w/ cost limit'),
             ]
         )
     else:
         benches.extend(
             [
                 (dogets, b'gets'),
                 (doinserts, b'inserts'),
                 (dosets, b'sets'),
                 (domixed, b'mixed'),
             ]
         )
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 
 @command(b'perfwrite', formatteropts)
 def perfwrite(ui, repo, **opts):
     """microbenchmark ui.write
     """
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def write():
         for i in range(100000):
             ui.writenoi18n(b'Testing write performance\n')
 
     timer(write)
     fm.end()
 
 
 def uisetup(ui):
     if util.safehasattr(cmdutil, b'openrevlog') and not util.safehasattr(
         commands, b'debugrevlogopts'
     ):
         # for "historical portability":
         # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
         # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
         # openrevlog() should cause failure, because it has been
         # available since 3.5 (or 49c583ca48c4).
         def openrevlog(orig, repo, cmd, file_, opts):
             if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
                 raise error.Abort(
                     b"This version doesn't support --dir option",
                     hint=b"use 3.5 or later",
                 )
             return orig(repo, cmd, file_, opts)
 
         extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
 
 
 @command(
     b'perfprogress',
     formatteropts
     + [
         (b'', b'topic', b'topic', b'topic for progress messages'),
         (b'c', b'total', 1000000, b'total value we are progressing to'),
     ],
     norepo=True,
 )
 def perfprogress(ui, topic=None, total=None, **opts):
     """printing of progress bars"""
     opts = _byteskwargs(opts)
 
     timer, fm = gettimer(ui, opts)
 
     def doprogress():
         with ui.makeprogress(topic, total=total) as progress:
             for i in _xrange(total):
                 progress.increment()
 
     timer(doprogress)
     fm.end()

contrib/python-zstandard/make_cffi.py

@@ -1,225 +1,228 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 from __future__ import absolute_import
 
 import cffi
 import distutils.ccompiler
 import os
 import re
 import subprocess
 import tempfile
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 SOURCES = [
     "zstd/%s" % p
     for p in (
         "common/debug.c",
         "common/entropy_common.c",
         "common/error_private.c",
         "common/fse_decompress.c",
         "common/pool.c",
         "common/threading.c",
         "common/xxhash.c",
         "common/zstd_common.c",
         "compress/fse_compress.c",
         "compress/hist.c",
         "compress/huf_compress.c",
         "compress/zstd_compress.c",
         "compress/zstd_compress_literals.c",
         "compress/zstd_compress_sequences.c",
         "compress/zstd_double_fast.c",
         "compress/zstd_fast.c",
         "compress/zstd_lazy.c",
         "compress/zstd_ldm.c",
         "compress/zstd_opt.c",
         "compress/zstdmt_compress.c",
         "decompress/huf_decompress.c",
         "decompress/zstd_ddict.c",
         "decompress/zstd_decompress.c",
         "decompress/zstd_decompress_block.c",
         "dictBuilder/cover.c",
         "dictBuilder/fastcover.c",
         "dictBuilder/divsufsort.c",
         "dictBuilder/zdict.c",
     )
 ]
 
 # Headers whose preprocessed output will be fed into cdef().
 HEADERS = [
-    os.path.join(HERE, "zstd", *p) for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
+    os.path.join(HERE, "zstd", *p)
+    for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
 ]
 
 INCLUDE_DIRS = [
     os.path.join(HERE, d)
     for d in (
         "zstd",
         "zstd/common",
         "zstd/compress",
         "zstd/decompress",
         "zstd/dictBuilder",
     )
 ]
 
 # cffi can't parse some of the primitives in zstd.h. So we invoke the
 # preprocessor and feed its output into cffi.
 compiler = distutils.ccompiler.new_compiler()
 
 # Needed for MSVC.
 if hasattr(compiler, "initialize"):
     compiler.initialize()
 
 # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor
 # manually.
 if compiler.compiler_type == "unix":
     args = list(compiler.executables["compiler"])
     args.extend(
         ["-E", "-DZSTD_STATIC_LINKING_ONLY", "-DZDICT_STATIC_LINKING_ONLY",]
     )
 elif compiler.compiler_type == "msvc":
     args = [compiler.cc]
     args.extend(
         ["/EP", "/DZSTD_STATIC_LINKING_ONLY", "/DZDICT_STATIC_LINKING_ONLY",]
     )
 else:
     raise Exception("unsupported compiler type: %s" % compiler.compiler_type)
 
 
 def preprocess(path):
     with open(path, "rb") as fh:
         lines = []
         it = iter(fh)
 
         for l in it:
             # zstd.h includes <stddef.h>, which is also included by cffi's
             # boilerplate. This can lead to duplicate declarations. So we strip
             # this include from the preprocessor invocation.
             #
             # The same things happens for including zstd.h, so give it the same
             # treatment.
             #
             # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
             # #define in zstdmt_compress.h and results in a compiler warning. So drop
             # the inline #define.
             if l.startswith(
                 (
                     b"#include <stddef.h>",
                     b'#include "zstd.h"',
                     b"#define ZSTD_STATIC_LINKING_ONLY",
                 )
             ):
                 continue
 
             # The preprocessor environment on Windows doesn't define include
             # paths, so the #include of limits.h fails. We work around this
             # by removing that import and defining INT_MAX ourselves. This is
             # a bit hacky. But it gets the job done.
             # TODO make limits.h work on Windows so we ensure INT_MAX is
             # correct.
             if l.startswith(b"#include <limits.h>"):
                 l = b"#define INT_MAX 2147483647\n"
 
             # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
             # important so just filter it out.
             if l.startswith(b"ZSTDLIB_API"):
                 l = l[len(b"ZSTDLIB_API ") :]
 
             lines.append(l)
 
     fd, input_file = tempfile.mkstemp(suffix=".h")
     os.write(fd, b"".join(lines))
     os.close(fd)
 
     try:
         env = dict(os.environ)
         if getattr(compiler, "_paths", None):
             env["PATH"] = compiler._paths
-        process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE, env=env)
+        process = subprocess.Popen(
+            args + [input_file], stdout=subprocess.PIPE, env=env
+        )
         output = process.communicate()[0]
         ret = process.poll()
         if ret:
             raise Exception("preprocessor exited with error")
 
         return output
     finally:
         os.unlink(input_file)
 
 
 def normalize_output(output):
     lines = []
     for line in output.splitlines():
         # CFFI's parser doesn't like __attribute__ on UNIX compilers.
         if line.startswith(b'__attribute__ ((visibility ("default"))) '):
             line = line[len(b'__attribute__ ((visibility ("default"))) ') :]
 
         if line.startswith(b"__attribute__((deprecated("):
             continue
         elif b"__declspec(deprecated(" in line:
             continue
 
         lines.append(line)
 
     return b"\n".join(lines)
 
 
 ffi = cffi.FFI()
 # zstd.h uses a possible undefined MIN(). Define it until
 # https://github.com/facebook/zstd/issues/976 is fixed.
 # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
 # when cffi uses the function. Since we statically link against zstd, even
 # if we use the deprecated functions it shouldn't be a huge problem.
 ffi.set_source(
     "_zstd_cffi",
     """
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>
 #define ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
 #include <zdict.h>
 """,
     sources=SOURCES,
     include_dirs=INCLUDE_DIRS,
     extra_compile_args=["-DZSTD_MULTITHREAD"],
 )
 
 DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
 
 sources = []
 
 # Feed normalized preprocessor output for headers into the cdef parser.
 for header in HEADERS:
     preprocessed = preprocess(header)
     sources.append(normalize_output(preprocessed))
 
     # #define's are effectively erased as part of going through preprocessor.
     # So perform a manual pass to re-add those to the cdef source.
     with open(header, "rb") as fh:
         for line in fh:
             line = line.strip()
             m = DEFINE.match(line)
             if not m:
                 continue
 
             if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
                 continue
 
             # The parser doesn't like some constants with complex values.
             if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
                 continue
 
             # The ... is magic syntax by the cdef parser to resolve the
             # value at compile time.
             sources.append(m.group(0) + b" ...")
 
 cdeflines = b"\n".join(sources).splitlines()
 cdeflines = [l for l in cdeflines if l.strip()]
 ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
 
 if __name__ == "__main__":
     ffi.compile()

contrib/python-zstandard/setup.py

@@ -1,118 +1,120 b''
 #!/usr/bin/env python
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 from __future__ import print_function
 
 from distutils.version import LooseVersion
 import os
 import sys
 from setuptools import setup
 
 # Need change in 1.10 for ffi.from_buffer() to handle all buffer types
 # (like memoryview).
 # Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
 # garbage collection pitfalls.
 MINIMUM_CFFI_VERSION = "1.11"
 
 try:
     import cffi
 
     # PyPy (and possibly other distros) have CFFI distributed as part of
     # them. The install_requires for CFFI below won't work. We need to sniff
     # out the CFFI version here and reject CFFI if it is too old.
     cffi_version = LooseVersion(cffi.__version__)
     if cffi_version < LooseVersion(MINIMUM_CFFI_VERSION):
         print(
             "CFFI 1.11 or newer required (%s found); "
             "not building CFFI backend" % cffi_version,
             file=sys.stderr,
         )
         cffi = None
 
 except ImportError:
     cffi = None
 
 import setup_zstd
 
 SUPPORT_LEGACY = False
 SYSTEM_ZSTD = False
 WARNINGS_AS_ERRORS = False
 
 if os.environ.get("ZSTD_WARNINGS_AS_ERRORS", ""):
     WARNINGS_AS_ERRORS = True
 
 if "--legacy" in sys.argv:
     SUPPORT_LEGACY = True
     sys.argv.remove("--legacy")
 
 if "--system-zstd" in sys.argv:
     SYSTEM_ZSTD = True
     sys.argv.remove("--system-zstd")
 
 if "--warnings-as-errors" in sys.argv:
     WARNINGS_AS_ERRORS = True
     sys.argv.remove("--warning-as-errors")
 
 # Code for obtaining the Extension instance is in its own module to
 # facilitate reuse in other projects.
 extensions = [
     setup_zstd.get_c_extension(
         name="zstd",
         support_legacy=SUPPORT_LEGACY,
         system_zstd=SYSTEM_ZSTD,
         warnings_as_errors=WARNINGS_AS_ERRORS,
     ),
 ]
 
 install_requires = []
 
 if cffi:
     import make_cffi
 
     extensions.append(make_cffi.ffi.distutils_extension())
     install_requires.append("cffi>=%s" % MINIMUM_CFFI_VERSION)
 
 version = None
 
 with open("c-ext/python-zstandard.h", "r") as fh:
     for line in fh:
         if not line.startswith("#define PYTHON_ZSTANDARD_VERSION"):
             continue
 
         version = line.split()[2][1:-1]
         break
 
 if not version:
-    raise Exception("could not resolve package version; " "this should never happen")
+    raise Exception(
+        "could not resolve package version; " "this should never happen"
+    )
 
 setup(
     name="zstandard",
     version=version,
     description="Zstandard bindings for Python",
     long_description=open("README.rst", "r").read(),
     url="https://github.com/indygreg/python-zstandard",
     author="Gregory Szorc",
     author_email="[e-mail]",
     license="BSD",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: C",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
     ],
     keywords="zstandard zstd compression",
     packages=["zstandard"],
     ext_modules=extensions,
     test_suite="tests",
     install_requires=install_requires,
     tests_require=["hypothesis"],
 )

contrib/python-zstandard/setup_zstd.py

@@ -1,206 +1,210 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 import distutils.ccompiler
 import os
 
 from distutils.extension import Extension
 
 
 zstd_sources = [
     "zstd/%s" % p
     for p in (
         "common/debug.c",
         "common/entropy_common.c",
         "common/error_private.c",
         "common/fse_decompress.c",
         "common/pool.c",
         "common/threading.c",
         "common/xxhash.c",
         "common/zstd_common.c",
         "compress/fse_compress.c",
         "compress/hist.c",
         "compress/huf_compress.c",
         "compress/zstd_compress_literals.c",
         "compress/zstd_compress_sequences.c",
         "compress/zstd_compress.c",
         "compress/zstd_double_fast.c",
         "compress/zstd_fast.c",
         "compress/zstd_lazy.c",
         "compress/zstd_ldm.c",
         "compress/zstd_opt.c",
         "compress/zstdmt_compress.c",
         "decompress/huf_decompress.c",
         "decompress/zstd_ddict.c",
         "decompress/zstd_decompress.c",
         "decompress/zstd_decompress_block.c",
         "dictBuilder/cover.c",
         "dictBuilder/divsufsort.c",
         "dictBuilder/fastcover.c",
         "dictBuilder/zdict.c",
     )
 ]
 
 zstd_sources_legacy = [
     "zstd/%s" % p
     for p in (
         "deprecated/zbuff_common.c",
         "deprecated/zbuff_compress.c",
         "deprecated/zbuff_decompress.c",
         "legacy/zstd_v01.c",
         "legacy/zstd_v02.c",
         "legacy/zstd_v03.c",
         "legacy/zstd_v04.c",
         "legacy/zstd_v05.c",
         "legacy/zstd_v06.c",
         "legacy/zstd_v07.c",
     )
 ]
 
 zstd_includes = [
     "zstd",
     "zstd/common",
     "zstd/compress",
     "zstd/decompress",
     "zstd/dictBuilder",
 ]
 
 zstd_includes_legacy = [
     "zstd/deprecated",
     "zstd/legacy",
 ]
 
 ext_includes = [
     "c-ext",
     "zstd/common",
 ]
 
 ext_sources = [
     "zstd/common/error_private.c",
     "zstd/common/pool.c",
     "zstd/common/threading.c",
     "zstd/common/zstd_common.c",
     "zstd.c",
     "c-ext/bufferutil.c",
     "c-ext/compressiondict.c",
     "c-ext/compressobj.c",
     "c-ext/compressor.c",
     "c-ext/compressoriterator.c",
     "c-ext/compressionchunker.c",
     "c-ext/compressionparams.c",
     "c-ext/compressionreader.c",
     "c-ext/compressionwriter.c",
     "c-ext/constants.c",
     "c-ext/decompressobj.c",
     "c-ext/decompressor.c",
     "c-ext/decompressoriterator.c",
     "c-ext/decompressionreader.c",
     "c-ext/decompressionwriter.c",
     "c-ext/frameparams.c",
 ]
 
 zstd_depends = [
     "c-ext/python-zstandard.h",
 ]
 
 
 def get_c_extension(
     support_legacy=False,
     system_zstd=False,
     name="zstd",
     warnings_as_errors=False,
     root=None,
 ):
     """Obtain a distutils.extension.Extension for the C extension.
 
     ``support_legacy`` controls whether to compile in legacy zstd format support.
 
     ``system_zstd`` controls whether to compile against the system zstd library.
     For this to work, the system zstd library and headers must match what
     python-zstandard is coded against exactly.
 
     ``name`` is the module name of the C extension to produce.
 
     ``warnings_as_errors`` controls whether compiler warnings are turned into
     compiler errors.
 
     ``root`` defines a root path that source should be computed as relative
     to. This should be the directory with the main ``setup.py`` that is
     being invoked. If not defined, paths will be relative to this file.
     """
     actual_root = os.path.abspath(os.path.dirname(__file__))
     root = root or actual_root
 
     sources = set([os.path.join(actual_root, p) for p in ext_sources])
     if not system_zstd:
         sources.update([os.path.join(actual_root, p) for p in zstd_sources])
         if support_legacy:
-            sources.update([os.path.join(actual_root, p) for p in zstd_sources_legacy])
+            sources.update(
+                [os.path.join(actual_root, p) for p in zstd_sources_legacy]
+            )
     sources = list(sources)
 
     include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
     if not system_zstd:
-        include_dirs.update([os.path.join(actual_root, d) for d in zstd_includes])
+        include_dirs.update(
+            [os.path.join(actual_root, d) for d in zstd_includes]
+        )
         if support_legacy:
             include_dirs.update(
                 [os.path.join(actual_root, d) for d in zstd_includes_legacy]
             )
     include_dirs = list(include_dirs)
 
     depends = [os.path.join(actual_root, p) for p in zstd_depends]
 
     compiler = distutils.ccompiler.new_compiler()
 
     # Needed for MSVC.
     if hasattr(compiler, "initialize"):
         compiler.initialize()
 
     if compiler.compiler_type == "unix":
         compiler_type = "unix"
     elif compiler.compiler_type == "msvc":
         compiler_type = "msvc"
     elif compiler.compiler_type == "mingw32":
         compiler_type = "mingw32"
     else:
         raise Exception("unhandled compiler type: %s" % compiler.compiler_type)
 
     extra_args = ["-DZSTD_MULTITHREAD"]
 
     if not system_zstd:
         extra_args.append("-DZSTDLIB_VISIBILITY=")
         extra_args.append("-DZDICTLIB_VISIBILITY=")
         extra_args.append("-DZSTDERRORLIB_VISIBILITY=")
 
         if compiler_type == "unix":
             extra_args.append("-fvisibility=hidden")
 
     if not system_zstd and support_legacy:
         extra_args.append("-DZSTD_LEGACY_SUPPORT=1")
 
     if warnings_as_errors:
         if compiler_type in ("unix", "mingw32"):
             extra_args.append("-Werror")
         elif compiler_type == "msvc":
             extra_args.append("/WX")
         else:
             assert False
 
     libraries = ["zstd"] if system_zstd else []
 
     # Python 3.7 doesn't like absolute paths. So normalize to relative.
     sources = [os.path.relpath(p, root) for p in sources]
     include_dirs = [os.path.relpath(p, root) for p in include_dirs]
     depends = [os.path.relpath(p, root) for p in depends]
 
     # TODO compile with optimizations.
     return Extension(
         name,
         sources,
         include_dirs=include_dirs,
         depends=depends,
         extra_compile_args=extra_args,
         libraries=libraries,
     )

contrib/python-zstandard/tests/common.py

@@ -1,197 +1,203 b''
 import imp
 import inspect
 import io
 import os
 import types
 import unittest
 
 try:
     import hypothesis
 except ImportError:
     hypothesis = None
 
 
 class TestCase(unittest.TestCase):
     if not getattr(unittest.TestCase, "assertRaisesRegex", False):
         assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
 
 
 def make_cffi(cls):
     """Decorator to add CFFI versions of each test method."""
 
     # The module containing this class definition should
     # `import zstandard as zstd`. Otherwise things may blow up.
     mod = inspect.getmodule(cls)
     if not hasattr(mod, "zstd"):
         raise Exception('test module does not contain "zstd" symbol')
 
     if not hasattr(mod.zstd, "backend"):
         raise Exception(
             'zstd symbol does not have "backend" attribute; did '
             "you `import zstandard as zstd`?"
         )
 
     # If `import zstandard` already chose the cffi backend, there is nothing
     # for us to do: we only add the cffi variation if the default backend
     # is the C extension.
     if mod.zstd.backend == "cffi":
         return cls
 
     old_env = dict(os.environ)
     os.environ["PYTHON_ZSTANDARD_IMPORT_POLICY"] = "cffi"
     try:
         try:
             mod_info = imp.find_module("zstandard")
             mod = imp.load_module("zstandard_cffi", *mod_info)
         except ImportError:
             return cls
     finally:
         os.environ.clear()
         os.environ.update(old_env)
 
     if mod.backend != "cffi":
-        raise Exception("got the zstandard %s backend instead of cffi" % mod.backend)
+        raise Exception(
+            "got the zstandard %s backend instead of cffi" % mod.backend
+        )
 
     # If CFFI version is available, dynamically construct test methods
     # that use it.
 
     for attr in dir(cls):
         fn = getattr(cls, attr)
         if not inspect.ismethod(fn) and not inspect.isfunction(fn):
             continue
 
         if not fn.__name__.startswith("test_"):
             continue
 
         name = "%s_cffi" % fn.__name__
 
         # Replace the "zstd" symbol with the CFFI module instance. Then copy
         # the function object and install it in a new attribute.
         if isinstance(fn, types.FunctionType):
             globs = dict(fn.__globals__)
             globs["zstd"] = mod
             new_fn = types.FunctionType(
                 fn.__code__, globs, name, fn.__defaults__, fn.__closure__
             )
             new_method = new_fn
         else:
             globs = dict(fn.__func__.func_globals)
             globs["zstd"] = mod
             new_fn = types.FunctionType(
                 fn.__func__.func_code,
                 globs,
                 name,
                 fn.__func__.func_defaults,
                 fn.__func__.func_closure,
             )
-            new_method = types.UnboundMethodType(new_fn, fn.im_self, fn.im_class)
+            new_method = types.UnboundMethodType(
+                new_fn, fn.im_self, fn.im_class
+            )
 
         setattr(cls, name, new_method)
 
     return cls
 
 
 class NonClosingBytesIO(io.BytesIO):
     """BytesIO that saves the underlying buffer on close().
 
     This allows us to access written data after close().
     """
 
     def __init__(self, *args, **kwargs):
         super(NonClosingBytesIO, self).__init__(*args, **kwargs)
         self._saved_buffer = None
 
     def close(self):
         self._saved_buffer = self.getvalue()
         return super(NonClosingBytesIO, self).close()
 
     def getvalue(self):
         if self.closed:
             return self._saved_buffer
         else:
             return super(NonClosingBytesIO, self).getvalue()
 
 
 class OpCountingBytesIO(NonClosingBytesIO):
     def __init__(self, *args, **kwargs):
         self._flush_count = 0
         self._read_count = 0
         self._write_count = 0
         return super(OpCountingBytesIO, self).__init__(*args, **kwargs)
 
     def flush(self):
         self._flush_count += 1
         return super(OpCountingBytesIO, self).flush()
 
     def read(self, *args):
         self._read_count += 1
         return super(OpCountingBytesIO, self).read(*args)
 
     def write(self, data):
         self._write_count += 1
         return super(OpCountingBytesIO, self).write(data)
 
 
 _source_files = []
 
 
 def random_input_data():
     """Obtain the raw content of source files.
 
     This is used for generating "random" data to feed into fuzzing, since it is
     faster than random content generation.
     """
     if _source_files:
         return _source_files
 
     for root, dirs, files in os.walk(os.path.dirname(__file__)):
         dirs[:] = list(sorted(dirs))
         for f in sorted(files):
             try:
                 with open(os.path.join(root, f), "rb") as fh:
                     data = fh.read()
                     if data:
                         _source_files.append(data)
             except OSError:
                 pass
 
     # Also add some actual random data.
     _source_files.append(os.urandom(100))
     _source_files.append(os.urandom(1000))
     _source_files.append(os.urandom(10000))
     _source_files.append(os.urandom(100000))
     _source_files.append(os.urandom(1000000))
 
     return _source_files
 
 
 def generate_samples():
     inputs = [
         b"foo",
         b"bar",
         b"abcdef",
         b"sometext",
         b"baz",
     ]
 
     samples = []
 
     for i in range(128):
         samples.append(inputs[i % 5])
         samples.append(inputs[i % 5] * (i + 3))
         samples.append(inputs[-(i % 5)] * (i + 2))
 
     return samples
 
 
 if hypothesis:
     default_settings = hypothesis.settings(deadline=10000)
     hypothesis.settings.register_profile("default", default_settings)
 
     ci_settings = hypothesis.settings(deadline=20000, max_examples=1000)
     hypothesis.settings.register_profile("ci", ci_settings)
 
     expensive_settings = hypothesis.settings(deadline=None, max_examples=10000)
     hypothesis.settings.register_profile("expensive", expensive_settings)
 
-    hypothesis.settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "default"))
+    hypothesis.settings.load_profile(
+        os.environ.get("HYPOTHESIS_PROFILE", "default")
+    )

contrib/python-zstandard/tests/test_buffer_util.py

@@ -1,146 +1,153 b''
 import struct
 import unittest
 
 import zstandard as zstd
 
 from .common import TestCase
 
 ss = struct.Struct("=QQ")
 
 
 class TestBufferWithSegments(TestCase):
     def test_arguments(self):
         if not hasattr(zstd, "BufferWithSegments"):
             self.skipTest("BufferWithSegments not available")
 
         with self.assertRaises(TypeError):
             zstd.BufferWithSegments()
 
         with self.assertRaises(TypeError):
             zstd.BufferWithSegments(b"foo")
 
         # Segments data should be a multiple of 16.
         with self.assertRaisesRegex(
             ValueError, "segments array size is not a multiple of 16"
         ):
             zstd.BufferWithSegments(b"foo", b"\x00\x00")
 
     def test_invalid_offset(self):
         if not hasattr(zstd, "BufferWithSegments"):
             self.skipTest("BufferWithSegments not available")
 
         with self.assertRaisesRegex(
             ValueError, "offset within segments array references memory"
         ):
             zstd.BufferWithSegments(b"foo", ss.pack(0, 4))
 
     def test_invalid_getitem(self):
         if not hasattr(zstd, "BufferWithSegments"):
             self.skipTest("BufferWithSegments not available")
 
         b = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
 
         with self.assertRaisesRegex(IndexError, "offset must be non-negative"):
             test = b[-10]
 
         with self.assertRaisesRegex(IndexError, "offset must be less than 1"):
             test = b[1]
 
         with self.assertRaisesRegex(IndexError, "offset must be less than 1"):
             test = b[2]
 
     def test_single(self):
         if not hasattr(zstd, "BufferWithSegments"):
             self.skipTest("BufferWithSegments not available")
 
         b = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
         self.assertEqual(len(b), 1)
         self.assertEqual(b.size, 3)
         self.assertEqual(b.tobytes(), b"foo")
 
         self.assertEqual(len(b[0]), 3)
         self.assertEqual(b[0].offset, 0)
         self.assertEqual(b[0].tobytes(), b"foo")
 
     def test_multiple(self):
         if not hasattr(zstd, "BufferWithSegments"):
             self.skipTest("BufferWithSegments not available")
 
         b = zstd.BufferWithSegments(
-            b"foofooxfooxy", b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)])
+            b"foofooxfooxy",
+            b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)]),
         )
         self.assertEqual(len(b), 3)
         self.assertEqual(b.size, 12)
         self.assertEqual(b.tobytes(), b"foofooxfooxy")
 
         self.assertEqual(b[0].tobytes(), b"foo")
         self.assertEqual(b[1].tobytes(), b"foox")
         self.assertEqual(b[2].tobytes(), b"fooxy")
 
 
 class TestBufferWithSegmentsCollection(TestCase):
     def test_empty_constructor(self):
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
-        with self.assertRaisesRegex(ValueError, "must pass at least 1 argument"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass at least 1 argument"
+        ):
             zstd.BufferWithSegmentsCollection()
 
     def test_argument_validation(self):
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
-        with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+        with self.assertRaisesRegex(
+            TypeError, "arguments must be BufferWithSegments"
+        ):
             zstd.BufferWithSegmentsCollection(None)
 
-        with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
+        with self.assertRaisesRegex(
+            TypeError, "arguments must be BufferWithSegments"
+        ):
             zstd.BufferWithSegmentsCollection(
                 zstd.BufferWithSegments(b"foo", ss.pack(0, 3)), None
             )
 
         with self.assertRaisesRegex(
             ValueError, "ZstdBufferWithSegments cannot be empty"
         ):
             zstd.BufferWithSegmentsCollection(zstd.BufferWithSegments(b"", b""))
 
     def test_length(self):
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
         b1 = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
         b2 = zstd.BufferWithSegments(
             b"barbaz", b"".join([ss.pack(0, 3), ss.pack(3, 3)])
         )
 
         c = zstd.BufferWithSegmentsCollection(b1)
         self.assertEqual(len(c), 1)
         self.assertEqual(c.size(), 3)
 
         c = zstd.BufferWithSegmentsCollection(b2)
         self.assertEqual(len(c), 2)
         self.assertEqual(c.size(), 6)
 
         c = zstd.BufferWithSegmentsCollection(b1, b2)
         self.assertEqual(len(c), 3)
         self.assertEqual(c.size(), 9)
 
     def test_getitem(self):
         if not hasattr(zstd, "BufferWithSegmentsCollection"):
             self.skipTest("BufferWithSegmentsCollection not available")
 
         b1 = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
         b2 = zstd.BufferWithSegments(
             b"barbaz", b"".join([ss.pack(0, 3), ss.pack(3, 3)])
         )
 
         c = zstd.BufferWithSegmentsCollection(b1, b2)
 
         with self.assertRaisesRegex(IndexError, "offset must be less than 3"):
             c[3]
 
         with self.assertRaisesRegex(IndexError, "offset must be less than 3"):
             c[4]
 
         self.assertEqual(c[0].tobytes(), b"foo")
         self.assertEqual(c[1].tobytes(), b"bar")
         self.assertEqual(c[2].tobytes(), b"baz")

contrib/python-zstandard/tests/test_compressor.py

@@ -1,1770 +1,1803 b''
 import hashlib
 import io
 import os
 import struct
 import sys
 import tarfile
 import tempfile
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     NonClosingBytesIO,
     OpCountingBytesIO,
     TestCase,
 )
 
 
 if sys.version_info[0] >= 3:
     next = lambda it: it.__next__()
 else:
     next = lambda it: it.next()
 
 
 def multithreaded_chunk_size(level, source_size=0):
-    params = zstd.ZstdCompressionParameters.from_level(level, source_size=source_size)
+    params = zstd.ZstdCompressionParameters.from_level(
+        level, source_size=source_size
+    )
 
     return 1 << (params.window_log + 2)
 
 
 @make_cffi
 class TestCompressor(TestCase):
     def test_level_bounds(self):
         with self.assertRaises(ValueError):
             zstd.ZstdCompressor(level=23)
 
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         self.assertGreater(cctx.memory_size(), 100)
 
 
 @make_cffi
 class TestCompressor_compress(TestCase):
     def test_compress_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         result = cctx.compress(b"")
         self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum, 0)
 
         cctx = zstd.ZstdCompressor()
         result = cctx.compress(b"")
         self.assertEqual(result, b"\x28\xb5\x2f\xfd\x20\x00\x01\x00\x00")
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, 0)
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         expected = b"\x28\xb5\x2f\xfd\x00\x00\x19\x00\x00\x66\x6f\x6f"
 
         mutable_array = bytearray(3)
         mutable_array[:] = b"foo"
 
         sources = [
             memoryview(b"foo"),
             bytearray(b"foo"),
             mutable_array,
         ]
 
         for source in sources:
             self.assertEqual(cctx.compress(source), expected)
 
     def test_compress_large(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct(">B").pack(i) * 16384)
 
         cctx = zstd.ZstdCompressor(level=3, write_content_size=False)
         result = cctx.compress(b"".join(chunks))
         self.assertEqual(len(result), 999)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         # This matches the test for read_to_iter() below.
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
-        result = cctx.compress(b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o")
+        result = cctx.compress(
+            b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b"o"
+        )
         self.assertEqual(
             result,
             b"\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00"
             b"\x10\x66\x66\x01\x00\xfb\xff\x39\xc0"
             b"\x02\x09\x00\x00\x6f",
         )
 
     def test_negative_level(self):
         cctx = zstd.ZstdCompressor(level=-4)
         result = cctx.compress(b"foo" * 256)
 
     def test_no_magic(self):
-        params = zstd.ZstdCompressionParameters.from_level(1, format=zstd.FORMAT_ZSTD1)
+        params = zstd.ZstdCompressionParameters.from_level(
+            1, format=zstd.FORMAT_ZSTD1
+        )
         cctx = zstd.ZstdCompressor(compression_params=params)
         magic = cctx.compress(b"foobar")
 
         params = zstd.ZstdCompressionParameters.from_level(
             1, format=zstd.FORMAT_ZSTD1_MAGICLESS
         )
         cctx = zstd.ZstdCompressor(compression_params=params)
         no_magic = cctx.compress(b"foobar")
 
         self.assertEqual(magic[0:4], b"\x28\xb5\x2f\xfd")
         self.assertEqual(magic[4:], no_magic)
 
     def test_write_checksum(self):
         cctx = zstd.ZstdCompressor(level=1)
         no_checksum = cctx.compress(b"foobar")
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         with_checksum = cctx.compress(b"foobar")
 
         self.assertEqual(len(with_checksum), len(no_checksum) + 4)
 
         no_params = zstd.get_frame_parameters(no_checksum)
         with_params = zstd.get_frame_parameters(with_checksum)
 
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         with_size = cctx.compress(b"foobar" * 256)
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         no_size = cctx.compress(b"foobar" * 256)
 
         self.assertEqual(len(with_size), len(no_size) + 1)
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
 
     def test_no_dict_id(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         with_dict_id = cctx.compress(b"foobarfoobar")
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
         no_dict_id = cctx.compress(b"foobarfoobar")
 
         self.assertEqual(len(with_dict_id), len(no_dict_id) + 4)
 
         no_params = zstd.get_frame_parameters(no_dict_id)
         with_params = zstd.get_frame_parameters(with_dict_id)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 1880053135)
 
     def test_compress_dict_multiple(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
 
         for i in range(32):
             cctx.compress(b"foo bar foobar foo bar foobar")
 
     def test_dict_precompute(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
         d.precompute_compress(level=1)
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
 
         for i in range(32):
             cctx.compress(b"foo bar foobar foo bar foobar")
 
     def test_multithreaded(self):
         chunk_size = multithreaded_chunk_size(1)
         source = b"".join([b"x" * chunk_size, b"y" * chunk_size])
 
         cctx = zstd.ZstdCompressor(level=1, threads=2)
         compressed = cctx.compress(source)
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, chunk_size * 2)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(dctx.decompress(compressed), source)
 
     def test_multithreaded_dict(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         cctx = zstd.ZstdCompressor(dict_data=d, threads=2)
 
         result = cctx.compress(b"foo")
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, 3)
         self.assertEqual(params.dict_id, d.dict_id())
 
         self.assertEqual(
             result,
-            b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00" b"\x66\x6f\x6f",
+            b"\x28\xb5\x2f\xfd\x23\x8f\x55\x0f\x70\x03\x19\x00\x00"
+            b"\x66\x6f\x6f",
         )
 
     def test_multithreaded_compression_params(self):
         params = zstd.ZstdCompressionParameters.from_level(0, threads=2)
         cctx = zstd.ZstdCompressor(compression_params=params)
 
         result = cctx.compress(b"foo")
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, 3)
 
-        self.assertEqual(result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f")
+        self.assertEqual(
+            result, b"\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f"
+        )
 
 
 @make_cffi
 class TestCompressor_compressobj(TestCase):
     def test_compressobj_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cobj = cctx.compressobj()
         self.assertEqual(cobj.compress(b""), b"")
         self.assertEqual(cobj.flush(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
 
     def test_input_types(self):
         expected = b"\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f"
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         mutable_array = bytearray(3)
         mutable_array[:] = b"foo"
 
         sources = [
             memoryview(b"foo"),
             bytearray(b"foo"),
             mutable_array,
         ]
 
         for source in sources:
             cobj = cctx.compressobj()
             self.assertEqual(cobj.compress(source), b"")
             self.assertEqual(cobj.flush(), expected)
 
     def test_compressobj_large(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct(">B").pack(i) * 16384)
 
         cctx = zstd.ZstdCompressor(level=3)
         cobj = cctx.compressobj()
 
         result = cobj.compress(b"".join(chunks)) + cobj.flush()
         self.assertEqual(len(result), 999)
         self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd")
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
     def test_write_checksum(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj()
         no_checksum = cobj.compress(b"foobar") + cobj.flush()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         cobj = cctx.compressobj()
         with_checksum = cobj.compress(b"foobar") + cobj.flush()
 
         no_params = zstd.get_frame_parameters(no_checksum)
         with_params = zstd.get_frame_parameters(with_checksum)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
         self.assertEqual(len(with_checksum), len(no_checksum) + 4)
 
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj(size=len(b"foobar" * 256))
         with_size = cobj.compress(b"foobar" * 256) + cobj.flush()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cobj = cctx.compressobj(size=len(b"foobar" * 256))
         no_size = cobj.compress(b"foobar" * 256) + cobj.flush()
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
         self.assertEqual(len(with_size), len(no_size) + 1)
 
     def test_compress_after_finished(self):
         cctx = zstd.ZstdCompressor()
         cobj = cctx.compressobj()
 
         cobj.compress(b"foo")
         cobj.flush()
 
         with self.assertRaisesRegex(
             zstd.ZstdError, r"cannot call compress\(\) after compressor"
         ):
             cobj.compress(b"foo")
 
         with self.assertRaisesRegex(
             zstd.ZstdError, "compressor object already finished"
         ):
             cobj.flush()
 
     def test_flush_block_repeated(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj()
 
         self.assertEqual(cobj.compress(b"foo"), b"")
         self.assertEqual(
             cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK),
             b"\x28\xb5\x2f\xfd\x00\x48\x18\x00\x00foo",
         )
         self.assertEqual(cobj.compress(b"bar"), b"")
         # 3 byte header plus content.
-        self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar")
+        self.assertEqual(
+            cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"\x18\x00\x00bar"
+        )
         self.assertEqual(cobj.flush(), b"\x01\x00\x00")
 
     def test_flush_empty_block(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
         cobj = cctx.compressobj()
 
         cobj.compress(b"foobar")
         cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK)
         # No-op if no block is active (this is internal to zstd).
         self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b"")
 
         trailing = cobj.flush()
         # 3 bytes block header + 4 bytes frame checksum
         self.assertEqual(len(trailing), 7)
         header = trailing[0:3]
         self.assertEqual(header, b"\x01\x00\x00")
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b"a" * 1048576)
         source.write(b"b" * 1048576)
         source.write(b"c" * 1048576)
         source.seek(0)
 
         cctx = zstd.ZstdCompressor(level=1, threads=2)
         cobj = cctx.compressobj()
 
         chunks = []
         while True:
             d = source.read(8192)
             if not d:
                 break
 
             chunks.append(cobj.compress(d))
 
         chunks.append(cobj.flush())
 
         compressed = b"".join(chunks)
 
         self.assertEqual(len(compressed), 119)
 
     def test_frame_progression(self):
         cctx = zstd.ZstdCompressor()
 
         self.assertEqual(cctx.frame_progression(), (0, 0, 0))
 
         cobj = cctx.compressobj()
 
         cobj.compress(b"foobar")
         self.assertEqual(cctx.frame_progression(), (6, 0, 0))
 
         cobj.flush()
         self.assertEqual(cctx.frame_progression(), (6, 6, 15))
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         cobj = cctx.compressobj(size=2)
         with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
             cobj.compress(b"foo")
 
         # Try another operation on this instance.
         with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
             cobj.compress(b"aa")
 
         # Try another operation on the compressor.
         cctx.compressobj(size=4)
         cctx.compress(b"foobar")
 
 
 @make_cffi
 class TestCompressor_copy_stream(TestCase):
     def test_no_read(self):
         source = object()
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor()
         with self.assertRaises(ValueError):
             cctx.copy_stream(source, dest)
 
     def test_no_write(self):
         source = io.BytesIO()
         dest = object()
 
         cctx = zstd.ZstdCompressor()
         with self.assertRaises(ValueError):
             cctx.copy_stream(source, dest)
 
     def test_empty(self):
         source = io.BytesIO()
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(int(r), 0)
         self.assertEqual(w, 9)
 
-        self.assertEqual(dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
+        self.assertEqual(
+            dest.getvalue(), b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00"
+        )
 
     def test_large_data(self):
         source = io.BytesIO()
         for i in range(255):
             source.write(struct.Struct(">B").pack(i) * 16384)
         source.seek(0)
 
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         r, w = cctx.copy_stream(source, dest)
 
         self.assertEqual(r, 255 * 16384)
         self.assertEqual(w, 999)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
     def test_write_checksum(self):
         source = io.BytesIO(b"foobar")
         no_checksum = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1)
         cctx.copy_stream(source, no_checksum)
 
         source.seek(0)
         with_checksum = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         cctx.copy_stream(source, with_checksum)
 
-        self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+        self.assertEqual(
+            len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+        )
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
     def test_write_content_size(self):
         source = io.BytesIO(b"foobar" * 256)
         no_size = io.BytesIO()
 
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cctx.copy_stream(source, no_size)
 
         source.seek(0)
         with_size = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         cctx.copy_stream(source, with_size)
 
         # Source content size is unknown, so no content size written.
         self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()))
 
         source.seek(0)
         with_size = io.BytesIO()
         cctx.copy_stream(source, with_size, size=len(source.getvalue()))
 
         # We specified source size, so content size header is present.
         self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()) + 1)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(b"foobarfoobar")
         dest = OpCountingBytesIO()
         cctx = zstd.ZstdCompressor()
         r, w = cctx.copy_stream(source, dest, read_size=1, write_size=1)
 
         self.assertEqual(r, len(source.getvalue()))
         self.assertEqual(w, 21)
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
         self.assertEqual(dest._write_count, len(dest.getvalue()))
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b"a" * 1048576)
         source.write(b"b" * 1048576)
         source.write(b"c" * 1048576)
         source.seek(0)
 
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor(threads=2, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(r, 3145728)
         self.assertEqual(w, 111)
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Writing content size and checksum works.
         cctx = zstd.ZstdCompressor(threads=2, write_checksum=True)
         dest = io.BytesIO()
         source.seek(0)
         cctx.copy_stream(source, dest, size=len(source.getvalue()))
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, 3145728)
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
 
     def test_bad_size(self):
         source = io.BytesIO()
         source.write(b"a" * 32768)
         source.write(b"b" * 32768)
         source.seek(0)
 
         dest = io.BytesIO()
 
         cctx = zstd.ZstdCompressor()
 
         with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
             cctx.copy_stream(source, dest, size=42)
 
         # Try another operation on this compressor.
         source.seek(0)
         dest = io.BytesIO()
         cctx.copy_stream(source, dest)
 
 
 @make_cffi
 class TestCompressor_stream_reader(TestCase):
     def test_context_manager(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"foo") as reader:
-            with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot __enter__ multiple times"
+            ):
                 with reader as reader2:
                     pass
 
     def test_no_context_manager(self):
         cctx = zstd.ZstdCompressor()
 
         reader = cctx.stream_reader(b"foo")
         reader.read(4)
         self.assertFalse(reader.closed)
 
         reader.close()
         self.assertTrue(reader.closed)
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             reader.read(1)
 
     def test_not_implemented(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"foo" * 60) as reader:
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readline()
 
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readlines()
 
             with self.assertRaises(io.UnsupportedOperation):
                 iter(reader)
 
             with self.assertRaises(io.UnsupportedOperation):
                 next(reader)
 
             with self.assertRaises(OSError):
                 reader.writelines([])
 
             with self.assertRaises(OSError):
                 reader.write(b"foo")
 
     def test_constant_methods(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"boo") as reader:
             self.assertTrue(reader.readable())
             self.assertFalse(reader.writable())
             self.assertFalse(reader.seekable())
             self.assertFalse(reader.isatty())
             self.assertFalse(reader.closed)
             self.assertIsNone(reader.flush())
             self.assertFalse(reader.closed)
 
         self.assertTrue(reader.closed)
 
     def test_read_closed(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"foo" * 60) as reader:
             reader.close()
             self.assertTrue(reader.closed)
             with self.assertRaisesRegex(ValueError, "stream is closed"):
                 reader.read(10)
 
     def test_read_sizes(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         with cctx.stream_reader(b"foo") as reader:
             with self.assertRaisesRegex(
                 ValueError, "cannot read negative amounts less than -1"
             ):
                 reader.read(-2)
 
             self.assertEqual(reader.read(0), b"")
             self.assertEqual(reader.read(), foo)
 
     def test_read_buffer(self):
         cctx = zstd.ZstdCompressor()
 
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         with cctx.stream_reader(source) as reader:
             self.assertEqual(reader.tell(), 0)
 
             # We should get entire frame in one read.
             result = reader.read(8192)
             self.assertEqual(result, frame)
             self.assertEqual(reader.tell(), len(result))
             self.assertEqual(reader.read(), b"")
             self.assertEqual(reader.tell(), len(result))
 
     def test_read_buffer_small_chunks(self):
         cctx = zstd.ZstdCompressor()
 
         source = b"foo" * 60
         chunks = []
 
         with cctx.stream_reader(source) as reader:
             self.assertEqual(reader.tell(), 0)
 
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b"".join(chunks), cctx.compress(source))
 
     def test_read_stream(self):
         cctx = zstd.ZstdCompressor()
 
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
             self.assertEqual(reader.tell(), 0)
 
             chunk = reader.read(8192)
             self.assertEqual(chunk, frame)
             self.assertEqual(reader.tell(), len(chunk))
             self.assertEqual(reader.read(), b"")
             self.assertEqual(reader.tell(), len(chunk))
 
     def test_read_stream_small_chunks(self):
         cctx = zstd.ZstdCompressor()
 
         source = b"foo" * 60
         chunks = []
 
         with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
             self.assertEqual(reader.tell(), 0)
 
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b"".join(chunks), cctx.compress(source))
 
     def test_read_after_exit(self):
         cctx = zstd.ZstdCompressor()
 
         with cctx.stream_reader(b"foo" * 60) as reader:
             while reader.read(8192):
                 pass
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             reader.read(10)
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         source = io.BytesIO(b"foobar")
 
         with cctx.stream_reader(source, size=2) as reader:
-            with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
+            with self.assertRaisesRegex(
+                zstd.ZstdError, "Src size is incorrect"
+            ):
                 reader.read(10)
 
         # Try another compression operation.
         with cctx.stream_reader(source, size=42):
             pass
 
     def test_readall(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b"foo" * 1024)
 
         reader = cctx.stream_reader(b"foo" * 1024)
         self.assertEqual(reader.readall(), frame)
 
     def test_readinto(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         reader = cctx.stream_reader(b"foo")
         with self.assertRaises(Exception):
             reader.readinto(b"foobar")
 
         # readinto() with sufficiently large destination.
         b = bytearray(1024)
         reader = cctx.stream_reader(b"foo")
         self.assertEqual(reader.readinto(b), len(foo))
         self.assertEqual(b[0 : len(foo)], foo)
         self.assertEqual(reader.readinto(b), 0)
         self.assertEqual(b[0 : len(foo)], foo)
 
         # readinto() with small reads.
         b = bytearray(1024)
         reader = cctx.stream_reader(b"foo", read_size=1)
         self.assertEqual(reader.readinto(b), len(foo))
         self.assertEqual(b[0 : len(foo)], foo)
 
         # Too small destination buffer.
         b = bytearray(2)
         reader = cctx.stream_reader(b"foo")
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[0:2])
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[2:4])
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], foo[4:6])
 
     def test_readinto1(self):
         cctx = zstd.ZstdCompressor()
         foo = b"".join(cctx.read_to_iter(io.BytesIO(b"foo")))
 
         reader = cctx.stream_reader(b"foo")
         with self.assertRaises(Exception):
             reader.readinto1(b"foobar")
 
         b = bytearray(1024)
         source = OpCountingBytesIO(b"foo")
         reader = cctx.stream_reader(source)
         self.assertEqual(reader.readinto1(b), len(foo))
         self.assertEqual(b[0 : len(foo)], foo)
         self.assertEqual(source._read_count, 2)
 
         # readinto1() with small reads.
         b = bytearray(1024)
         source = OpCountingBytesIO(b"foo")
         reader = cctx.stream_reader(source, read_size=1)
         self.assertEqual(reader.readinto1(b), len(foo))
         self.assertEqual(b[0 : len(foo)], foo)
         self.assertEqual(source._read_count, 4)
 
     def test_read1(self):
         cctx = zstd.ZstdCompressor()
         foo = b"".join(cctx.read_to_iter(io.BytesIO(b"foo")))
 
         b = OpCountingBytesIO(b"foo")
         reader = cctx.stream_reader(b)
 
         self.assertEqual(reader.read1(), foo)
         self.assertEqual(b._read_count, 2)
 
         b = OpCountingBytesIO(b"foo")
         reader = cctx.stream_reader(b)
 
         self.assertEqual(reader.read1(0), b"")
         self.assertEqual(reader.read1(2), foo[0:2])
         self.assertEqual(b._read_count, 2)
         self.assertEqual(reader.read1(2), foo[2:4])
         self.assertEqual(reader.read1(1024), foo[4:])
 
 
 @make_cffi
 class TestCompressor_stream_writer(TestCase):
     def test_io_api(self):
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         writer = cctx.stream_writer(buffer)
 
         self.assertFalse(writer.isatty())
         self.assertFalse(writer.readable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(hint=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(0)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(10, os.SEEK_SET)
 
         self.assertFalse(writer.seekable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(size=42)
 
         self.assertTrue(writer.writable())
 
         with self.assertRaises(NotImplementedError):
             writer.writelines([])
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readall()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readinto(None)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.fileno()
 
         self.assertFalse(writer.closed)
 
     def test_fileno_file(self):
         with tempfile.TemporaryFile("wb") as tf:
             cctx = zstd.ZstdCompressor()
             writer = cctx.stream_writer(tf)
 
             self.assertEqual(writer.fileno(), tf.fileno())
 
     def test_close(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         writer = cctx.stream_writer(buffer)
 
         writer.write(b"foo" * 1024)
         self.assertFalse(writer.closed)
         self.assertFalse(buffer.closed)
         writer.close()
         self.assertTrue(writer.closed)
         self.assertTrue(buffer.closed)
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             writer.write(b"foo")
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             writer.flush()
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             with writer:
                 pass
 
         self.assertEqual(
             buffer.getvalue(),
             b"\x28\xb5\x2f\xfd\x00\x48\x55\x00\x00\x18\x66\x6f"
             b"\x6f\x01\x00\xfa\xd3\x77\x43",
         )
 
         # Context manager exit should close stream.
         buffer = io.BytesIO()
         writer = cctx.stream_writer(buffer)
 
         with writer:
             writer.write(b"foo")
 
         self.assertTrue(writer.closed)
 
     def test_empty(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(buffer) as compressor:
             compressor.write(b"")
 
         result = buffer.getvalue()
         self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Test without context manager.
         buffer = io.BytesIO()
         compressor = cctx.stream_writer(buffer)
         self.assertEqual(compressor.write(b""), 0)
         self.assertEqual(buffer.getvalue(), b"")
         self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 9)
         result = buffer.getvalue()
         self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Test write_return_read=True
         compressor = cctx.stream_writer(buffer, write_return_read=True)
         self.assertEqual(compressor.write(b""), 0)
 
     def test_input_types(self):
         expected = b"\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f"
         cctx = zstd.ZstdCompressor(level=1)
 
         mutable_array = bytearray(3)
         mutable_array[:] = b"foo"
 
         sources = [
             memoryview(b"foo"),
             bytearray(b"foo"),
             mutable_array,
         ]
 
         for source in sources:
             buffer = NonClosingBytesIO()
             with cctx.stream_writer(buffer) as compressor:
                 compressor.write(source)
 
             self.assertEqual(buffer.getvalue(), expected)
 
             compressor = cctx.stream_writer(buffer, write_return_read=True)
             self.assertEqual(compressor.write(source), len(source))
 
     def test_multiple_compress(self):
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=5)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b"foo"), 0)
             self.assertEqual(compressor.write(b"bar"), 0)
             self.assertEqual(compressor.write(b"x" * 8192), 0)
 
         result = buffer.getvalue()
         self.assertEqual(
             result,
             b"\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f"
             b"\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23",
         )
 
         # Test without context manager.
         buffer = io.BytesIO()
         compressor = cctx.stream_writer(buffer)
         self.assertEqual(compressor.write(b"foo"), 0)
         self.assertEqual(compressor.write(b"bar"), 0)
         self.assertEqual(compressor.write(b"x" * 8192), 0)
         self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
         result = buffer.getvalue()
         self.assertEqual(
             result,
             b"\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f"
             b"\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23",
         )
 
         # Test with write_return_read=True.
         compressor = cctx.stream_writer(buffer, write_return_read=True)
         self.assertEqual(compressor.write(b"foo"), 3)
         self.assertEqual(compressor.write(b"barbiz"), 6)
         self.assertEqual(compressor.write(b"x" * 8192), 8192)
 
     def test_dictionary(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         h = hashlib.sha1(d.as_bytes()).hexdigest()
         self.assertEqual(h, "7a2e59a876db958f74257141045af8f912e00d4e")
 
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=9, dict_data=d)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b"foo"), 0)
             self.assertEqual(compressor.write(b"bar"), 0)
             self.assertEqual(compressor.write(b"foo" * 16384), 0)
 
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, d.dict_id())
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
         self.assertEqual(h, "0a7c05635061f58039727cdbe76388c6f4cfef06")
 
         source = b"foo" + b"bar" + (b"foo" * 16384)
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
 
         self.assertEqual(
             dctx.decompress(compressed, max_output_size=len(source)), source
         )
 
     def test_compression_params(self):
         params = zstd.ZstdCompressionParameters(
             window_log=20,
             chain_log=6,
             hash_log=12,
             min_match=5,
             search_log=4,
             target_length=10,
             strategy=zstd.STRATEGY_FAST,
         )
 
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(compression_params=params)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b"foo"), 0)
             self.assertEqual(compressor.write(b"bar"), 0)
             self.assertEqual(compressor.write(b"foobar" * 16384), 0)
 
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1048576)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
         self.assertEqual(h, "dd4bb7d37c1a0235b38a2f6b462814376843ef0b")
 
     def test_write_checksum(self):
         no_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(no_checksum) as compressor:
             self.assertEqual(compressor.write(b"foobar"), 0)
 
         with_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         with cctx.stream_writer(with_checksum) as compressor:
             self.assertEqual(compressor.write(b"foobar"), 0)
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertTrue(with_params.has_checksum)
 
-        self.assertEqual(len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4)
+        self.assertEqual(
+            len(with_checksum.getvalue()), len(no_checksum.getvalue()) + 4
+        )
 
     def test_write_content_size(self):
         no_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(no_size) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 256), 0)
 
         with_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(with_size) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 256), 0)
 
         # Source size is not known in streaming mode, so header not
         # written.
         self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()))
 
         # Declaring size will write the header.
         with_size = NonClosingBytesIO()
-        with cctx.stream_writer(with_size, size=len(b"foobar" * 256)) as compressor:
+        with cctx.stream_writer(
+            with_size, size=len(b"foobar" * 256)
+        ) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 256), 0)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
         self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()) + 1)
 
     def test_no_dict_id(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(1024, samples)
 
         with_dict_id = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         with cctx.stream_writer(with_dict_id) as compressor:
             self.assertEqual(compressor.write(b"foobarfoobar"), 0)
 
         self.assertEqual(with_dict_id.getvalue()[4:5], b"\x03")
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
         no_dict_id = NonClosingBytesIO()
         with cctx.stream_writer(no_dict_id) as compressor:
             self.assertEqual(compressor.write(b"foobarfoobar"), 0)
 
         self.assertEqual(no_dict_id.getvalue()[4:5], b"\x00")
 
         no_params = zstd.get_frame_parameters(no_dict_id.getvalue())
         with_params = zstd.get_frame_parameters(with_dict_id.getvalue())
         self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, d.dict_id())
         self.assertFalse(no_params.has_checksum)
         self.assertFalse(with_params.has_checksum)
 
-        self.assertEqual(len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4)
+        self.assertEqual(
+            len(with_dict_id.getvalue()), len(no_dict_id.getvalue()) + 4
+        )
 
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         buffer = io.BytesIO()
         with cctx.stream_writer(buffer) as compressor:
             compressor.write(b"foo")
             size = compressor.memory_size()
 
         self.assertGreater(size, 100000)
 
     def test_write_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest, write_size=1) as compressor:
             self.assertEqual(compressor.write(b"foo"), 0)
             self.assertEqual(compressor.write(b"bar"), 0)
             self.assertEqual(compressor.write(b"foobar"), 0)
 
         self.assertEqual(len(dest.getvalue()), dest._write_count)
 
     def test_flush_repeated(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b"foo"), 0)
             self.assertEqual(dest._write_count, 0)
             self.assertEqual(compressor.flush(), 12)
             self.assertEqual(dest._write_count, 1)
             self.assertEqual(compressor.write(b"bar"), 0)
             self.assertEqual(dest._write_count, 1)
             self.assertEqual(compressor.flush(), 6)
             self.assertEqual(dest._write_count, 2)
             self.assertEqual(compressor.write(b"baz"), 0)
 
         self.assertEqual(dest._write_count, 3)
 
     def test_flush_empty_block(self):
         cctx = zstd.ZstdCompressor(level=3, write_checksum=True)
         dest = OpCountingBytesIO()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 8192), 0)
             count = dest._write_count
             offset = dest.tell()
             self.assertEqual(compressor.flush(), 23)
             self.assertGreater(dest._write_count, count)
             self.assertGreater(dest.tell(), offset)
             offset = dest.tell()
             # Ending the write here should cause an empty block to be written
             # to denote end of frame.
 
         trailing = dest.getvalue()[offset:]
         # 3 bytes block header + 4 bytes frame checksum
         self.assertEqual(len(trailing), 7)
 
         header = trailing[0:3]
         self.assertEqual(header, b"\x01\x00\x00")
 
     def test_flush_frame(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
 
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b"foobar" * 8192), 0)
             self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
             compressor.write(b"biz" * 16384)
 
         self.assertEqual(
             dest.getvalue(),
             # Frame 1.
             b"\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x30\x66\x6f\x6f"
             b"\x62\x61\x72\x01\x00\xf7\xbf\xe8\xa5\x08"
             # Frame 2.
             b"\x28\xb5\x2f\xfd\x00\x58\x5d\x00\x00\x18\x62\x69\x7a"
             b"\x01\x00\xfa\x3f\x75\x37\x04",
         )
 
     def test_bad_flush_mode(self):
         cctx = zstd.ZstdCompressor()
         dest = io.BytesIO()
         with cctx.stream_writer(dest) as compressor:
             with self.assertRaisesRegex(ValueError, "unknown flush_mode: 42"):
                 compressor.flush(flush_mode=42)
 
     def test_multithreaded(self):
         dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(threads=2)
         with cctx.stream_writer(dest) as compressor:
             compressor.write(b"a" * 1048576)
             compressor.write(b"b" * 1048576)
             compressor.write(b"c" * 1048576)
 
         self.assertEqual(len(dest.getvalue()), 111)
 
     def test_tell(self):
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.tell(), 0)
 
             for i in range(256):
                 compressor.write(b"foo" * (i + 1))
                 self.assertEqual(compressor.tell(), dest.tell())
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         dest = io.BytesIO()
 
         with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
             with cctx.stream_writer(dest, size=2) as compressor:
                 compressor.write(b"foo")
 
         # Test another operation.
         with cctx.stream_writer(dest, size=42):
             pass
 
     def test_tarfile_compat(self):
         dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor()
         with cctx.stream_writer(dest) as compressor:
             with tarfile.open("tf", mode="w|", fileobj=compressor) as tf:
                 tf.add(__file__, "test_compressor.py")
 
         dest = io.BytesIO(dest.getvalue())
 
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_reader(dest) as reader:
             with tarfile.open(mode="r|", fileobj=reader) as tf:
                 for member in tf:
                     self.assertEqual(member.name, "test_compressor.py")
 
 
 @make_cffi
 class TestCompressor_read_to_iter(TestCase):
     def test_type_validation(self):
         cctx = zstd.ZstdCompressor()
 
         # Object with read() works.
         for chunk in cctx.read_to_iter(io.BytesIO()):
             pass
 
         # Buffer protocol works.
         for chunk in cctx.read_to_iter(b"foobar"):
             pass
 
-        with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass an object with a read"
+        ):
             for chunk in cctx.read_to_iter(True):
                 pass
 
     def test_read_empty(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
         it = cctx.read_to_iter(source)
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed = b"".join(chunks)
         self.assertEqual(compressed, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
 
         # And again with the buffer protocol.
         it = cctx.read_to_iter(b"")
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed2 = b"".join(chunks)
         self.assertEqual(compressed2, compressed)
 
     def test_read_large(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
         source.write(b"f" * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE)
         source.write(b"o")
         source.seek(0)
 
         # Creating an iterator should not perform any compression until
         # first read.
         it = cctx.read_to_iter(source, size=len(source.getvalue()))
         self.assertEqual(source.tell(), 0)
 
         # We should have exactly 2 output chunks.
         chunks = []
         chunk = next(it)
         self.assertIsNotNone(chunk)
         self.assertEqual(source.tell(), zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE)
         chunks.append(chunk)
         chunk = next(it)
         self.assertIsNotNone(chunk)
         chunks.append(chunk)
 
         self.assertEqual(source.tell(), len(source.getvalue()))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         # And again for good measure.
         with self.assertRaises(StopIteration):
             next(it)
 
         # We should get the same output as the one-shot compression mechanism.
         self.assertEqual(b"".join(chunks), cctx.compress(source.getvalue()))
 
         params = zstd.get_frame_parameters(b"".join(chunks))
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Now check the buffer protocol.
         it = cctx.read_to_iter(source.getvalue())
         chunks = list(it)
         self.assertEqual(len(chunks), 2)
 
         params = zstd.get_frame_parameters(b"".join(chunks))
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         # self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         self.assertEqual(b"".join(chunks), cctx.compress(source.getvalue()))
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(b"foobarfoobar")
         cctx = zstd.ZstdCompressor(level=3)
         for chunk in cctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
 
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
 
     def test_multithreaded(self):
         source = io.BytesIO()
         source.write(b"a" * 1048576)
         source.write(b"b" * 1048576)
         source.write(b"c" * 1048576)
         source.seek(0)
 
         cctx = zstd.ZstdCompressor(threads=2)
 
         compressed = b"".join(cctx.read_to_iter(source))
         self.assertEqual(len(compressed), 111)
 
     def test_bad_size(self):
         cctx = zstd.ZstdCompressor()
 
         source = io.BytesIO(b"a" * 42)
 
         with self.assertRaisesRegex(zstd.ZstdError, "Src size is incorrect"):
             b"".join(cctx.read_to_iter(source, size=2))
 
         # Test another operation on errored compressor.
         b"".join(cctx.read_to_iter(source))
 
 
 @make_cffi
 class TestCompressor_chunker(TestCase):
     def test_empty(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         chunker = cctx.chunker()
 
         it = chunker.compress(b"")
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.finish()
 
         self.assertEqual(next(it), b"\x28\xb5\x2f\xfd\x00\x58\x01\x00\x00")
 
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_simple_input(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         it = chunker.compress(b"foobar")
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.compress(b"baz" * 30)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.finish()
 
         self.assertEqual(
             next(it),
             b"\x28\xb5\x2f\xfd\x00\x58\x7d\x00\x00\x48\x66\x6f"
             b"\x6f\x62\x61\x72\x62\x61\x7a\x01\x00\xe4\xe4\x8e",
         )
 
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_input_size(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker(size=1024)
 
         it = chunker.compress(b"x" * 1000)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         it = chunker.compress(b"y" * 24)
 
         with self.assertRaises(StopIteration):
             next(it)
 
         chunks = list(chunker.finish())
 
         self.assertEqual(
             chunks,
             [
                 b"\x28\xb5\x2f\xfd\x60\x00\x03\x65\x00\x00\x18\x78\x78\x79\x02\x00"
                 b"\xa0\x16\xe3\x2b\x80\x05"
             ],
         )
 
         dctx = zstd.ZstdDecompressor()
 
-        self.assertEqual(dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24))
+        self.assertEqual(
+            dctx.decompress(b"".join(chunks)), (b"x" * 1000) + (b"y" * 24)
+        )
 
     def test_small_chunk_size(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker(chunk_size=1)
 
         chunks = list(chunker.compress(b"foo" * 1024))
         self.assertEqual(chunks, [])
 
         chunks = list(chunker.finish())
         self.assertTrue(all(len(chunk) == 1 for chunk in chunks))
 
         self.assertEqual(
             b"".join(chunks),
             b"\x28\xb5\x2f\xfd\x00\x58\x55\x00\x00\x18\x66\x6f\x6f\x01\x00"
             b"\xfa\xd3\x77\x43",
         )
 
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(
-            dctx.decompress(b"".join(chunks), max_output_size=10000), b"foo" * 1024
+            dctx.decompress(b"".join(chunks), max_output_size=10000),
+            b"foo" * 1024,
         )
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor()
 
         mutable_array = bytearray(3)
         mutable_array[:] = b"foo"
 
         sources = [
             memoryview(b"foo"),
             bytearray(b"foo"),
             mutable_array,
         ]
 
         for source in sources:
             chunker = cctx.chunker()
 
             self.assertEqual(list(chunker.compress(source)), [])
             self.assertEqual(
                 list(chunker.finish()),
                 [b"\x28\xb5\x2f\xfd\x00\x58\x19\x00\x00\x66\x6f\x6f"],
             )
 
     def test_flush(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         self.assertEqual(list(chunker.compress(b"foo" * 1024)), [])
         self.assertEqual(list(chunker.compress(b"bar" * 1024)), [])
 
         chunks1 = list(chunker.flush())
 
         self.assertEqual(
             chunks1,
             [
                 b"\x28\xb5\x2f\xfd\x00\x58\x8c\x00\x00\x30\x66\x6f\x6f\x62\x61\x72"
                 b"\x02\x00\xfa\x03\xfe\xd0\x9f\xbe\x1b\x02"
             ],
         )
 
         self.assertEqual(list(chunker.flush()), [])
         self.assertEqual(list(chunker.flush()), [])
 
         self.assertEqual(list(chunker.compress(b"baz" * 1024)), [])
 
         chunks2 = list(chunker.flush())
         self.assertEqual(len(chunks2), 1)
 
         chunks3 = list(chunker.finish())
         self.assertEqual(len(chunks2), 1)
 
         dctx = zstd.ZstdDecompressor()
 
         self.assertEqual(
             dctx.decompress(
                 b"".join(chunks1 + chunks2 + chunks3), max_output_size=10000
             ),
             (b"foo" * 1024) + (b"bar" * 1024) + (b"baz" * 1024),
         )
 
     def test_compress_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b"foo"))
         list(chunker.finish())
 
         with self.assertRaisesRegex(
-            zstd.ZstdError, r"cannot call compress\(\) after compression finished"
+            zstd.ZstdError,
+            r"cannot call compress\(\) after compression finished",
         ):
             list(chunker.compress(b"foo"))
 
     def test_flush_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b"foo"))
         list(chunker.finish())
 
         with self.assertRaisesRegex(
             zstd.ZstdError, r"cannot call flush\(\) after compression finished"
         ):
             list(chunker.flush())
 
     def test_finish_after_finish(self):
         cctx = zstd.ZstdCompressor()
         chunker = cctx.chunker()
 
         list(chunker.compress(b"foo"))
         list(chunker.finish())
 
         with self.assertRaisesRegex(
             zstd.ZstdError, r"cannot call finish\(\) after compression finished"
         ):
             list(chunker.finish())
 
 
 class TestCompressor_multi_compress_to_buffer(TestCase):
     def test_invalid_inputs(self):
         cctx = zstd.ZstdCompressor()
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer(True)
 
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer((1, 2))
 
-        with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+        with self.assertRaisesRegex(
+            TypeError, "item 0 not a bytes like object"
+        ):
             cctx.multi_compress_to_buffer([u"foo"])
 
     def test_empty_input(self):
         cctx = zstd.ZstdCompressor()
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         with self.assertRaisesRegex(ValueError, "no source elements found"):
             cctx.multi_compress_to_buffer([])
 
         with self.assertRaisesRegex(ValueError, "source elements are empty"):
             cctx.multi_compress_to_buffer([b"", b"", b""])
 
     def test_list_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         original = [b"foo" * 12, b"bar" * 6]
         frames = [cctx.compress(c) for c in original]
         b = cctx.multi_compress_to_buffer(original)
 
         self.assertIsInstance(b, zstd.BufferWithSegmentsCollection)
 
         self.assertEqual(len(b), 2)
         self.assertEqual(b.size(), 44)
 
         self.assertEqual(b[0].tobytes(), frames[0])
         self.assertEqual(b[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         original = [b"foo" * 4, b"bar" * 6]
         frames = [cctx.compress(c) for c in original]
 
         offsets = struct.pack(
             "=QQQQ", 0, len(original[0]), len(original[0]), len(original[1])
         )
         segments = zstd.BufferWithSegments(b"".join(original), offsets)
 
         result = cctx.multi_compress_to_buffer(segments)
 
         self.assertEqual(len(result), 2)
         self.assertEqual(result.size(), 47)
 
         self.assertEqual(result[0].tobytes(), frames[0])
         self.assertEqual(result[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_collection_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         original = [
             b"foo1",
             b"foo2" * 2,
             b"foo3" * 3,
             b"foo4" * 4,
             b"foo5" * 5,
         ]
 
         frames = [cctx.compress(c) for c in original]
 
         b = b"".join([original[0], original[1]])
         b1 = zstd.BufferWithSegments(
             b,
             struct.pack(
                 "=QQQQ", 0, len(original[0]), len(original[0]), len(original[1])
             ),
         )
         b = b"".join([original[2], original[3], original[4]])
         b2 = zstd.BufferWithSegments(
             b,
             struct.pack(
                 "=QQQQQQ",
                 0,
                 len(original[2]),
                 len(original[2]),
                 len(original[3]),
                 len(original[2]) + len(original[3]),
                 len(original[4]),
             ),
         )
 
         c = zstd.BufferWithSegmentsCollection(b1, b2)
 
         result = cctx.multi_compress_to_buffer(c)
 
         self.assertEqual(len(result), len(frames))
 
         for i, frame in enumerate(frames):
             self.assertEqual(result[i].tobytes(), frame)
 
     def test_multiple_threads(self):
         # threads argument will cause multi-threaded ZSTD APIs to be used, which will
         # make output different.
         refcctx = zstd.ZstdCompressor(write_checksum=True)
         reference = [refcctx.compress(b"x" * 64), refcctx.compress(b"y" * 64)]
 
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         frames = []
         frames.extend(b"x" * 64 for i in range(256))
         frames.extend(b"y" * 64 for i in range(256))
 
         result = cctx.multi_compress_to_buffer(frames, threads=-1)
 
         self.assertEqual(len(result), 512)
         for i in range(512):
             if i < 256:
                 self.assertEqual(result[i].tobytes(), reference[0])
             else:
                 self.assertEqual(result[i].tobytes(), reference[1])

contrib/python-zstandard/tests/test_compressor_fuzzing.py

@@ -1,836 +1,884 b''
 import io
 import os
 import unittest
 
 try:
     import hypothesis
     import hypothesis.strategies as strategies
 except ImportError:
     raise unittest.SkipTest("hypothesis not available")
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     NonClosingBytesIO,
     random_input_data,
     TestCase,
 )
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_stream_reader_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_read(self, original, level, source_read_size, read_size):
+    def test_stream_source_read(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 chunk = reader.read(read_size)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_read(self, original, level, source_read_size, read_size):
+    def test_buffer_source_read(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 chunk = reader.read(read_size)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_stream_source_read_variance(
         self, original, level, source_read_size, read_sizes
     ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 16384))
                 chunk = reader.read(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_buffer_source_read_variance(
         self, original, level, source_read_size, read_sizes
     ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 16384))
                 chunk = reader.read(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_readinto(self, original, level, source_read_size, read_size):
+    def test_stream_source_readinto(
+        self, original, level, source_read_size, read_size
+    ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 b = bytearray(read_size)
                 count = reader.readinto(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_readinto(self, original, level, source_read_size, read_size):
+    def test_buffer_source_readinto(
+        self, original, level, source_read_size, read_size
+    ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 b = bytearray(read_size)
                 count = reader.readinto(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_stream_source_readinto_variance(
         self, original, level, source_read_size, read_sizes
     ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(1, 16384))
                 b = bytearray(read_size)
                 count = reader.readinto(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_buffer_source_readinto_variance(
         self, original, level, source_read_size, read_sizes
     ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(1, 16384))
                 b = bytearray(read_size)
                 count = reader.readinto(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_stream_source_read1(self, original, level, source_read_size, read_size):
+    def test_stream_source_read1(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 chunk = reader.read1(read_size)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            -1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
-    def test_buffer_source_read1(self, original, level, source_read_size, read_size):
+    def test_buffer_source_read1(
+        self, original, level, source_read_size, read_size
+    ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 chunk = reader.read1(read_size)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_stream_source_read1_variance(
         self, original, level, source_read_size, read_sizes
     ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 16384))
                 chunk = reader.read1(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_buffer_source_read1_variance(
         self, original, level, source_read_size, read_sizes
     ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 16384))
                 chunk = reader.read1(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
     def test_stream_source_readinto1(
         self, original, level, source_read_size, read_size
     ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 b = bytearray(read_size)
                 count = reader.readinto1(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
-        read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+        read_size=strategies.integers(
+            1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        ),
     )
     def test_buffer_source_readinto1(
         self, original, level, source_read_size, read_size
     ):
         if read_size == 0:
             read_size = -1
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 b = bytearray(read_size)
                 count = reader.readinto1(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_stream_source_readinto1_variance(
         self, original, level, source_read_size, read_sizes
     ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             io.BytesIO(original), size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(1, 16384))
                 b = bytearray(read_size)
                 count = reader.readinto1(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 16384),
         read_sizes=strategies.data(),
     )
     def test_buffer_source_readinto1_variance(
         self, original, level, source_read_size, read_sizes
     ):
 
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         with cctx.stream_reader(
             original, size=len(original), read_size=source_read_size
         ) as reader:
             chunks = []
             while True:
                 read_size = read_sizes.draw(strategies.integers(1, 16384))
                 b = bytearray(read_size)
                 count = reader.readinto1(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_stream_writer_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         write_size=strategies.integers(min_value=1, max_value=1048576),
     )
     def test_write_size_variance(self, original, level, write_size):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         b = NonClosingBytesIO()
         with cctx.stream_writer(
             b, size=len(original), write_size=write_size
         ) as compressor:
             compressor.write(original)
 
         self.assertEqual(b.getvalue(), ref_frame)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_copy_stream_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         read_size=strategies.integers(min_value=1, max_value=1048576),
         write_size=strategies.integers(min_value=1, max_value=1048576),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         source = io.BytesIO(original)
         dest = io.BytesIO()
 
         cctx.copy_stream(
-            source, dest, size=len(original), read_size=read_size, write_size=write_size
+            source,
+            dest,
+            size=len(original),
+            read_size=read_size,
+            write_size=write_size,
         )
 
         self.assertEqual(dest.getvalue(), ref_frame)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_compressobj_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         chunk_sizes=strategies.data(),
     )
     def test_random_input_sizes(self, original, level, chunk_sizes):
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         cobj = cctx.compressobj(size=len(original))
 
         chunks = []
         i = 0
         while True:
             chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
             source = original[i : i + chunk_size]
             if not source:
                 break
 
             chunks.append(cobj.compress(source))
             i += chunk_size
 
         chunks.append(cobj.flush())
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         chunk_sizes=strategies.data(),
         flushes=strategies.data(),
     )
     def test_flush_block(self, original, level, chunk_sizes, flushes):
         cctx = zstd.ZstdCompressor(level=level)
         cobj = cctx.compressobj()
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
 
         compressed_chunks = []
         decompressed_chunks = []
         i = 0
         while True:
             input_size = chunk_sizes.draw(strategies.integers(1, 4096))
             source = original[i : i + input_size]
             if not source:
                 break
 
             i += input_size
 
             chunk = cobj.compress(source)
             compressed_chunks.append(chunk)
             decompressed_chunks.append(dobj.decompress(chunk))
 
             if not flushes.draw(strategies.booleans()):
                 continue
 
             chunk = cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK)
             compressed_chunks.append(chunk)
             decompressed_chunks.append(dobj.decompress(chunk))
 
             self.assertEqual(b"".join(decompressed_chunks), original[0:i])
 
         chunk = cobj.flush(zstd.COMPRESSOBJ_FLUSH_FINISH)
         compressed_chunks.append(chunk)
         decompressed_chunks.append(dobj.decompress(chunk))
 
         self.assertEqual(
-            dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+            dctx.decompress(
+                b"".join(compressed_chunks), max_output_size=len(original)
+            ),
             original,
         )
         self.assertEqual(b"".join(decompressed_chunks), original)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_read_to_iter_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         read_size=strategies.integers(min_value=1, max_value=4096),
         write_size=strategies.integers(min_value=1, max_value=4096),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         refcctx = zstd.ZstdCompressor(level=level)
         ref_frame = refcctx.compress(original)
 
         source = io.BytesIO(original)
 
         cctx = zstd.ZstdCompressor(level=level)
         chunks = list(
             cctx.read_to_iter(
-                source, size=len(original), read_size=read_size, write_size=write_size
+                source,
+                size=len(original),
+                read_size=read_size,
+                write_size=write_size,
             )
         )
 
         self.assertEqual(b"".join(chunks), ref_frame)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 class TestCompressor_multi_compress_to_buffer_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.lists(
-            strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+            strategies.sampled_from(random_input_data()),
+            min_size=1,
+            max_size=1024,
         ),
         threads=strategies.integers(min_value=1, max_value=8),
         use_dict=strategies.booleans(),
     )
     def test_data_equivalence(self, original, threads, use_dict):
         kwargs = {}
 
         # Use a content dictionary because it is cheap to create.
         if use_dict:
             kwargs["dict_data"] = zstd.ZstdCompressionDict(original[0])
 
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True, **kwargs)
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         result = cctx.multi_compress_to_buffer(original, threads=-1)
 
         self.assertEqual(len(result), len(original))
 
         # The frame produced via the batch APIs may not be bit identical to that
         # produced by compress() because compression parameters are adjusted
         # from the first input in batch mode. So the only thing we can do is
         # verify the decompressed data matches the input.
         dctx = zstd.ZstdDecompressor(**kwargs)
 
         for i, frame in enumerate(result):
             self.assertEqual(dctx.decompress(frame), original[i])
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestCompressor_chunker_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         chunk_size=strategies.integers(min_value=1, max_value=32 * 1048576),
         input_sizes=strategies.data(),
     )
     def test_random_input_sizes(self, original, level, chunk_size, input_sizes):
         cctx = zstd.ZstdCompressor(level=level)
         chunker = cctx.chunker(chunk_size=chunk_size)
 
         chunks = []
         i = 0
         while True:
             input_size = input_sizes.draw(strategies.integers(1, 4096))
             source = original[i : i + input_size]
             if not source:
                 break
 
             chunks.extend(chunker.compress(source))
             i += input_size
 
         chunks.extend(chunker.finish())
 
         dctx = zstd.ZstdDecompressor()
 
         self.assertEqual(
-            dctx.decompress(b"".join(chunks), max_output_size=len(original)), original
+            dctx.decompress(b"".join(chunks), max_output_size=len(original)),
+            original,
         )
 
         self.assertTrue(all(len(chunk) == chunk_size for chunk in chunks[:-1]))
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         chunk_size=strategies.integers(min_value=1, max_value=32 * 1048576),
         input_sizes=strategies.data(),
         flushes=strategies.data(),
     )
-    def test_flush_block(self, original, level, chunk_size, input_sizes, flushes):
+    def test_flush_block(
+        self, original, level, chunk_size, input_sizes, flushes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         chunker = cctx.chunker(chunk_size=chunk_size)
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
 
         compressed_chunks = []
         decompressed_chunks = []
         i = 0
         while True:
             input_size = input_sizes.draw(strategies.integers(1, 4096))
             source = original[i : i + input_size]
             if not source:
                 break
 
             i += input_size
 
             chunks = list(chunker.compress(source))
             compressed_chunks.extend(chunks)
             decompressed_chunks.append(dobj.decompress(b"".join(chunks)))
 
             if not flushes.draw(strategies.booleans()):
                 continue
 
             chunks = list(chunker.flush())
             compressed_chunks.extend(chunks)
             decompressed_chunks.append(dobj.decompress(b"".join(chunks)))
 
             self.assertEqual(b"".join(decompressed_chunks), original[0:i])
 
         chunks = list(chunker.finish())
         compressed_chunks.extend(chunks)
         decompressed_chunks.append(dobj.decompress(b"".join(chunks)))
 
         self.assertEqual(
-            dctx.decompress(b"".join(compressed_chunks), max_output_size=len(original)),
+            dctx.decompress(
+                b"".join(compressed_chunks), max_output_size=len(original)
+            ),
             original,
         )
         self.assertEqual(b"".join(decompressed_chunks), original)

contrib/python-zstandard/tests/test_data_structures.py

@@ -1,241 +1,255 b''
 import sys
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     TestCase,
 )
 
 
 @make_cffi
 class TestCompressionParameters(TestCase):
     def test_bounds(self):
         zstd.ZstdCompressionParameters(
             window_log=zstd.WINDOWLOG_MIN,
             chain_log=zstd.CHAINLOG_MIN,
             hash_log=zstd.HASHLOG_MIN,
             search_log=zstd.SEARCHLOG_MIN,
             min_match=zstd.MINMATCH_MIN + 1,
             target_length=zstd.TARGETLENGTH_MIN,
             strategy=zstd.STRATEGY_FAST,
         )
 
         zstd.ZstdCompressionParameters(
             window_log=zstd.WINDOWLOG_MAX,
             chain_log=zstd.CHAINLOG_MAX,
             hash_log=zstd.HASHLOG_MAX,
             search_log=zstd.SEARCHLOG_MAX,
             min_match=zstd.MINMATCH_MAX - 1,
             target_length=zstd.TARGETLENGTH_MAX,
             strategy=zstd.STRATEGY_BTULTRA2,
         )
 
     def test_from_level(self):
         p = zstd.ZstdCompressionParameters.from_level(1)
         self.assertIsInstance(p, zstd.CompressionParameters)
 
         self.assertEqual(p.window_log, 19)
 
         p = zstd.ZstdCompressionParameters.from_level(-4)
         self.assertEqual(p.window_log, 19)
 
     def test_members(self):
         p = zstd.ZstdCompressionParameters(
             window_log=10,
             chain_log=6,
             hash_log=7,
             search_log=4,
             min_match=5,
             target_length=8,
             strategy=1,
         )
         self.assertEqual(p.window_log, 10)
         self.assertEqual(p.chain_log, 6)
         self.assertEqual(p.hash_log, 7)
         self.assertEqual(p.search_log, 4)
         self.assertEqual(p.min_match, 5)
         self.assertEqual(p.target_length, 8)
         self.assertEqual(p.compression_strategy, 1)
 
         p = zstd.ZstdCompressionParameters(compression_level=2)
         self.assertEqual(p.compression_level, 2)
 
         p = zstd.ZstdCompressionParameters(threads=4)
         self.assertEqual(p.threads, 4)
 
-        p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576, overlap_log=6)
+        p = zstd.ZstdCompressionParameters(
+            threads=2, job_size=1048576, overlap_log=6
+        )
         self.assertEqual(p.threads, 2)
         self.assertEqual(p.job_size, 1048576)
         self.assertEqual(p.overlap_log, 6)
         self.assertEqual(p.overlap_size_log, 6)
 
         p = zstd.ZstdCompressionParameters(compression_level=-1)
         self.assertEqual(p.compression_level, -1)
 
         p = zstd.ZstdCompressionParameters(compression_level=-2)
         self.assertEqual(p.compression_level, -2)
 
         p = zstd.ZstdCompressionParameters(force_max_window=True)
         self.assertEqual(p.force_max_window, 1)
 
         p = zstd.ZstdCompressionParameters(enable_ldm=True)
         self.assertEqual(p.enable_ldm, 1)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_log=7)
         self.assertEqual(p.ldm_hash_log, 7)
 
         p = zstd.ZstdCompressionParameters(ldm_min_match=6)
         self.assertEqual(p.ldm_min_match, 6)
 
         p = zstd.ZstdCompressionParameters(ldm_bucket_size_log=7)
         self.assertEqual(p.ldm_bucket_size_log, 7)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
         self.assertEqual(p.ldm_hash_rate_log, 8)
 
     def test_estimated_compression_context_size(self):
         p = zstd.ZstdCompressionParameters(
             window_log=20,
             chain_log=16,
             hash_log=17,
             search_log=1,
             min_match=5,
             target_length=16,
             strategy=zstd.STRATEGY_DFAST,
         )
 
         # 32-bit has slightly different values from 64-bit.
         self.assertAlmostEqual(
             p.estimated_compression_context_size(), 1294464, delta=400
         )
 
     def test_strategy(self):
         with self.assertRaisesRegex(
             ValueError, "cannot specify both compression_strategy"
         ):
             zstd.ZstdCompressionParameters(strategy=0, compression_strategy=0)
 
         p = zstd.ZstdCompressionParameters(strategy=2)
         self.assertEqual(p.compression_strategy, 2)
 
         p = zstd.ZstdCompressionParameters(strategy=3)
         self.assertEqual(p.compression_strategy, 3)
 
     def test_ldm_hash_rate_log(self):
         with self.assertRaisesRegex(
             ValueError, "cannot specify both ldm_hash_rate_log"
         ):
-            zstd.ZstdCompressionParameters(ldm_hash_rate_log=8, ldm_hash_every_log=4)
+            zstd.ZstdCompressionParameters(
+                ldm_hash_rate_log=8, ldm_hash_every_log=4
+            )
 
         p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
 
         p = zstd.ZstdCompressionParameters(ldm_hash_every_log=16)
         self.assertEqual(p.ldm_hash_every_log, 16)
 
     def test_overlap_log(self):
-        with self.assertRaisesRegex(ValueError, "cannot specify both overlap_log"):
+        with self.assertRaisesRegex(
+            ValueError, "cannot specify both overlap_log"
+        ):
             zstd.ZstdCompressionParameters(overlap_log=1, overlap_size_log=9)
 
         p = zstd.ZstdCompressionParameters(overlap_log=2)
         self.assertEqual(p.overlap_log, 2)
         self.assertEqual(p.overlap_size_log, 2)
 
         p = zstd.ZstdCompressionParameters(overlap_size_log=4)
         self.assertEqual(p.overlap_log, 4)
         self.assertEqual(p.overlap_size_log, 4)
 
 
 @make_cffi
 class TestFrameParameters(TestCase):
     def test_invalid_type(self):
         with self.assertRaises(TypeError):
             zstd.get_frame_parameters(None)
 
         # Python 3 doesn't appear to convert unicode to Py_buffer.
         if sys.version_info[0] >= 3:
             with self.assertRaises(TypeError):
                 zstd.get_frame_parameters(u"foobarbaz")
         else:
             # CPython will convert unicode to Py_buffer. But CFFI won't.
             if zstd.backend == "cffi":
                 with self.assertRaises(TypeError):
                     zstd.get_frame_parameters(u"foobarbaz")
             else:
                 with self.assertRaises(zstd.ZstdError):
                     zstd.get_frame_parameters(u"foobarbaz")
 
     def test_invalid_input_sizes(self):
-        with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "not enough data for frame"
+        ):
             zstd.get_frame_parameters(b"")
 
-        with self.assertRaisesRegex(zstd.ZstdError, "not enough data for frame"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "not enough data for frame"
+        ):
             zstd.get_frame_parameters(zstd.FRAME_HEADER)
 
     def test_invalid_frame(self):
         with self.assertRaisesRegex(zstd.ZstdError, "Unknown frame descriptor"):
             zstd.get_frame_parameters(b"foobarbaz")
 
     def test_attributes(self):
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x00\x00")
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Lowest 2 bits indicate a dictionary and length. Here, the dict id is 1 byte.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x01\x00\xff")
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 255)
         self.assertFalse(params.has_checksum)
 
         # Lowest 3rd bit indicates if checksum is present.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x04\x00")
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
 
         # Upper 2 bits indicate content size.
-        params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x40\x00\xff\x00")
+        params = zstd.get_frame_parameters(
+            zstd.FRAME_HEADER + b"\x40\x00\xff\x00"
+        )
         self.assertEqual(params.content_size, 511)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Window descriptor is 2nd byte after frame header.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x00\x40")
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Set multiple things.
-        params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00")
+        params = zstd.get_frame_parameters(
+            zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00"
+        )
         self.assertEqual(params.content_size, 272)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 15)
         self.assertTrue(params.has_checksum)
 
     def test_input_types(self):
         v = zstd.FRAME_HEADER + b"\x00\x00"
 
         mutable_array = bytearray(len(v))
         mutable_array[:] = v
 
         sources = [
             memoryview(v),
             bytearray(v),
             mutable_array,
         ]
 
         for source in sources:
             params = zstd.get_frame_parameters(source)
             self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
             self.assertEqual(params.window_size, 1024)
             self.assertEqual(params.dict_id, 0)
             self.assertFalse(params.has_checksum)

contrib/python-zstandard/tests/test_data_structures_fuzzing.py

@@ -1,105 +1,121 b''
 import io
 import os
 import sys
 import unittest
 
 try:
     import hypothesis
     import hypothesis.strategies as strategies
 except ImportError:
     raise unittest.SkipTest("hypothesis not available")
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     TestCase,
 )
 
 
 s_windowlog = strategies.integers(
     min_value=zstd.WINDOWLOG_MIN, max_value=zstd.WINDOWLOG_MAX
 )
 s_chainlog = strategies.integers(
     min_value=zstd.CHAINLOG_MIN, max_value=zstd.CHAINLOG_MAX
 )
-s_hashlog = strategies.integers(min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX)
+s_hashlog = strategies.integers(
+    min_value=zstd.HASHLOG_MIN, max_value=zstd.HASHLOG_MAX
+)
 s_searchlog = strategies.integers(
     min_value=zstd.SEARCHLOG_MIN, max_value=zstd.SEARCHLOG_MAX
 )
 s_minmatch = strategies.integers(
     min_value=zstd.MINMATCH_MIN, max_value=zstd.MINMATCH_MAX
 )
 s_targetlength = strategies.integers(
     min_value=zstd.TARGETLENGTH_MIN, max_value=zstd.TARGETLENGTH_MAX
 )
 s_strategy = strategies.sampled_from(
     (
         zstd.STRATEGY_FAST,
         zstd.STRATEGY_DFAST,
         zstd.STRATEGY_GREEDY,
         zstd.STRATEGY_LAZY,
         zstd.STRATEGY_LAZY2,
         zstd.STRATEGY_BTLAZY2,
         zstd.STRATEGY_BTOPT,
         zstd.STRATEGY_BTULTRA,
         zstd.STRATEGY_BTULTRA2,
     )
 )
 
 
 @make_cffi
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 class TestCompressionParametersHypothesis(TestCase):
     @hypothesis.given(
         s_windowlog,
         s_chainlog,
         s_hashlog,
         s_searchlog,
         s_minmatch,
         s_targetlength,
         s_strategy,
     )
     def test_valid_init(
-        self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+        self,
+        windowlog,
+        chainlog,
+        hashlog,
+        searchlog,
+        minmatch,
+        targetlength,
+        strategy,
     ):
         zstd.ZstdCompressionParameters(
             window_log=windowlog,
             chain_log=chainlog,
             hash_log=hashlog,
             search_log=searchlog,
             min_match=minmatch,
             target_length=targetlength,
             strategy=strategy,
         )
 
     @hypothesis.given(
         s_windowlog,
         s_chainlog,
         s_hashlog,
         s_searchlog,
         s_minmatch,
         s_targetlength,
         s_strategy,
     )
     def test_estimated_compression_context_size(
-        self, windowlog, chainlog, hashlog, searchlog, minmatch, targetlength, strategy
+        self,
+        windowlog,
+        chainlog,
+        hashlog,
+        searchlog,
+        minmatch,
+        targetlength,
+        strategy,
     ):
         if minmatch == zstd.MINMATCH_MIN and strategy in (
             zstd.STRATEGY_FAST,
             zstd.STRATEGY_GREEDY,
         ):
             minmatch += 1
         elif minmatch == zstd.MINMATCH_MAX and strategy != zstd.STRATEGY_FAST:
             minmatch -= 1
 
         p = zstd.ZstdCompressionParameters(
             window_log=windowlog,
             chain_log=chainlog,
             hash_log=hashlog,
             search_log=searchlog,
             min_match=minmatch,
             target_length=targetlength,
             strategy=strategy,
         )
         size = p.estimated_compression_context_size()

contrib/python-zstandard/tests/test_decompressor.py

@@ -1,1670 +1,1714 b''
 import io
 import os
 import random
 import struct
 import sys
 import tempfile
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     generate_samples,
     make_cffi,
     NonClosingBytesIO,
     OpCountingBytesIO,
     TestCase,
 )
 
 
 if sys.version_info[0] >= 3:
     next = lambda it: it.__next__()
 else:
     next = lambda it: it.next()
 
 
 @make_cffi
 class TestFrameHeaderSize(TestCase):
     def test_empty(self):
         with self.assertRaisesRegex(
             zstd.ZstdError,
             "could not determine frame header size: Src size " "is incorrect",
         ):
             zstd.frame_header_size(b"")
 
     def test_too_small(self):
         with self.assertRaisesRegex(
             zstd.ZstdError,
             "could not determine frame header size: Src size " "is incorrect",
         ):
             zstd.frame_header_size(b"foob")
 
     def test_basic(self):
         # It doesn't matter that it isn't a valid frame.
         self.assertEqual(zstd.frame_header_size(b"long enough but no magic"), 6)
 
 
 @make_cffi
 class TestFrameContentSize(TestCase):
     def test_empty(self):
         with self.assertRaisesRegex(
             zstd.ZstdError, "error when determining content size"
         ):
             zstd.frame_content_size(b"")
 
     def test_too_small(self):
         with self.assertRaisesRegex(
             zstd.ZstdError, "error when determining content size"
         ):
             zstd.frame_content_size(b"foob")
 
     def test_bad_frame(self):
         with self.assertRaisesRegex(
             zstd.ZstdError, "error when determining content size"
         ):
             zstd.frame_content_size(b"invalid frame header")
 
     def test_unknown(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         frame = cctx.compress(b"foobar")
 
         self.assertEqual(zstd.frame_content_size(frame), -1)
 
     def test_empty(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b"")
 
         self.assertEqual(zstd.frame_content_size(frame), 0)
 
     def test_basic(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b"foobar")
 
         self.assertEqual(zstd.frame_content_size(frame), 6)
 
 
 @make_cffi
 class TestDecompressor(TestCase):
     def test_memory_size(self):
         dctx = zstd.ZstdDecompressor()
 
         self.assertGreater(dctx.memory_size(), 100)
 
 
 @make_cffi
 class TestDecompressor_decompress(TestCase):
     def test_empty_input(self):
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegex(
             zstd.ZstdError, "error determining content size from frame header"
         ):
             dctx.decompress(b"")
 
     def test_invalid_input(self):
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegex(
             zstd.ZstdError, "error determining content size from frame header"
         ):
             dctx.decompress(b"foobar")
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor(level=1)
         compressed = cctx.compress(b"foo")
 
         mutable_array = bytearray(len(compressed))
         mutable_array[:] = compressed
 
         sources = [
             memoryview(compressed),
             bytearray(compressed),
             mutable_array,
         ]
 
         dctx = zstd.ZstdDecompressor()
         for source in sources:
             self.assertEqual(dctx.decompress(source), b"foo")
 
     def test_no_content_size_in_frame(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         compressed = cctx.compress(b"foobar")
 
         dctx = zstd.ZstdDecompressor()
         with self.assertRaisesRegex(
             zstd.ZstdError, "could not determine content size in frame header"
         ):
             dctx.decompress(compressed)
 
     def test_content_size_present(self):
         cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(b"foobar")
 
         dctx = zstd.ZstdDecompressor()
         decompressed = dctx.decompress(compressed)
         self.assertEqual(decompressed, b"foobar")
 
     def test_empty_roundtrip(self):
         cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(b"")
 
         dctx = zstd.ZstdDecompressor()
         decompressed = dctx.decompress(compressed)
 
         self.assertEqual(decompressed, b"")
 
     def test_max_output_size(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         source = b"foobar" * 256
         compressed = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         # Will fit into buffer exactly the size of input.
         decompressed = dctx.decompress(compressed, max_output_size=len(source))
         self.assertEqual(decompressed, source)
 
         # Input size - 1 fails
         with self.assertRaisesRegex(
             zstd.ZstdError, "decompression error: did not decompress full frame"
         ):
             dctx.decompress(compressed, max_output_size=len(source) - 1)
 
         # Input size + 1 works
-        decompressed = dctx.decompress(compressed, max_output_size=len(source) + 1)
+        decompressed = dctx.decompress(
+            compressed, max_output_size=len(source) + 1
+        )
         self.assertEqual(decompressed, source)
 
         # A much larger buffer works.
-        decompressed = dctx.decompress(compressed, max_output_size=len(source) * 64)
+        decompressed = dctx.decompress(
+            compressed, max_output_size=len(source) * 64
+        )
         self.assertEqual(decompressed, source)
 
     def test_stupidly_large_output_buffer(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         compressed = cctx.compress(b"foobar" * 256)
         dctx = zstd.ZstdDecompressor()
 
         # Will get OverflowError on some Python distributions that can't
         # handle really large integers.
         with self.assertRaises((MemoryError, OverflowError)):
             dctx.decompress(compressed, max_output_size=2 ** 62)
 
     def test_dictionary(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         orig = b"foobar" * 16384
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         compressed = cctx.compress(orig)
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
         decompressed = dctx.decompress(compressed)
 
         self.assertEqual(decompressed, orig)
 
     def test_dictionary_multiple(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         sources = (b"foobar" * 8192, b"foo" * 8192, b"bar" * 8192)
         compressed = []
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         for source in sources:
             compressed.append(cctx.compress(source))
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
         for i in range(len(sources)):
             decompressed = dctx.decompress(compressed[i])
             self.assertEqual(decompressed, sources[i])
 
     def test_max_window_size(self):
         with open(__file__, "rb") as fh:
             source = fh.read()
 
         # If we write a content size, the decompressor engages single pass
         # mode and the window size doesn't come into play.
         cctx = zstd.ZstdCompressor(write_content_size=False)
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor(max_window_size=2 ** zstd.WINDOWLOG_MIN)
 
         with self.assertRaisesRegex(
-            zstd.ZstdError, "decompression error: Frame requires too much memory"
+            zstd.ZstdError,
+            "decompression error: Frame requires too much memory",
         ):
             dctx.decompress(frame, max_output_size=len(source))
 
 
 @make_cffi
 class TestDecompressor_copy_stream(TestCase):
     def test_no_read(self):
         source = object()
         dest = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor()
         with self.assertRaises(ValueError):
             dctx.copy_stream(source, dest)
 
     def test_no_write(self):
         source = io.BytesIO()
         dest = object()
 
         dctx = zstd.ZstdDecompressor()
         with self.assertRaises(ValueError):
             dctx.copy_stream(source, dest)
 
     def test_empty(self):
         source = io.BytesIO()
         dest = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor()
         # TODO should this raise an error?
         r, w = dctx.copy_stream(source, dest)
 
         self.assertEqual(r, 0)
         self.assertEqual(w, 0)
         self.assertEqual(dest.getvalue(), b"")
 
     def test_large_data(self):
         source = io.BytesIO()
         for i in range(255):
             source.write(struct.Struct(">B").pack(i) * 16384)
         source.seek(0)
 
         compressed = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         cctx.copy_stream(source, compressed)
 
         compressed.seek(0)
         dest = io.BytesIO()
         dctx = zstd.ZstdDecompressor()
         r, w = dctx.copy_stream(compressed, dest)
 
         self.assertEqual(r, len(compressed.getvalue()))
         self.assertEqual(w, len(source.getvalue()))
 
     def test_read_write_size(self):
-        source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+        source = OpCountingBytesIO(
+            zstd.ZstdCompressor().compress(b"foobarfoobar")
+        )
 
         dest = OpCountingBytesIO()
         dctx = zstd.ZstdDecompressor()
         r, w = dctx.copy_stream(source, dest, read_size=1, write_size=1)
 
         self.assertEqual(r, len(source.getvalue()))
         self.assertEqual(w, len(b"foobarfoobar"))
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
         self.assertEqual(dest._write_count, len(dest.getvalue()))
 
 
 @make_cffi
 class TestDecompressor_stream_reader(TestCase):
     def test_context_manager(self):
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b"foo") as reader:
-            with self.assertRaisesRegex(ValueError, "cannot __enter__ multiple times"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot __enter__ multiple times"
+            ):
                 with reader as reader2:
                     pass
 
     def test_not_implemented(self):
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b"foo") as reader:
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readline()
 
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readlines()
 
             with self.assertRaises(io.UnsupportedOperation):
                 iter(reader)
 
             with self.assertRaises(io.UnsupportedOperation):
                 next(reader)
 
             with self.assertRaises(io.UnsupportedOperation):
                 reader.write(b"foo")
 
             with self.assertRaises(io.UnsupportedOperation):
                 reader.writelines([])
 
     def test_constant_methods(self):
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b"foo") as reader:
             self.assertFalse(reader.closed)
             self.assertTrue(reader.readable())
             self.assertFalse(reader.writable())
             self.assertTrue(reader.seekable())
             self.assertFalse(reader.isatty())
             self.assertFalse(reader.closed)
             self.assertIsNone(reader.flush())
             self.assertFalse(reader.closed)
 
         self.assertTrue(reader.closed)
 
     def test_read_closed(self):
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b"foo") as reader:
             reader.close()
             self.assertTrue(reader.closed)
             with self.assertRaisesRegex(ValueError, "stream is closed"):
                 reader.read(1)
 
     def test_read_sizes(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(foo) as reader:
             with self.assertRaisesRegex(
                 ValueError, "cannot read negative amounts less than -1"
             ):
                 reader.read(-2)
 
             self.assertEqual(reader.read(0), b"")
             self.assertEqual(reader.read(), b"foo")
 
     def test_read_buffer(self):
         cctx = zstd.ZstdCompressor()
 
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame) as reader:
             self.assertEqual(reader.tell(), 0)
 
             # We should get entire frame in one read.
             result = reader.read(8192)
             self.assertEqual(result, source)
             self.assertEqual(reader.tell(), len(source))
 
             # Read after EOF should return empty bytes.
             self.assertEqual(reader.read(1), b"")
             self.assertEqual(reader.tell(), len(result))
 
         self.assertTrue(reader.closed)
 
     def test_read_buffer_small_chunks(self):
         cctx = zstd.ZstdCompressor()
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         chunks = []
 
         with dctx.stream_reader(frame, read_size=1) as reader:
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b"".join(chunks), source)
 
     def test_read_stream(self):
         cctx = zstd.ZstdCompressor()
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_reader(io.BytesIO(frame)) as reader:
             self.assertEqual(reader.tell(), 0)
 
             chunk = reader.read(8192)
             self.assertEqual(chunk, source)
             self.assertEqual(reader.tell(), len(source))
             self.assertEqual(reader.read(1), b"")
             self.assertEqual(reader.tell(), len(source))
             self.assertFalse(reader.closed)
 
         self.assertTrue(reader.closed)
 
     def test_read_stream_small_chunks(self):
         cctx = zstd.ZstdCompressor()
         source = b"".join([b"foo" * 60, b"bar" * 60, b"baz" * 60])
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         chunks = []
 
         with dctx.stream_reader(io.BytesIO(frame), read_size=1) as reader:
             while True:
                 chunk = reader.read(1)
                 if not chunk:
                     break
 
                 chunks.append(chunk)
                 self.assertEqual(reader.tell(), sum(map(len, chunks)))
 
         self.assertEqual(b"".join(chunks), source)
 
     def test_read_after_exit(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b"foo" * 60)
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame) as reader:
             while reader.read(16):
                 pass
 
         self.assertTrue(reader.closed)
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             reader.read(10)
 
     def test_illegal_seeks(self):
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(b"foo" * 60)
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame) as reader:
-            with self.assertRaisesRegex(ValueError, "cannot seek to negative position"):
+            with self.assertRaisesRegex(
+                ValueError, "cannot seek to negative position"
+            ):
                 reader.seek(-1, os.SEEK_SET)
 
             reader.read(1)
 
             with self.assertRaisesRegex(
                 ValueError, "cannot seek zstd decompression stream backwards"
             ):
                 reader.seek(0, os.SEEK_SET)
 
             with self.assertRaisesRegex(
                 ValueError, "cannot seek zstd decompression stream backwards"
             ):
                 reader.seek(-1, os.SEEK_CUR)
 
             with self.assertRaisesRegex(
-                ValueError, "zstd decompression streams cannot be seeked with SEEK_END"
+                ValueError,
+                "zstd decompression streams cannot be seeked with SEEK_END",
             ):
                 reader.seek(0, os.SEEK_END)
 
             reader.close()
 
             with self.assertRaisesRegex(ValueError, "stream is closed"):
                 reader.seek(4, os.SEEK_SET)
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             reader.seek(0)
 
     def test_seek(self):
         source = b"foobar" * 60
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame) as reader:
             reader.seek(3)
             self.assertEqual(reader.read(3), b"bar")
 
             reader.seek(4, os.SEEK_CUR)
             self.assertEqual(reader.read(2), b"ar")
 
     def test_no_context_manager(self):
         source = b"foobar" * 60
         cctx = zstd.ZstdCompressor()
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         reader = dctx.stream_reader(frame)
 
         self.assertEqual(reader.read(6), b"foobar")
         self.assertEqual(reader.read(18), b"foobar" * 3)
         self.assertFalse(reader.closed)
 
         # Calling close prevents subsequent use.
         reader.close()
         self.assertTrue(reader.closed)
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             reader.read(6)
 
     def test_read_after_error(self):
         source = io.BytesIO(b"")
         dctx = zstd.ZstdDecompressor()
 
         reader = dctx.stream_reader(source)
 
         with reader:
             reader.read(0)
 
         with reader:
             with self.assertRaisesRegex(ValueError, "stream is closed"):
                 reader.read(100)
 
     def test_partial_read(self):
         # Inspired by https://github.com/indygreg/python-zstandard/issues/71.
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor()
         writer = cctx.stream_writer(buffer)
         writer.write(bytearray(os.urandom(1000000)))
         writer.flush(zstd.FLUSH_FRAME)
         buffer.seek(0)
 
         dctx = zstd.ZstdDecompressor()
         reader = dctx.stream_reader(buffer)
 
         while True:
             chunk = reader.read(8192)
             if not chunk:
                 break
 
     def test_read_multiple_frames(self):
         cctx = zstd.ZstdCompressor()
         source = io.BytesIO()
         writer = cctx.stream_writer(source)
         writer.write(b"foo")
         writer.flush(zstd.FLUSH_FRAME)
         writer.write(b"bar")
         writer.flush(zstd.FLUSH_FRAME)
 
         dctx = zstd.ZstdDecompressor()
 
         reader = dctx.stream_reader(source.getvalue())
         self.assertEqual(reader.read(2), b"fo")
         self.assertEqual(reader.read(2), b"o")
         self.assertEqual(reader.read(2), b"ba")
         self.assertEqual(reader.read(2), b"r")
 
         source.seek(0)
         reader = dctx.stream_reader(source)
         self.assertEqual(reader.read(2), b"fo")
         self.assertEqual(reader.read(2), b"o")
         self.assertEqual(reader.read(2), b"ba")
         self.assertEqual(reader.read(2), b"r")
 
         reader = dctx.stream_reader(source.getvalue())
         self.assertEqual(reader.read(3), b"foo")
         self.assertEqual(reader.read(3), b"bar")
 
         source.seek(0)
         reader = dctx.stream_reader(source)
         self.assertEqual(reader.read(3), b"foo")
         self.assertEqual(reader.read(3), b"bar")
 
         reader = dctx.stream_reader(source.getvalue())
         self.assertEqual(reader.read(4), b"foo")
         self.assertEqual(reader.read(4), b"bar")
 
         source.seek(0)
         reader = dctx.stream_reader(source)
         self.assertEqual(reader.read(4), b"foo")
         self.assertEqual(reader.read(4), b"bar")
 
         reader = dctx.stream_reader(source.getvalue())
         self.assertEqual(reader.read(128), b"foo")
         self.assertEqual(reader.read(128), b"bar")
 
         source.seek(0)
         reader = dctx.stream_reader(source)
         self.assertEqual(reader.read(128), b"foo")
         self.assertEqual(reader.read(128), b"bar")
 
         # Now tests for reads spanning frames.
         reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
         self.assertEqual(reader.read(3), b"foo")
         self.assertEqual(reader.read(3), b"bar")
 
         source.seek(0)
         reader = dctx.stream_reader(source, read_across_frames=True)
         self.assertEqual(reader.read(3), b"foo")
         self.assertEqual(reader.read(3), b"bar")
 
         reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
         self.assertEqual(reader.read(6), b"foobar")
 
         source.seek(0)
         reader = dctx.stream_reader(source, read_across_frames=True)
         self.assertEqual(reader.read(6), b"foobar")
 
         reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
         self.assertEqual(reader.read(7), b"foobar")
 
         source.seek(0)
         reader = dctx.stream_reader(source, read_across_frames=True)
         self.assertEqual(reader.read(7), b"foobar")
 
         reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
         self.assertEqual(reader.read(128), b"foobar")
 
         source.seek(0)
         reader = dctx.stream_reader(source, read_across_frames=True)
         self.assertEqual(reader.read(128), b"foobar")
 
     def test_readinto(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
 
         # Attempting to readinto() a non-writable buffer fails.
         # The exact exception varies based on the backend.
         reader = dctx.stream_reader(foo)
         with self.assertRaises(Exception):
             reader.readinto(b"foobar")
 
         # readinto() with sufficiently large destination.
         b = bytearray(1024)
         reader = dctx.stream_reader(foo)
         self.assertEqual(reader.readinto(b), 3)
         self.assertEqual(b[0:3], b"foo")
         self.assertEqual(reader.readinto(b), 0)
         self.assertEqual(b[0:3], b"foo")
 
         # readinto() with small reads.
         b = bytearray(1024)
         reader = dctx.stream_reader(foo, read_size=1)
         self.assertEqual(reader.readinto(b), 3)
         self.assertEqual(b[0:3], b"foo")
 
         # Too small destination buffer.
         b = bytearray(2)
         reader = dctx.stream_reader(foo)
         self.assertEqual(reader.readinto(b), 2)
         self.assertEqual(b[:], b"fo")
 
     def test_readinto1(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
 
         reader = dctx.stream_reader(foo)
         with self.assertRaises(Exception):
             reader.readinto1(b"foobar")
 
         # Sufficiently large destination.
         b = bytearray(1024)
         reader = dctx.stream_reader(foo)
         self.assertEqual(reader.readinto1(b), 3)
         self.assertEqual(b[0:3], b"foo")
         self.assertEqual(reader.readinto1(b), 0)
         self.assertEqual(b[0:3], b"foo")
 
         # readinto() with small reads.
         b = bytearray(1024)
         reader = dctx.stream_reader(foo, read_size=1)
         self.assertEqual(reader.readinto1(b), 3)
         self.assertEqual(b[0:3], b"foo")
 
         # Too small destination buffer.
         b = bytearray(2)
         reader = dctx.stream_reader(foo)
         self.assertEqual(reader.readinto1(b), 2)
         self.assertEqual(b[:], b"fo")
 
     def test_readall(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
         reader = dctx.stream_reader(foo)
 
         self.assertEqual(reader.readall(), b"foo")
 
     def test_read1(self):
         cctx = zstd.ZstdCompressor()
         foo = cctx.compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
 
         b = OpCountingBytesIO(foo)
         reader = dctx.stream_reader(b)
 
         self.assertEqual(reader.read1(), b"foo")
         self.assertEqual(b._read_count, 1)
 
         b = OpCountingBytesIO(foo)
         reader = dctx.stream_reader(b)
 
         self.assertEqual(reader.read1(0), b"")
         self.assertEqual(reader.read1(2), b"fo")
         self.assertEqual(b._read_count, 1)
         self.assertEqual(reader.read1(1), b"o")
         self.assertEqual(b._read_count, 1)
         self.assertEqual(reader.read1(1), b"")
         self.assertEqual(b._read_count, 2)
 
     def test_read_lines(self):
         cctx = zstd.ZstdCompressor()
-        source = b"\n".join(("line %d" % i).encode("ascii") for i in range(1024))
+        source = b"\n".join(
+            ("line %d" % i).encode("ascii") for i in range(1024)
+        )
 
         frame = cctx.compress(source)
 
         dctx = zstd.ZstdDecompressor()
         reader = dctx.stream_reader(frame)
         tr = io.TextIOWrapper(reader, encoding="utf-8")
 
         lines = []
         for line in tr:
             lines.append(line.encode("utf-8"))
 
         self.assertEqual(len(lines), 1024)
         self.assertEqual(b"".join(lines), source)
 
         reader = dctx.stream_reader(frame)
         tr = io.TextIOWrapper(reader, encoding="utf-8")
 
         lines = tr.readlines()
         self.assertEqual(len(lines), 1024)
         self.assertEqual("".join(lines).encode("utf-8"), source)
 
         reader = dctx.stream_reader(frame)
         tr = io.TextIOWrapper(reader, encoding="utf-8")
 
         lines = []
         while True:
             line = tr.readline()
             if not line:
                 break
 
             lines.append(line.encode("utf-8"))
 
         self.assertEqual(len(lines), 1024)
         self.assertEqual(b"".join(lines), source)
 
 
 @make_cffi
 class TestDecompressor_decompressobj(TestCase):
     def test_simple(self):
         data = zstd.ZstdCompressor(level=1).compress(b"foobar")
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
         self.assertEqual(dobj.decompress(data), b"foobar")
         self.assertIsNone(dobj.flush())
         self.assertIsNone(dobj.flush(10))
         self.assertIsNone(dobj.flush(length=100))
 
     def test_input_types(self):
         compressed = zstd.ZstdCompressor(level=1).compress(b"foo")
 
         dctx = zstd.ZstdDecompressor()
 
         mutable_array = bytearray(len(compressed))
         mutable_array[:] = compressed
 
         sources = [
             memoryview(compressed),
             bytearray(compressed),
             mutable_array,
         ]
 
         for source in sources:
             dobj = dctx.decompressobj()
             self.assertIsNone(dobj.flush())
             self.assertIsNone(dobj.flush(10))
             self.assertIsNone(dobj.flush(length=100))
             self.assertEqual(dobj.decompress(source), b"foo")
             self.assertIsNone(dobj.flush())
 
     def test_reuse(self):
         data = zstd.ZstdCompressor(level=1).compress(b"foobar")
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
         dobj.decompress(data)
 
-        with self.assertRaisesRegex(zstd.ZstdError, "cannot use a decompressobj"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "cannot use a decompressobj"
+        ):
             dobj.decompress(data)
             self.assertIsNone(dobj.flush())
 
     def test_bad_write_size(self):
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegex(ValueError, "write_size must be positive"):
             dctx.decompressobj(write_size=0)
 
     def test_write_size(self):
         source = b"foo" * 64 + b"bar" * 128
         data = zstd.ZstdCompressor(level=1).compress(source)
 
         dctx = zstd.ZstdDecompressor()
 
         for i in range(128):
             dobj = dctx.decompressobj(write_size=i + 1)
             self.assertEqual(dobj.decompress(data), source)
 
 
 def decompress_via_writer(data):
     buffer = io.BytesIO()
     dctx = zstd.ZstdDecompressor()
     decompressor = dctx.stream_writer(buffer)
     decompressor.write(data)
 
     return buffer.getvalue()
 
 
 @make_cffi
 class TestDecompressor_stream_writer(TestCase):
     def test_io_api(self):
         buffer = io.BytesIO()
         dctx = zstd.ZstdDecompressor()
         writer = dctx.stream_writer(buffer)
 
         self.assertFalse(writer.closed)
         self.assertFalse(writer.isatty())
         self.assertFalse(writer.readable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readline(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readlines(hint=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(0)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.seek(10, os.SEEK_SET)
 
         self.assertFalse(writer.seekable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.tell()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.truncate(size=42)
 
         self.assertTrue(writer.writable())
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.writelines([])
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.read(size=42)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readall()
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.readinto(None)
 
         with self.assertRaises(io.UnsupportedOperation):
             writer.fileno()
 
     def test_fileno_file(self):
         with tempfile.TemporaryFile("wb") as tf:
             dctx = zstd.ZstdDecompressor()
             writer = dctx.stream_writer(tf)
 
             self.assertEqual(writer.fileno(), tf.fileno())
 
     def test_close(self):
         foo = zstd.ZstdCompressor().compress(b"foo")
 
         buffer = NonClosingBytesIO()
         dctx = zstd.ZstdDecompressor()
         writer = dctx.stream_writer(buffer)
 
         writer.write(foo)
         self.assertFalse(writer.closed)
         self.assertFalse(buffer.closed)
         writer.close()
         self.assertTrue(writer.closed)
         self.assertTrue(buffer.closed)
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             writer.write(b"")
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             writer.flush()
 
         with self.assertRaisesRegex(ValueError, "stream is closed"):
             with writer:
                 pass
 
         self.assertEqual(buffer.getvalue(), b"foo")
 
         # Context manager exit should close stream.
         buffer = NonClosingBytesIO()
         writer = dctx.stream_writer(buffer)
 
         with writer:
             writer.write(foo)
 
         self.assertTrue(writer.closed)
         self.assertEqual(buffer.getvalue(), b"foo")
 
     def test_flush(self):
         buffer = OpCountingBytesIO()
         dctx = zstd.ZstdDecompressor()
         writer = dctx.stream_writer(buffer)
 
         writer.flush()
         self.assertEqual(buffer._flush_count, 1)
         writer.flush()
         self.assertEqual(buffer._flush_count, 2)
 
     def test_empty_roundtrip(self):
         cctx = zstd.ZstdCompressor()
         empty = cctx.compress(b"")
         self.assertEqual(decompress_via_writer(empty), b"")
 
     def test_input_types(self):
         cctx = zstd.ZstdCompressor(level=1)
         compressed = cctx.compress(b"foo")
 
         mutable_array = bytearray(len(compressed))
         mutable_array[:] = compressed
 
         sources = [
             memoryview(compressed),
             bytearray(compressed),
             mutable_array,
         ]
 
         dctx = zstd.ZstdDecompressor()
         for source in sources:
             buffer = io.BytesIO()
 
             decompressor = dctx.stream_writer(buffer)
             decompressor.write(source)
             self.assertEqual(buffer.getvalue(), b"foo")
 
             buffer = NonClosingBytesIO()
 
             with dctx.stream_writer(buffer) as decompressor:
                 self.assertEqual(decompressor.write(source), 3)
 
             self.assertEqual(buffer.getvalue(), b"foo")
 
             buffer = io.BytesIO()
             writer = dctx.stream_writer(buffer, write_return_read=True)
             self.assertEqual(writer.write(source), len(source))
             self.assertEqual(buffer.getvalue(), b"foo")
 
     def test_large_roundtrip(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct(">B").pack(i) * 16384)
         orig = b"".join(chunks)
         cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(orig)
 
         self.assertEqual(decompress_via_writer(compressed), orig)
 
     def test_multiple_calls(self):
         chunks = []
         for i in range(255):
             for j in range(255):
                 chunks.append(struct.Struct(">B").pack(j) * i)
 
         orig = b"".join(chunks)
         cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(orig)
 
         buffer = NonClosingBytesIO()
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_writer(buffer) as decompressor:
             pos = 0
             while pos < len(compressed):
                 pos2 = pos + 8192
                 decompressor.write(compressed[pos:pos2])
                 pos += 8192
         self.assertEqual(buffer.getvalue(), orig)
 
         # Again with write_return_read=True
         buffer = io.BytesIO()
         writer = dctx.stream_writer(buffer, write_return_read=True)
         pos = 0
         while pos < len(compressed):
             pos2 = pos + 8192
             chunk = compressed[pos:pos2]
             self.assertEqual(writer.write(chunk), len(chunk))
             pos += 8192
         self.assertEqual(buffer.getvalue(), orig)
 
     def test_dictionary(self):
         samples = []
         for i in range(128):
             samples.append(b"foo" * 64)
             samples.append(b"bar" * 64)
             samples.append(b"foobar" * 64)
 
         d = zstd.train_dictionary(8192, samples)
 
         orig = b"foobar" * 16384
         buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(dict_data=d)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(orig), 0)
 
         compressed = buffer.getvalue()
         buffer = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
         decompressor = dctx.stream_writer(buffer)
         self.assertEqual(decompressor.write(compressed), len(orig))
         self.assertEqual(buffer.getvalue(), orig)
 
         buffer = NonClosingBytesIO()
 
         with dctx.stream_writer(buffer) as decompressor:
             self.assertEqual(decompressor.write(compressed), len(orig))
 
         self.assertEqual(buffer.getvalue(), orig)
 
     def test_memory_size(self):
         dctx = zstd.ZstdDecompressor()
         buffer = io.BytesIO()
 
         decompressor = dctx.stream_writer(buffer)
         size = decompressor.memory_size()
         self.assertGreater(size, 100000)
 
         with dctx.stream_writer(buffer) as decompressor:
             size = decompressor.memory_size()
 
         self.assertGreater(size, 100000)
 
     def test_write_size(self):
         source = zstd.ZstdCompressor().compress(b"foobarfoobar")
         dest = OpCountingBytesIO()
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_writer(dest, write_size=1) as decompressor:
             s = struct.Struct(">B")
             for c in source:
                 if not isinstance(c, str):
                     c = s.pack(c)
                 decompressor.write(c)
 
         self.assertEqual(dest.getvalue(), b"foobarfoobar")
         self.assertEqual(dest._write_count, len(dest.getvalue()))
 
 
 @make_cffi
 class TestDecompressor_read_to_iter(TestCase):
     def test_type_validation(self):
         dctx = zstd.ZstdDecompressor()
 
         # Object with read() works.
         dctx.read_to_iter(io.BytesIO())
 
         # Buffer protocol works.
         dctx.read_to_iter(b"foobar")
 
-        with self.assertRaisesRegex(ValueError, "must pass an object with a read"):
+        with self.assertRaisesRegex(
+            ValueError, "must pass an object with a read"
+        ):
             b"".join(dctx.read_to_iter(True))
 
     def test_empty_input(self):
         dctx = zstd.ZstdDecompressor()
 
         source = io.BytesIO()
         it = dctx.read_to_iter(source)
         # TODO this is arguably wrong. Should get an error about missing frame foo.
         with self.assertRaises(StopIteration):
             next(it)
 
         it = dctx.read_to_iter(b"")
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_invalid_input(self):
         dctx = zstd.ZstdDecompressor()
 
         source = io.BytesIO(b"foobar")
         it = dctx.read_to_iter(source)
         with self.assertRaisesRegex(zstd.ZstdError, "Unknown frame descriptor"):
             next(it)
 
         it = dctx.read_to_iter(b"foobar")
         with self.assertRaisesRegex(zstd.ZstdError, "Unknown frame descriptor"):
             next(it)
 
     def test_empty_roundtrip(self):
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         empty = cctx.compress(b"")
 
         source = io.BytesIO(empty)
         source.seek(0)
 
         dctx = zstd.ZstdDecompressor()
         it = dctx.read_to_iter(source)
 
         # No chunks should be emitted since there is no data.
         with self.assertRaises(StopIteration):
             next(it)
 
         # Again for good measure.
         with self.assertRaises(StopIteration):
             next(it)
 
     def test_skip_bytes_too_large(self):
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegex(
             ValueError, "skip_bytes must be smaller than read_size"
         ):
             b"".join(dctx.read_to_iter(b"", skip_bytes=1, read_size=1))
 
         with self.assertRaisesRegex(
             ValueError, "skip_bytes larger than first input chunk"
         ):
             b"".join(dctx.read_to_iter(b"foobar", skip_bytes=10))
 
     def test_skip_bytes(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         compressed = cctx.compress(b"foobar")
 
         dctx = zstd.ZstdDecompressor()
         output = b"".join(dctx.read_to_iter(b"hdr" + compressed, skip_bytes=3))
         self.assertEqual(output, b"foobar")
 
     def test_large_output(self):
         source = io.BytesIO()
         source.write(b"f" * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
         source.write(b"o")
         source.seek(0)
 
         cctx = zstd.ZstdCompressor(level=1)
         compressed = io.BytesIO(cctx.compress(source.getvalue()))
         compressed.seek(0)
 
         dctx = zstd.ZstdDecompressor()
         it = dctx.read_to_iter(compressed)
 
         chunks = []
         chunks.append(next(it))
         chunks.append(next(it))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         decompressed = b"".join(chunks)
         self.assertEqual(decompressed, source.getvalue())
 
         # And again with buffer protocol.
         it = dctx.read_to_iter(compressed.getvalue())
         chunks = []
         chunks.append(next(it))
         chunks.append(next(it))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         decompressed = b"".join(chunks)
         self.assertEqual(decompressed, source.getvalue())
 
-    @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
+    @unittest.skipUnless(
+        "ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set"
+    )
     def test_large_input(self):
         bytes = list(struct.Struct(">B").pack(i) for i in range(256))
         compressed = NonClosingBytesIO()
         input_size = 0
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(compressed) as compressor:
             while True:
                 compressor.write(random.choice(bytes))
                 input_size += 1
 
                 have_compressed = (
                     len(compressed.getvalue())
                     > zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
                 )
-                have_raw = input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+                have_raw = (
+                    input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2
+                )
                 if have_compressed and have_raw:
                     break
 
         compressed = io.BytesIO(compressed.getvalue())
         self.assertGreater(
-            len(compressed.getvalue()), zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
+            len(compressed.getvalue()),
+            zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         )
 
         dctx = zstd.ZstdDecompressor()
         it = dctx.read_to_iter(compressed)
 
         chunks = []
         chunks.append(next(it))
         chunks.append(next(it))
         chunks.append(next(it))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         decompressed = b"".join(chunks)
         self.assertEqual(len(decompressed), input_size)
 
         # And again with buffer protocol.
         it = dctx.read_to_iter(compressed.getvalue())
 
         chunks = []
         chunks.append(next(it))
         chunks.append(next(it))
         chunks.append(next(it))
 
         with self.assertRaises(StopIteration):
             next(it)
 
         decompressed = b"".join(chunks)
         self.assertEqual(len(decompressed), input_size)
 
     def test_interesting(self):
         # Found this edge case via fuzzing.
         cctx = zstd.ZstdCompressor(level=1)
 
         source = io.BytesIO()
 
         compressed = NonClosingBytesIO()
         with cctx.stream_writer(compressed) as compressor:
             for i in range(256):
                 chunk = b"\0" * 1024
                 compressor.write(chunk)
                 source.write(chunk)
 
         dctx = zstd.ZstdDecompressor()
 
         simple = dctx.decompress(
             compressed.getvalue(), max_output_size=len(source.getvalue())
         )
         self.assertEqual(simple, source.getvalue())
 
         compressed = io.BytesIO(compressed.getvalue())
         streamed = b"".join(dctx.read_to_iter(compressed))
         self.assertEqual(streamed, source.getvalue())
 
     def test_read_write_size(self):
-        source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b"foobarfoobar"))
+        source = OpCountingBytesIO(
+            zstd.ZstdCompressor().compress(b"foobarfoobar")
+        )
         dctx = zstd.ZstdDecompressor()
         for chunk in dctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
 
         self.assertEqual(source._read_count, len(source.getvalue()))
 
     def test_magic_less(self):
         params = zstd.CompressionParameters.from_level(
             1, format=zstd.FORMAT_ZSTD1_MAGICLESS
         )
         cctx = zstd.ZstdCompressor(compression_params=params)
         frame = cctx.compress(b"foobar")
 
         self.assertNotEqual(frame[0:4], b"\x28\xb5\x2f\xfd")
 
         dctx = zstd.ZstdDecompressor()
         with self.assertRaisesRegex(
             zstd.ZstdError, "error determining content size from frame header"
         ):
             dctx.decompress(frame)
 
         dctx = zstd.ZstdDecompressor(format=zstd.FORMAT_ZSTD1_MAGICLESS)
         res = b"".join(dctx.read_to_iter(frame))
         self.assertEqual(res, b"foobar")
 
 
 @make_cffi
 class TestDecompressor_content_dict_chain(TestCase):
     def test_bad_inputs_simple(self):
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaises(TypeError):
             dctx.decompress_content_dict_chain(b"foo")
 
         with self.assertRaises(TypeError):
             dctx.decompress_content_dict_chain((b"foo", b"bar"))
 
         with self.assertRaisesRegex(ValueError, "empty input chain"):
             dctx.decompress_content_dict_chain([])
 
         with self.assertRaisesRegex(ValueError, "chunk 0 must be bytes"):
             dctx.decompress_content_dict_chain([u"foo"])
 
         with self.assertRaisesRegex(ValueError, "chunk 0 must be bytes"):
             dctx.decompress_content_dict_chain([True])
 
         with self.assertRaisesRegex(
             ValueError, "chunk 0 is too small to contain a zstd frame"
         ):
             dctx.decompress_content_dict_chain([zstd.FRAME_HEADER])
 
-        with self.assertRaisesRegex(ValueError, "chunk 0 is not a valid zstd frame"):
+        with self.assertRaisesRegex(
+            ValueError, "chunk 0 is not a valid zstd frame"
+        ):
             dctx.decompress_content_dict_chain([b"foo" * 8])
 
-        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+            b"foo" * 64
+        )
 
         with self.assertRaisesRegex(
             ValueError, "chunk 0 missing content size in frame"
         ):
             dctx.decompress_content_dict_chain([no_size])
 
         # Corrupt first frame.
         frame = zstd.ZstdCompressor().compress(b"foo" * 64)
         frame = frame[0:12] + frame[15:]
         with self.assertRaisesRegex(
             zstd.ZstdError, "chunk 0 did not decompress full frame"
         ):
             dctx.decompress_content_dict_chain([frame])
 
     def test_bad_subsequent_input(self):
         initial = zstd.ZstdCompressor().compress(b"foo" * 64)
 
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegex(ValueError, "chunk 1 must be bytes"):
             dctx.decompress_content_dict_chain([initial, u"foo"])
 
         with self.assertRaisesRegex(ValueError, "chunk 1 must be bytes"):
             dctx.decompress_content_dict_chain([initial, None])
 
         with self.assertRaisesRegex(
             ValueError, "chunk 1 is too small to contain a zstd frame"
         ):
             dctx.decompress_content_dict_chain([initial, zstd.FRAME_HEADER])
 
-        with self.assertRaisesRegex(ValueError, "chunk 1 is not a valid zstd frame"):
+        with self.assertRaisesRegex(
+            ValueError, "chunk 1 is not a valid zstd frame"
+        ):
             dctx.decompress_content_dict_chain([initial, b"foo" * 8])
 
-        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b"foo" * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(
+            b"foo" * 64
+        )
 
         with self.assertRaisesRegex(
             ValueError, "chunk 1 missing content size in frame"
         ):
             dctx.decompress_content_dict_chain([initial, no_size])
 
         # Corrupt second frame.
-        cctx = zstd.ZstdCompressor(dict_data=zstd.ZstdCompressionDict(b"foo" * 64))
+        cctx = zstd.ZstdCompressor(
+            dict_data=zstd.ZstdCompressionDict(b"foo" * 64)
+        )
         frame = cctx.compress(b"bar" * 64)
         frame = frame[0:12] + frame[15:]
 
         with self.assertRaisesRegex(
             zstd.ZstdError, "chunk 1 did not decompress full frame"
         ):
             dctx.decompress_content_dict_chain([initial, frame])
 
     def test_simple(self):
         original = [
             b"foo" * 64,
             b"foobar" * 64,
             b"baz" * 64,
             b"foobaz" * 64,
             b"foobarbaz" * 64,
         ]
 
         chunks = []
         chunks.append(zstd.ZstdCompressor().compress(original[0]))
         for i, chunk in enumerate(original[1:]):
             d = zstd.ZstdCompressionDict(original[i])
             cctx = zstd.ZstdCompressor(dict_data=d)
             chunks.append(cctx.compress(chunk))
 
         for i in range(1, len(original)):
             chain = chunks[0:i]
             expected = original[i - 1]
             dctx = zstd.ZstdDecompressor()
             decompressed = dctx.decompress_content_dict_chain(chain)
             self.assertEqual(decompressed, expected)
 
 
 # TODO enable for CFFI
 class TestDecompressor_multi_decompress_to_buffer(TestCase):
     def test_invalid_inputs(self):
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         with self.assertRaises(TypeError):
             dctx.multi_decompress_to_buffer(True)
 
         with self.assertRaises(TypeError):
             dctx.multi_decompress_to_buffer((1, 2))
 
-        with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
+        with self.assertRaisesRegex(
+            TypeError, "item 0 not a bytes like object"
+        ):
             dctx.multi_decompress_to_buffer([u"foo"])
 
         with self.assertRaisesRegex(
             ValueError, "could not determine decompressed size of item 0"
         ):
             dctx.multi_decompress_to_buffer([b"foobarbaz"])
 
     def test_list_input(self):
         cctx = zstd.ZstdCompressor()
 
         original = [b"foo" * 4, b"bar" * 6]
         frames = [cctx.compress(d) for d in original]
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         result = dctx.multi_decompress_to_buffer(frames)
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result.size(), sum(map(len, original)))
 
         for i, data in enumerate(original):
             self.assertEqual(result[i].tobytes(), data)
 
         self.assertEqual(result[0].offset, 0)
         self.assertEqual(len(result[0]), 12)
         self.assertEqual(result[1].offset, 12)
         self.assertEqual(len(result[1]), 18)
 
     def test_list_input_frame_sizes(self):
         cctx = zstd.ZstdCompressor()
 
         original = [b"foo" * 4, b"bar" * 6, b"baz" * 8]
         frames = [cctx.compress(d) for d in original]
         sizes = struct.pack("=" + "Q" * len(original), *map(len, original))
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
-        result = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
+        result = dctx.multi_decompress_to_buffer(
+            frames, decompressed_sizes=sizes
+        )
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result.size(), sum(map(len, original)))
 
         for i, data in enumerate(original):
             self.assertEqual(result[i].tobytes(), data)
 
     def test_buffer_with_segments_input(self):
         cctx = zstd.ZstdCompressor()
 
         original = [b"foo" * 4, b"bar" * 6]
         frames = [cctx.compress(d) for d in original]
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         segments = struct.pack(
             "=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1])
         )
         b = zstd.BufferWithSegments(b"".join(frames), segments)
 
         result = dctx.multi_decompress_to_buffer(b)
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result[0].offset, 0)
         self.assertEqual(len(result[0]), 12)
         self.assertEqual(result[1].offset, 12)
         self.assertEqual(len(result[1]), 18)
 
     def test_buffer_with_segments_sizes(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         original = [b"foo" * 4, b"bar" * 6, b"baz" * 8]
         frames = [cctx.compress(d) for d in original]
         sizes = struct.pack("=" + "Q" * len(original), *map(len, original))
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         segments = struct.pack(
             "=QQQQQQ",
             0,
             len(frames[0]),
             len(frames[0]),
             len(frames[1]),
             len(frames[0]) + len(frames[1]),
             len(frames[2]),
         )
         b = zstd.BufferWithSegments(b"".join(frames), segments)
 
         result = dctx.multi_decompress_to_buffer(b, decompressed_sizes=sizes)
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result.size(), sum(map(len, original)))
 
         for i, data in enumerate(original):
             self.assertEqual(result[i].tobytes(), data)
 
     def test_buffer_with_segments_collection_input(self):
         cctx = zstd.ZstdCompressor()
 
         original = [
             b"foo0" * 2,
             b"foo1" * 3,
             b"foo2" * 4,
             b"foo3" * 5,
             b"foo4" * 6,
         ]
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         frames = cctx.multi_compress_to_buffer(original)
 
         # Check round trip.
         dctx = zstd.ZstdDecompressor()
 
         decompressed = dctx.multi_decompress_to_buffer(frames, threads=3)
 
         self.assertEqual(len(decompressed), len(original))
 
         for i, data in enumerate(original):
             self.assertEqual(data, decompressed[i].tobytes())
 
         # And a manual mode.
         b = b"".join([frames[0].tobytes(), frames[1].tobytes()])
         b1 = zstd.BufferWithSegments(
-            b, struct.pack("=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1]))
+            b,
+            struct.pack(
+                "=QQQQ", 0, len(frames[0]), len(frames[0]), len(frames[1])
+            ),
         )
 
-        b = b"".join([frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()])
+        b = b"".join(
+            [frames[2].tobytes(), frames[3].tobytes(), frames[4].tobytes()]
+        )
         b2 = zstd.BufferWithSegments(
             b,
             struct.pack(
                 "=QQQQQQ",
                 0,
                 len(frames[2]),
                 len(frames[2]),
                 len(frames[3]),
                 len(frames[2]) + len(frames[3]),
                 len(frames[4]),
             ),
         )
 
         c = zstd.BufferWithSegmentsCollection(b1, b2)
 
         dctx = zstd.ZstdDecompressor()
         decompressed = dctx.multi_decompress_to_buffer(c)
 
         self.assertEqual(len(decompressed), 5)
         for i in range(5):
             self.assertEqual(decompressed[i].tobytes(), original[i])
 
     def test_dict(self):
         d = zstd.train_dictionary(16384, generate_samples(), k=64, d=16)
 
         cctx = zstd.ZstdCompressor(dict_data=d, level=1)
         frames = [cctx.compress(s) for s in generate_samples()]
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         result = dctx.multi_decompress_to_buffer(frames)
 
         self.assertEqual([o.tobytes() for o in result], generate_samples())
 
     def test_multiple_threads(self):
         cctx = zstd.ZstdCompressor()
 
         frames = []
         frames.extend(cctx.compress(b"x" * 64) for i in range(256))
         frames.extend(cctx.compress(b"y" * 64) for i in range(256))
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         result = dctx.multi_decompress_to_buffer(frames, threads=-1)
 
         self.assertEqual(len(result), len(frames))
         self.assertEqual(result.size(), 2 * 64 * 256)
         self.assertEqual(result[0].tobytes(), b"x" * 64)
         self.assertEqual(result[256].tobytes(), b"y" * 64)
 
     def test_item_failure(self):
         cctx = zstd.ZstdCompressor()
         frames = [cctx.compress(b"x" * 128), cctx.compress(b"y" * 128)]
 
         frames[1] = frames[1][0:15] + b"extra" + frames[1][15:]
 
         dctx = zstd.ZstdDecompressor()
 
         if not hasattr(dctx, "multi_decompress_to_buffer"):
             self.skipTest("multi_decompress_to_buffer not available")
 
         with self.assertRaisesRegex(
             zstd.ZstdError,
             "error decompressing item 1: ("
             "Corrupted block|"
             "Destination buffer is too small)",
         ):
             dctx.multi_decompress_to_buffer(frames)
 
         with self.assertRaisesRegex(
             zstd.ZstdError,
             "error decompressing item 1: ("
             "Corrupted block|"
             "Destination buffer is too small)",
         ):
             dctx.multi_decompress_to_buffer(frames, threads=2)

contrib/python-zstandard/tests/test_decompressor_fuzzing.py

@@ -1,576 +1,593 b''
 import io
 import os
 import unittest
 
 try:
     import hypothesis
     import hypothesis.strategies as strategies
 except ImportError:
     raise unittest.SkipTest("hypothesis not available")
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
     NonClosingBytesIO,
     random_input_data,
     TestCase,
 )
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestDecompressor_stream_reader_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_sizes=strategies.data(),
     )
     def test_stream_source_read_variance(
         self, original, level, streaming, source_read_size, read_sizes
     ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             source.seek(0)
         else:
             frame = cctx.compress(original)
             source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
 
         chunks = []
         with dctx.stream_reader(source, read_size=source_read_size) as reader:
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 131072))
                 chunk = reader.read(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), original)
 
     # Similar to above except we have a constant read() size.
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_size=strategies.integers(-1, 131072),
     )
     def test_stream_source_read_size(
         self, original, level, streaming, source_read_size, read_size
     ):
         if read_size == 0:
             read_size = 1
 
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             source.seek(0)
         else:
             frame = cctx.compress(original)
             source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
 
         chunks = []
         reader = dctx.stream_reader(source, read_size=source_read_size)
         while True:
             chunk = reader.read(read_size)
             if not chunk and read_size:
                 break
 
             chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), original)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_sizes=strategies.data(),
     )
     def test_buffer_source_read_variance(
         self, original, level, streaming, source_read_size, read_sizes
     ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             frame = source.getvalue()
         else:
             frame = cctx.compress(original)
 
         dctx = zstd.ZstdDecompressor()
         chunks = []
 
         with dctx.stream_reader(frame, read_size=source_read_size) as reader:
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 131072))
                 chunk = reader.read(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), original)
 
     # Similar to above except we have a constant read() size.
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_size=strategies.integers(-1, 131072),
     )
     def test_buffer_source_constant_read_size(
         self, original, level, streaming, source_read_size, read_size
     ):
         if read_size == 0:
             read_size = -1
 
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             frame = source.getvalue()
         else:
             frame = cctx.compress(original)
 
         dctx = zstd.ZstdDecompressor()
         chunks = []
 
         reader = dctx.stream_reader(frame, read_size=source_read_size)
         while True:
             chunk = reader.read(read_size)
             if not chunk and read_size:
                 break
 
             chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), original)
 
     @hypothesis.settings(
         suppress_health_check=[hypothesis.HealthCheck.large_base_example]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
     )
-    def test_stream_source_readall(self, original, level, streaming, source_read_size):
+    def test_stream_source_readall(
+        self, original, level, streaming, source_read_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             source.seek(0)
         else:
             frame = cctx.compress(original)
             source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
 
         data = dctx.stream_reader(source, read_size=source_read_size).readall()
         self.assertEqual(data, original)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_sizes=strategies.data(),
     )
     def test_stream_source_read1_variance(
         self, original, level, streaming, source_read_size, read_sizes
     ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             source.seek(0)
         else:
             frame = cctx.compress(original)
             source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
 
         chunks = []
         with dctx.stream_reader(source, read_size=source_read_size) as reader:
             while True:
                 read_size = read_sizes.draw(strategies.integers(-1, 131072))
                 chunk = reader.read1(read_size)
                 if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
 
         self.assertEqual(b"".join(chunks), original)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         streaming=strategies.booleans(),
         source_read_size=strategies.integers(1, 1048576),
         read_sizes=strategies.data(),
     )
     def test_stream_source_readinto1_variance(
         self, original, level, streaming, source_read_size, read_sizes
     ):
         cctx = zstd.ZstdCompressor(level=level)
 
         if streaming:
             source = io.BytesIO()
             writer = cctx.stream_writer(source)
             writer.write(original)
             writer.flush(zstd.FLUSH_FRAME)
             source.seek(0)
         else:
             frame = cctx.compress(original)
             source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
 
         chunks = []
         with dctx.stream_reader(source, read_size=source_read_size) as reader:
             while True:
                 read_size = read_sizes.draw(strategies.integers(1, 131072))
                 b = bytearray(read_size)
                 count = reader.readinto1(b)
 
                 if not count:
                     break
 
                 chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b"".join(chunks), original)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 1048576),
         seek_amounts=strategies.data(),
         read_sizes=strategies.data(),
     )
     def test_relative_seeks(
         self, original, level, source_read_size, seek_amounts, read_sizes
     ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(frame, read_size=source_read_size) as reader:
             while True:
                 amount = seek_amounts.draw(strategies.integers(0, 16384))
                 reader.seek(amount, os.SEEK_CUR)
 
                 offset = reader.tell()
                 read_amount = read_sizes.draw(strategies.integers(1, 16384))
                 chunk = reader.read(read_amount)
 
                 if not chunk:
                     break
 
                 self.assertEqual(original[offset : offset + len(chunk)], chunk)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         originals=strategies.data(),
         frame_count=strategies.integers(min_value=2, max_value=10),
         level=strategies.integers(min_value=1, max_value=5),
         source_read_size=strategies.integers(1, 1048576),
         read_sizes=strategies.data(),
     )
     def test_multiple_frames(
         self, originals, frame_count, level, source_read_size, read_sizes
     ):
 
         cctx = zstd.ZstdCompressor(level=level)
         source = io.BytesIO()
         buffer = io.BytesIO()
         writer = cctx.stream_writer(buffer)
 
         for i in range(frame_count):
             data = originals.draw(strategies.sampled_from(random_input_data()))
             source.write(data)
             writer.write(data)
             writer.flush(zstd.FLUSH_FRAME)
 
         dctx = zstd.ZstdDecompressor()
         buffer.seek(0)
         reader = dctx.stream_reader(
             buffer, read_size=source_read_size, read_across_frames=True
         )
 
         chunks = []
 
         while True:
             read_amount = read_sizes.draw(strategies.integers(-1, 16384))
             chunk = reader.read(read_amount)
 
             if not chunk and read_amount:
                 break
 
             chunks.append(chunk)
 
         self.assertEqual(source.getvalue(), b"".join(chunks))
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestDecompressor_stream_writer_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         write_size=strategies.integers(min_value=1, max_value=8192),
         input_sizes=strategies.data(),
     )
-    def test_write_size_variance(self, original, level, write_size, input_sizes):
+    def test_write_size_variance(
+        self, original, level, write_size, input_sizes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         dctx = zstd.ZstdDecompressor()
         source = io.BytesIO(frame)
         dest = NonClosingBytesIO()
 
         with dctx.stream_writer(dest, write_size=write_size) as decompressor:
             while True:
                 input_size = input_sizes.draw(strategies.integers(1, 4096))
                 chunk = source.read(input_size)
                 if not chunk:
                     break
 
                 decompressor.write(chunk)
 
         self.assertEqual(dest.getvalue(), original)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestDecompressor_copy_stream_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         read_size=strategies.integers(min_value=1, max_value=8192),
         write_size=strategies.integers(min_value=1, max_value=8192),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         source = io.BytesIO(frame)
         dest = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor()
-        dctx.copy_stream(source, dest, read_size=read_size, write_size=write_size)
+        dctx.copy_stream(
+            source, dest, read_size=read_size, write_size=write_size
+        )
 
         self.assertEqual(dest.getvalue(), original)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestDecompressor_decompressobj_fuzzing(TestCase):
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         chunk_sizes=strategies.data(),
     )
     def test_random_input_sizes(self, original, level, chunk_sizes):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
 
         chunks = []
         while True:
             chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
             chunk = source.read(chunk_size)
             if not chunk:
                 break
 
             chunks.append(dobj.decompress(chunk))
 
         self.assertEqual(b"".join(chunks), original)
 
     @hypothesis.settings(
         suppress_health_check=[
             hypothesis.HealthCheck.large_base_example,
             hypothesis.HealthCheck.too_slow,
         ]
     )
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         write_size=strategies.integers(
-            min_value=1, max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
+            min_value=1,
+            max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         ),
         chunk_sizes=strategies.data(),
     )
-    def test_random_output_sizes(self, original, level, write_size, chunk_sizes):
+    def test_random_output_sizes(
+        self, original, level, write_size, chunk_sizes
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj(write_size=write_size)
 
         chunks = []
         while True:
             chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
             chunk = source.read(chunk_size)
             if not chunk:
                 break
 
             chunks.append(dobj.decompress(chunk))
 
         self.assertEqual(b"".join(chunks), original)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 @make_cffi
 class TestDecompressor_read_to_iter_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
         read_size=strategies.integers(min_value=1, max_value=4096),
         write_size=strategies.integers(min_value=1, max_value=4096),
     )
-    def test_read_write_size_variance(self, original, level, read_size, write_size):
+    def test_read_write_size_variance(
+        self, original, level, read_size, write_size
+    ):
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
         source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
         chunks = list(
-            dctx.read_to_iter(source, read_size=read_size, write_size=write_size)
+            dctx.read_to_iter(
+                source, read_size=read_size, write_size=write_size
+            )
         )
 
         self.assertEqual(b"".join(chunks), original)
 
 
 @unittest.skipUnless("ZSTD_SLOW_TESTS" in os.environ, "ZSTD_SLOW_TESTS not set")
 class TestDecompressor_multi_decompress_to_buffer_fuzzing(TestCase):
     @hypothesis.given(
         original=strategies.lists(
-            strategies.sampled_from(random_input_data()), min_size=1, max_size=1024
+            strategies.sampled_from(random_input_data()),
+            min_size=1,
+            max_size=1024,
         ),
         threads=strategies.integers(min_value=1, max_value=8),
         use_dict=strategies.booleans(),
     )
     def test_data_equivalence(self, original, threads, use_dict):
         kwargs = {}
         if use_dict:
             kwargs["dict_data"] = zstd.ZstdCompressionDict(original[0])
 
         cctx = zstd.ZstdCompressor(
             level=1, write_content_size=True, write_checksum=True, **kwargs
         )
 
         if not hasattr(cctx, "multi_compress_to_buffer"):
             self.skipTest("multi_compress_to_buffer not available")
 
         frames_buffer = cctx.multi_compress_to_buffer(original, threads=-1)
 
         dctx = zstd.ZstdDecompressor(**kwargs)
         result = dctx.multi_decompress_to_buffer(frames_buffer)
 
         self.assertEqual(len(result), len(original))
         for i, frame in enumerate(result):
             self.assertEqual(frame.tobytes(), original[i])
 
         frames_list = [f.tobytes() for f in frames_buffer]
         result = dctx.multi_decompress_to_buffer(frames_list)
 
         self.assertEqual(len(result), len(original))
         for i, frame in enumerate(result):
             self.assertEqual(frame.tobytes(), original[i])

contrib/python-zstandard/tests/test_train_dictionary.py

@@ -1,92 +1,102 b''
 import struct
 import sys
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     generate_samples,
     make_cffi,
     random_input_data,
     TestCase,
 )
 
 if sys.version_info[0] >= 3:
     int_type = int
 else:
     int_type = long
 
 
 @make_cffi
 class TestTrainDictionary(TestCase):
     def test_no_args(self):
         with self.assertRaises(TypeError):
             zstd.train_dictionary()
 
     def test_bad_args(self):
         with self.assertRaises(TypeError):
             zstd.train_dictionary(8192, u"foo")
 
         with self.assertRaises(ValueError):
             zstd.train_dictionary(8192, [u"foo"])
 
     def test_no_params(self):
         d = zstd.train_dictionary(8192, random_input_data())
         self.assertIsInstance(d.dict_id(), int_type)
 
         # The dictionary ID may be different across platforms.
         expected = b"\x37\xa4\x30\xec" + struct.pack("<I", d.dict_id())
 
         data = d.as_bytes()
         self.assertEqual(data[0:8], expected)
 
     def test_basic(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
         self.assertIsInstance(d.dict_id(), int_type)
 
         data = d.as_bytes()
         self.assertEqual(data[0:4], b"\x37\xa4\x30\xec")
 
         self.assertEqual(d.k, 64)
         self.assertEqual(d.d, 16)
 
     def test_set_dict_id(self):
-        d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16, dict_id=42)
+        d = zstd.train_dictionary(
+            8192, generate_samples(), k=64, d=16, dict_id=42
+        )
         self.assertEqual(d.dict_id(), 42)
 
     def test_optimize(self):
-        d = zstd.train_dictionary(8192, generate_samples(), threads=-1, steps=1, d=16)
+        d = zstd.train_dictionary(
+            8192, generate_samples(), threads=-1, steps=1, d=16
+        )
 
         # This varies by platform.
         self.assertIn(d.k, (50, 2000))
         self.assertEqual(d.d, 16)
 
 
 @make_cffi
 class TestCompressionDict(TestCase):
     def test_bad_mode(self):
         with self.assertRaisesRegex(ValueError, "invalid dictionary load mode"):
             zstd.ZstdCompressionDict(b"foo", dict_type=42)
 
     def test_bad_precompute_compress(self):
         d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
 
-        with self.assertRaisesRegex(ValueError, "must specify one of level or "):
+        with self.assertRaisesRegex(
+            ValueError, "must specify one of level or "
+        ):
             d.precompute_compress()
 
-        with self.assertRaisesRegex(ValueError, "must only specify one of level or "):
+        with self.assertRaisesRegex(
+            ValueError, "must only specify one of level or "
+        ):
             d.precompute_compress(
                 level=3, compression_params=zstd.CompressionParameters()
             )
 
     def test_precompute_compress_rawcontent(self):
         d = zstd.ZstdCompressionDict(
             b"dictcontent" * 64, dict_type=zstd.DICT_TYPE_RAWCONTENT
         )
         d.precompute_compress(level=1)
 
         d = zstd.ZstdCompressionDict(
             b"dictcontent" * 64, dict_type=zstd.DICT_TYPE_FULLDICT
         )
-        with self.assertRaisesRegex(zstd.ZstdError, "unable to precompute dictionary"):
+        with self.assertRaisesRegex(
+            zstd.ZstdError, "unable to precompute dictionary"
+        ):
             d.precompute_compress(level=1)

contrib/python-zstandard/zstandard/cffi.py

@@ -1,2615 +1,2769 b''
 # Copyright (c) 2016-present, Gregory Szorc
 # All rights reserved.
 #
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
 """Python interface to the Zstandard (zstd) compression library."""
 
 from __future__ import absolute_import, unicode_literals
 
 # This should match what the C extension exports.
 __all__ = [
     #'BufferSegment',
     #'BufferSegments',
     #'BufferWithSegments',
     #'BufferWithSegmentsCollection',
     "CompressionParameters",
     "ZstdCompressionDict",
     "ZstdCompressionParameters",
     "ZstdCompressor",
     "ZstdError",
     "ZstdDecompressor",
     "FrameParameters",
     "estimate_decompression_context_size",
     "frame_content_size",
     "frame_header_size",
     "get_frame_parameters",
     "train_dictionary",
     # Constants.
     "FLUSH_BLOCK",
     "FLUSH_FRAME",
     "COMPRESSOBJ_FLUSH_FINISH",
     "COMPRESSOBJ_FLUSH_BLOCK",
     "ZSTD_VERSION",
     "FRAME_HEADER",
     "CONTENTSIZE_UNKNOWN",
     "CONTENTSIZE_ERROR",
     "MAX_COMPRESSION_LEVEL",
     "COMPRESSION_RECOMMENDED_INPUT_SIZE",
     "COMPRESSION_RECOMMENDED_OUTPUT_SIZE",
     "DECOMPRESSION_RECOMMENDED_INPUT_SIZE",
     "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE",
     "MAGIC_NUMBER",
     "BLOCKSIZELOG_MAX",
     "BLOCKSIZE_MAX",
     "WINDOWLOG_MIN",
     "WINDOWLOG_MAX",
     "CHAINLOG_MIN",
     "CHAINLOG_MAX",
     "HASHLOG_MIN",
     "HASHLOG_MAX",
     "HASHLOG3_MAX",
     "MINMATCH_MIN",
     "MINMATCH_MAX",
     "SEARCHLOG_MIN",
     "SEARCHLOG_MAX",
     "SEARCHLENGTH_MIN",
     "SEARCHLENGTH_MAX",
     "TARGETLENGTH_MIN",
     "TARGETLENGTH_MAX",
     "LDM_MINMATCH_MIN",
     "LDM_MINMATCH_MAX",
     "LDM_BUCKETSIZELOG_MAX",
     "STRATEGY_FAST",
     "STRATEGY_DFAST",
     "STRATEGY_GREEDY",
     "STRATEGY_LAZY",
     "STRATEGY_LAZY2",
     "STRATEGY_BTLAZY2",
     "STRATEGY_BTOPT",
     "STRATEGY_BTULTRA",
     "STRATEGY_BTULTRA2",
     "DICT_TYPE_AUTO",
     "DICT_TYPE_RAWCONTENT",
     "DICT_TYPE_FULLDICT",
     "FORMAT_ZSTD1",
     "FORMAT_ZSTD1_MAGICLESS",
 ]
 
 import io
 import os
 import sys
 
 from _zstd_cffi import (
     ffi,
     lib,
 )
 
 if sys.version_info[0] == 2:
     bytes_type = str
     int_type = long
 else:
     bytes_type = bytes
     int_type = int
 
 
 COMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_CStreamInSize()
 COMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_CStreamOutSize()
 DECOMPRESSION_RECOMMENDED_INPUT_SIZE = lib.ZSTD_DStreamInSize()
 DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE = lib.ZSTD_DStreamOutSize()
 
 new_nonzero = ffi.new_allocator(should_clear_after_alloc=False)
 
 
 MAX_COMPRESSION_LEVEL = lib.ZSTD_maxCLevel()
 MAGIC_NUMBER = lib.ZSTD_MAGICNUMBER
 FRAME_HEADER = b"\x28\xb5\x2f\xfd"
 CONTENTSIZE_UNKNOWN = lib.ZSTD_CONTENTSIZE_UNKNOWN
 CONTENTSIZE_ERROR = lib.ZSTD_CONTENTSIZE_ERROR
 ZSTD_VERSION = (
     lib.ZSTD_VERSION_MAJOR,
     lib.ZSTD_VERSION_MINOR,
     lib.ZSTD_VERSION_RELEASE,
 )
 
 BLOCKSIZELOG_MAX = lib.ZSTD_BLOCKSIZELOG_MAX
 BLOCKSIZE_MAX = lib.ZSTD_BLOCKSIZE_MAX
 WINDOWLOG_MIN = lib.ZSTD_WINDOWLOG_MIN
 WINDOWLOG_MAX = lib.ZSTD_WINDOWLOG_MAX
 CHAINLOG_MIN = lib.ZSTD_CHAINLOG_MIN
 CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX
 HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN
 HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX
 HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX
 MINMATCH_MIN = lib.ZSTD_MINMATCH_MIN
 MINMATCH_MAX = lib.ZSTD_MINMATCH_MAX
 SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN
 SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX
 SEARCHLENGTH_MIN = lib.ZSTD_MINMATCH_MIN
 SEARCHLENGTH_MAX = lib.ZSTD_MINMATCH_MAX
 TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN
 TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX
 LDM_MINMATCH_MIN = lib.ZSTD_LDM_MINMATCH_MIN
 LDM_MINMATCH_MAX = lib.ZSTD_LDM_MINMATCH_MAX
 LDM_BUCKETSIZELOG_MAX = lib.ZSTD_LDM_BUCKETSIZELOG_MAX
 
 STRATEGY_FAST = lib.ZSTD_fast
 STRATEGY_DFAST = lib.ZSTD_dfast
 STRATEGY_GREEDY = lib.ZSTD_greedy
 STRATEGY_LAZY = lib.ZSTD_lazy
 STRATEGY_LAZY2 = lib.ZSTD_lazy2
 STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2
 STRATEGY_BTOPT = lib.ZSTD_btopt
 STRATEGY_BTULTRA = lib.ZSTD_btultra
 STRATEGY_BTULTRA2 = lib.ZSTD_btultra2
 
 DICT_TYPE_AUTO = lib.ZSTD_dct_auto
 DICT_TYPE_RAWCONTENT = lib.ZSTD_dct_rawContent
 DICT_TYPE_FULLDICT = lib.ZSTD_dct_fullDict
 
 FORMAT_ZSTD1 = lib.ZSTD_f_zstd1
 FORMAT_ZSTD1_MAGICLESS = lib.ZSTD_f_zstd1_magicless
 
 FLUSH_BLOCK = 0
 FLUSH_FRAME = 1
 
 COMPRESSOBJ_FLUSH_FINISH = 0
 COMPRESSOBJ_FLUSH_BLOCK = 1
 
 
 def _cpu_count():
     # os.cpu_count() was introducd in Python 3.4.
     try:
         return os.cpu_count() or 0
     except AttributeError:
         pass
 
     # Linux.
     try:
         if sys.version_info[0] == 2:
             return os.sysconf(b"SC_NPROCESSORS_ONLN")
         else:
             return os.sysconf("SC_NPROCESSORS_ONLN")
     except (AttributeError, ValueError):
         pass
 
     # TODO implement on other platforms.
     return 0
 
 
 class ZstdError(Exception):
     pass
 
 
 def _zstd_error(zresult):
     # Resolves to bytes on Python 2 and 3. We use the string for formatting
     # into error messages, which will be literal unicode. So convert it to
     # unicode.
     return ffi.string(lib.ZSTD_getErrorName(zresult)).decode("utf-8")
 
 
 def _make_cctx_params(params):
     res = lib.ZSTD_createCCtxParams()
     if res == ffi.NULL:
         raise MemoryError()
 
     res = ffi.gc(res, lib.ZSTD_freeCCtxParams)
 
     attrs = [
         (lib.ZSTD_c_format, params.format),
         (lib.ZSTD_c_compressionLevel, params.compression_level),
         (lib.ZSTD_c_windowLog, params.window_log),
         (lib.ZSTD_c_hashLog, params.hash_log),
         (lib.ZSTD_c_chainLog, params.chain_log),
         (lib.ZSTD_c_searchLog, params.search_log),
         (lib.ZSTD_c_minMatch, params.min_match),
         (lib.ZSTD_c_targetLength, params.target_length),
         (lib.ZSTD_c_strategy, params.compression_strategy),
         (lib.ZSTD_c_contentSizeFlag, params.write_content_size),
         (lib.ZSTD_c_checksumFlag, params.write_checksum),
         (lib.ZSTD_c_dictIDFlag, params.write_dict_id),
         (lib.ZSTD_c_nbWorkers, params.threads),
         (lib.ZSTD_c_jobSize, params.job_size),
         (lib.ZSTD_c_overlapLog, params.overlap_log),
         (lib.ZSTD_c_forceMaxWindow, params.force_max_window),
         (lib.ZSTD_c_enableLongDistanceMatching, params.enable_ldm),
         (lib.ZSTD_c_ldmHashLog, params.ldm_hash_log),
         (lib.ZSTD_c_ldmMinMatch, params.ldm_min_match),
         (lib.ZSTD_c_ldmBucketSizeLog, params.ldm_bucket_size_log),
         (lib.ZSTD_c_ldmHashRateLog, params.ldm_hash_rate_log),
     ]
 
     for param, value in attrs:
         _set_compression_parameter(res, param, value)
 
     return res
 
 
 class ZstdCompressionParameters(object):
     @staticmethod
     def from_level(level, source_size=0, dict_size=0, **kwargs):
         params = lib.ZSTD_getCParams(level, source_size, dict_size)
 
         args = {
             "window_log": "windowLog",
             "chain_log": "chainLog",
             "hash_log": "hashLog",
             "search_log": "searchLog",
             "min_match": "minMatch",
             "target_length": "targetLength",
             "compression_strategy": "strategy",
         }
 
         for arg, attr in args.items():
             if arg not in kwargs:
                 kwargs[arg] = getattr(params, attr)
 
         return ZstdCompressionParameters(**kwargs)
 
     def __init__(
         self,
         format=0,
         compression_level=0,
         window_log=0,
         hash_log=0,
         chain_log=0,
         search_log=0,
         min_match=0,
         target_length=0,
         strategy=-1,
         compression_strategy=-1,
         write_content_size=1,
         write_checksum=0,
         write_dict_id=0,
         job_size=0,
         overlap_log=-1,
         overlap_size_log=-1,
         force_max_window=0,
         enable_ldm=0,
         ldm_hash_log=0,
         ldm_min_match=0,
         ldm_bucket_size_log=0,
         ldm_hash_rate_log=-1,
         ldm_hash_every_log=-1,
         threads=0,
     ):
 
         params = lib.ZSTD_createCCtxParams()
         if params == ffi.NULL:
             raise MemoryError()
 
         params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
         self._params = params
 
         if threads < 0:
             threads = _cpu_count()
 
         # We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
         # because setting ZSTD_c_nbWorkers resets the other parameters.
         _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads)
 
         _set_compression_parameter(params, lib.ZSTD_c_format, format)
         _set_compression_parameter(
             params, lib.ZSTD_c_compressionLevel, compression_level
         )
         _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log)
         _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log)
         _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
         _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
         _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
-        _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_targetLength, target_length
+        )
 
         if strategy != -1 and compression_strategy != -1:
-            raise ValueError("cannot specify both compression_strategy and strategy")
+            raise ValueError(
+                "cannot specify both compression_strategy and strategy"
+            )
 
         if compression_strategy != -1:
             strategy = compression_strategy
         elif strategy == -1:
             strategy = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy)
         _set_compression_parameter(
             params, lib.ZSTD_c_contentSizeFlag, write_content_size
         )
-        _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_checksumFlag, write_checksum
+        )
         _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
         _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
 
         if overlap_log != -1 and overlap_size_log != -1:
-            raise ValueError("cannot specify both overlap_log and overlap_size_log")
+            raise ValueError(
+                "cannot specify both overlap_log and overlap_size_log"
+            )
 
         if overlap_size_log != -1:
             overlap_log = overlap_size_log
         elif overlap_log == -1:
             overlap_log = 0
 
         _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
-        _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_forceMaxWindow, force_max_window
+        )
         _set_compression_parameter(
             params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm
         )
         _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
-        _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_ldmMinMatch, ldm_min_match
+        )
         _set_compression_parameter(
             params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log
         )
 
         if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1:
             raise ValueError(
                 "cannot specify both ldm_hash_rate_log and ldm_hash_every_log"
             )
 
         if ldm_hash_every_log != -1:
             ldm_hash_rate_log = ldm_hash_every_log
         elif ldm_hash_rate_log == -1:
             ldm_hash_rate_log = 0
 
-        _set_compression_parameter(params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log)
+        _set_compression_parameter(
+            params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log
+        )
 
     @property
     def format(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_format)
 
     @property
     def compression_level(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_compressionLevel)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_compressionLevel
+        )
 
     @property
     def window_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_windowLog)
 
     @property
     def hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_hashLog)
 
     @property
     def chain_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_chainLog)
 
     @property
     def search_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_searchLog)
 
     @property
     def min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_minMatch)
 
     @property
     def target_length(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_targetLength)
 
     @property
     def compression_strategy(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_strategy)
 
     @property
     def write_content_size(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_contentSizeFlag)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_contentSizeFlag
+        )
 
     @property
     def write_checksum(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_checksumFlag)
 
     @property
     def write_dict_id(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_dictIDFlag)
 
     @property
     def job_size(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_jobSize)
 
     @property
     def overlap_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_overlapLog)
 
     @property
     def overlap_size_log(self):
         return self.overlap_log
 
     @property
     def force_max_window(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_forceMaxWindow)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_forceMaxWindow
+        )
 
     @property
     def enable_ldm(self):
         return _get_compression_parameter(
             self._params, lib.ZSTD_c_enableLongDistanceMatching
         )
 
     @property
     def ldm_hash_log(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashLog)
 
     @property
     def ldm_min_match(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_ldmMinMatch)
 
     @property
     def ldm_bucket_size_log(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmBucketSizeLog)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_ldmBucketSizeLog
+        )
 
     @property
     def ldm_hash_rate_log(self):
-        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashRateLog)
+        return _get_compression_parameter(
+            self._params, lib.ZSTD_c_ldmHashRateLog
+        )
 
     @property
     def ldm_hash_every_log(self):
         return self.ldm_hash_rate_log
 
     @property
     def threads(self):
         return _get_compression_parameter(self._params, lib.ZSTD_c_nbWorkers)
 
     def estimated_compression_context_size(self):
         return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params)
 
 
 CompressionParameters = ZstdCompressionParameters
 
 
 def estimate_decompression_context_size():
     return lib.ZSTD_estimateDCtxSize()
 
 
 def _set_compression_parameter(params, param, value):
     zresult = lib.ZSTD_CCtxParams_setParameter(params, param, value)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
-            "unable to set compression context parameter: %s" % _zstd_error(zresult)
+            "unable to set compression context parameter: %s"
+            % _zstd_error(zresult)
         )
 
 
 def _get_compression_parameter(params, param):
     result = ffi.new("int *")
 
     zresult = lib.ZSTD_CCtxParams_getParameter(params, param, result)
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
-            "unable to get compression context parameter: %s" % _zstd_error(zresult)
+            "unable to get compression context parameter: %s"
+            % _zstd_error(zresult)
         )
 
     return result[0]
 
 
 class ZstdCompressionWriter(object):
-    def __init__(self, compressor, writer, source_size, write_size, write_return_read):
+    def __init__(
+        self, compressor, writer, source_size, write_size, write_return_read
+    ):
         self._compressor = compressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
 
         self._dst_buffer = ffi.new("char[]", write_size)
         self._out_buffer = ffi.new("ZSTD_outBuffer *")
         self._out_buffer.dst = self._dst_buffer
         self._out_buffer.size = len(self._dst_buffer)
         self._out_buffer.pos = 0
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx, source_size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
     def __enter__(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._entered:
             raise ZstdError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
 
         if not exc_type and not exc_value and not exc_tb:
             self.close()
 
         self._compressor = None
 
         return False
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._compressor._cctx)
 
     def fileno(self):
         f = getattr(self._writer, "fileno", None)
         if f:
             return f()
         else:
             raise OSError("fileno not available on underlying writer")
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush(FLUSH_FRAME)
         finally:
             self._closed = True
 
         # Call close() on underlying stream as well.
         f = getattr(self._writer, "close", None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise NotImplementedError("writelines() is not yet implemented")
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         data_buffer = ffi.from_buffer(data)
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_compressStream2(
-                self._compressor._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
+                self._compressor._cctx,
+                out_buffer,
+                in_buffer,
+                lib.ZSTD_e_continue,
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
     def flush(self, flush_mode=FLUSH_BLOCK):
         if flush_mode == FLUSH_BLOCK:
             flush = lib.ZSTD_e_flush
         elif flush_mode == FLUSH_FRAME:
             flush = lib.ZSTD_e_end
         else:
             raise ValueError("unknown flush_mode: %r" % flush_mode)
 
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, out_buffer, in_buffer, flush
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
             if not zresult:
                 break
 
         return total_write
 
     def tell(self):
         return self._bytes_compressed
 
 
 class ZstdCompressionObj(object):
     def compress(self, data):
         if self._finished:
             raise ZstdError("cannot call compress() after compressor finished")
 
         data_buffer = ffi.from_buffer(data)
         source = ffi.new("ZSTD_inBuffer *")
         source.src = data_buffer
         source.size = len(data_buffer)
         source.pos = 0
 
         chunks = []
 
         while source.pos < len(data):
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, source, lib.ZSTD_e_continue
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
         return b"".join(chunks)
 
     def flush(self, flush_mode=COMPRESSOBJ_FLUSH_FINISH):
-        if flush_mode not in (COMPRESSOBJ_FLUSH_FINISH, COMPRESSOBJ_FLUSH_BLOCK):
+        if flush_mode not in (
+            COMPRESSOBJ_FLUSH_FINISH,
+            COMPRESSOBJ_FLUSH_BLOCK,
+        ):
             raise ValueError("flush mode not recognized")
 
         if self._finished:
             raise ZstdError("compressor object already finished")
 
         if flush_mode == COMPRESSOBJ_FLUSH_BLOCK:
             z_flush_mode = lib.ZSTD_e_flush
         elif flush_mode == COMPRESSOBJ_FLUSH_FINISH:
             z_flush_mode = lib.ZSTD_e_end
             self._finished = True
         else:
             raise ZstdError("unhandled flush mode")
 
         assert self._out.pos == 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, in_buffer, z_flush_mode
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if self._out.pos:
                 chunks.append(ffi.buffer(self._out.dst, self._out.pos)[:])
                 self._out.pos = 0
 
             if not zresult:
                 break
 
         return b"".join(chunks)
 
 
 class ZstdCompressionChunker(object):
     def __init__(self, compressor, chunk_size):
         self._compressor = compressor
         self._out = ffi.new("ZSTD_outBuffer *")
         self._dst_buffer = ffi.new("char[]", chunk_size)
         self._out.dst = self._dst_buffer
         self._out.size = chunk_size
         self._out.pos = 0
 
         self._in = ffi.new("ZSTD_inBuffer *")
         self._in.src = ffi.NULL
         self._in.size = 0
         self._in.pos = 0
         self._finished = False
 
     def compress(self, data):
         if self._finished:
             raise ZstdError("cannot call compress() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
                 "cannot perform operation before consuming output "
                 "from previous operation"
             )
 
         data_buffer = ffi.from_buffer(data)
 
         if not len(data_buffer):
             return
 
         self._in.src = data_buffer
         self._in.size = len(data_buffer)
         self._in.pos = 0
 
         while self._in.pos < self._in.size:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_continue
             )
 
             if self._in.pos == self._in.size:
                 self._in.src = ffi.NULL
                 self._in.size = 0
                 self._in.pos = 0
 
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos == self._out.size:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
     def flush(self):
         if self._finished:
             raise ZstdError("cannot call flush() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
-                "cannot call flush() before consuming output from " "previous operation"
+                "cannot call flush() before consuming output from "
+                "previous operation"
             )
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_flush
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 return
 
     def finish(self):
         if self._finished:
             raise ZstdError("cannot call finish() after compression finished")
 
         if self._in.src != ffi.NULL:
             raise ZstdError(
                 "cannot call finish() before consuming output from "
                 "previous operation"
             )
 
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._compressor._cctx, self._out, self._in, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd compress error: %s" % _zstd_error(zresult)
+                )
 
             if self._out.pos:
                 yield ffi.buffer(self._out.dst, self._out.pos)[:]
                 self._out.pos = 0
 
             if not zresult:
                 self._finished = True
                 return
 
 
 class ZstdCompressionReader(object):
     def __init__(self, compressor, source, read_size):
         self._compressor = compressor
         self._source = source
         self._read_size = read_size
         self._entered = False
         self._closed = False
         self._bytes_compressed = 0
         self._finished_input = False
         self._finished_output = False
 
         self._in_buffer = ffi.new("ZSTD_inBuffer *")
         # Holds a ref so backing bytes in self._in_buffer stay alive.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._compressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return False
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise OSError("stream is not writable")
 
     def writelines(self, ignored):
         raise OSError("stream is not writable")
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_compressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b"".join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         if self._finished_input:
             return
 
         if hasattr(self._source, "read"):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _compress_into_buffer(self, out_buffer):
         if self._in_buffer.pos >= self._in_buffer.size:
             return
 
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
-            self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_continue
+            self._compressor._cctx,
+            out_buffer,
+            self._in_buffer,
+            lib.ZSTD_e_continue,
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, "read"):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("zstd compress error: %s", _zstd_error(zresult))
 
         return out_buffer.pos and out_buffer.pos == out_buffer.size
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if size == -1:
             return self.readall()
 
         if self._finished_output or size == 0:
             return b""
 
         # Need a dedicated ref to dest buffer otherwise it gets collected.
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         # EOF
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+            raise ZstdError(
+                "error ending compression stream: %s", _zstd_error(zresult)
+            )
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if self._finished_output or size == 0:
             return b""
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to the
         # underlying stream to get input. However, we can't satisfy this
         # restriction with compression because not all input generates output.
         # It is possible to perform a block flush in order to ensure output.
         # But this may not be desirable behavior. So we allow multiple read()
         # to the underlying stream. But unlike read(), we stop once we have
         # any output.
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
 
             # If we've filled the output buffer, return immediately.
             if self._compress_into_buffer(out_buffer):
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # If we've populated the output buffer and we're not at EOF,
             # also return, as we've satisfied the read1() limits.
             if out_buffer.pos and not self._finished_input:
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
             # Else if we're at EOS and we have room left in the buffer,
             # fall through to below and try to add more data to the output.
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s" % _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         if self._compress_into_buffer(out_buffer):
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error ending compression stream: %s", _zstd_error(zresult))
+            raise ZstdError(
+                "error ending compression stream: %s", _zstd_error(zresult)
+            )
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._compress_into_buffer(out_buffer)
         if out_buffer.pos:
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
 
             if self._compress_into_buffer(out_buffer):
                 return out_buffer.pos
 
             if out_buffer.pos and not self._finished_input:
                 return out_buffer.pos
 
         # EOF.
         old_pos = out_buffer.pos
 
         zresult = lib.ZSTD_compressStream2(
             self._compressor._cctx, out_buffer, self._in_buffer, lib.ZSTD_e_end
         )
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
                 "error ending compression stream: %s" % _zstd_error(zresult)
             )
 
         if zresult == 0:
             self._finished_output = True
 
         return out_buffer.pos
 
 
 class ZstdCompressor(object):
     def __init__(
         self,
         level=3,
         dict_data=None,
         compression_params=None,
         write_checksum=None,
         write_content_size=None,
         write_dict_id=None,
         threads=0,
     ):
         if level > lib.ZSTD_maxCLevel():
-            raise ValueError("level must be less than %d" % lib.ZSTD_maxCLevel())
+            raise ValueError(
+                "level must be less than %d" % lib.ZSTD_maxCLevel()
+            )
 
         if threads < 0:
             threads = _cpu_count()
 
         if compression_params and write_checksum is not None:
-            raise ValueError("cannot define compression_params and " "write_checksum")
+            raise ValueError(
+                "cannot define compression_params and " "write_checksum"
+            )
 
         if compression_params and write_content_size is not None:
             raise ValueError(
                 "cannot define compression_params and " "write_content_size"
             )
 
         if compression_params and write_dict_id is not None:
-            raise ValueError("cannot define compression_params and " "write_dict_id")
+            raise ValueError(
+                "cannot define compression_params and " "write_dict_id"
+            )
 
         if compression_params and threads:
             raise ValueError("cannot define compression_params and threads")
 
         if compression_params:
             self._params = _make_cctx_params(compression_params)
         else:
             if write_dict_id is None:
                 write_dict_id = True
 
             params = lib.ZSTD_createCCtxParams()
             if params == ffi.NULL:
                 raise MemoryError()
 
             self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
-            _set_compression_parameter(self._params, lib.ZSTD_c_compressionLevel, level)
+            _set_compression_parameter(
+                self._params, lib.ZSTD_c_compressionLevel, level
+            )
 
             _set_compression_parameter(
                 self._params,
                 lib.ZSTD_c_contentSizeFlag,
                 write_content_size if write_content_size is not None else 1,
             )
 
             _set_compression_parameter(
-                self._params, lib.ZSTD_c_checksumFlag, 1 if write_checksum else 0
+                self._params,
+                lib.ZSTD_c_checksumFlag,
+                1 if write_checksum else 0,
             )
 
             _set_compression_parameter(
                 self._params, lib.ZSTD_c_dictIDFlag, 1 if write_dict_id else 0
             )
 
             if threads:
-                _set_compression_parameter(self._params, lib.ZSTD_c_nbWorkers, threads)
+                _set_compression_parameter(
+                    self._params, lib.ZSTD_c_nbWorkers, threads
+                )
 
         cctx = lib.ZSTD_createCCtx()
         if cctx == ffi.NULL:
             raise MemoryError()
 
         self._cctx = cctx
         self._dict_data = dict_data
 
         # We defer setting up garbage collection until after calling
         # _setup_cctx() to ensure the memory size estimate is more accurate.
         try:
             self._setup_cctx()
         finally:
             self._cctx = ffi.gc(
                 cctx, lib.ZSTD_freeCCtx, size=lib.ZSTD_sizeof_CCtx(cctx)
             )
 
     def _setup_cctx(self):
-        zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(self._cctx, self._params)
+        zresult = lib.ZSTD_CCtx_setParametersUsingCCtxParams(
+            self._cctx, self._params
+        )
         if lib.ZSTD_isError(zresult):
             raise ZstdError(
-                "could not set compression parameters: %s" % _zstd_error(zresult)
+                "could not set compression parameters: %s"
+                % _zstd_error(zresult)
             )
 
         dict_data = self._dict_data
 
         if dict_data:
             if dict_data._cdict:
                 zresult = lib.ZSTD_CCtx_refCDict(self._cctx, dict_data._cdict)
             else:
                 zresult = lib.ZSTD_CCtx_loadDictionary_advanced(
                     self._cctx,
                     dict_data.as_bytes(),
                     len(dict_data),
                     lib.ZSTD_dlm_byRef,
                     dict_data._dict_type,
                 )
 
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
-                    "could not load compression dictionary: %s" % _zstd_error(zresult)
+                    "could not load compression dictionary: %s"
+                    % _zstd_error(zresult)
                 )
 
     def memory_size(self):
         return lib.ZSTD_sizeof_CCtx(self._cctx)
 
     def compress(self, data):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         data_buffer = ffi.from_buffer(data)
 
         dest_size = lib.ZSTD_compressBound(len(data_buffer))
         out = new_nonzero("char[]", dest_size)
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, len(data_buffer))
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         in_buffer = ffi.new("ZSTD_inBuffer *")
 
         out_buffer.dst = out
         out_buffer.size = dest_size
         out_buffer.pos = 0
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_compressStream2(
             self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
         )
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("cannot compress: %s" % _zstd_error(zresult))
         elif zresult:
             raise ZstdError("unexpected partial frame flush")
 
         return ffi.buffer(out, out_buffer.pos)[:]
 
     def compressobj(self, size=-1):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         cobj = ZstdCompressionObj()
         cobj._out = ffi.new("ZSTD_outBuffer *")
-        cobj._dst_buffer = ffi.new("char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+        cobj._dst_buffer = ffi.new(
+            "char[]", COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+        )
         cobj._out.dst = cobj._dst_buffer
         cobj._out.size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
         cobj._out.pos = 0
         cobj._compressor = self
         cobj._finished = False
 
         return cobj
 
     def chunker(self, size=-1, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         return ZstdCompressionChunker(self, chunk_size=chunk_size)
 
     def copy_stream(
         self,
         ifh,
         ofh,
         size=-1,
         read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
 
         if not hasattr(ifh, "read"):
             raise ValueError("first argument must have a read() method")
         if not hasattr(ofh, "write"):
             raise ValueError("second argument must have a write() method")
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd compress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
         # We've finished reading. Flush the compressor.
         while True:
             zresult = lib.ZSTD_compressStream2(
                 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
             if zresult == 0:
                 break
 
         return total_read, total_write
 
     def stream_reader(
         self, source, size=-1, read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE
     ):
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         try:
             size = len(source)
         except Exception:
             pass
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         return ZstdCompressionReader(self, source, read_size)
 
     def stream_writer(
         self,
         writer,
         size=-1,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         write_return_read=False,
     ):
 
         if not hasattr(writer, "write"):
             raise ValueError("must pass an object with a write() method")
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
-        return ZstdCompressionWriter(self, writer, size, write_size, write_return_read)
+        return ZstdCompressionWriter(
+            self, writer, size, write_size, write_return_read
+        )
 
     write_to = stream_writer
 
     def read_to_iter(
         self,
         reader,
         size=-1,
         read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
         if hasattr(reader, "read"):
             have_read = True
         elif hasattr(reader, "__getitem__"):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError(
                 "must pass an object with a read() method or "
                 "conforms to buffer protocol"
             )
 
         lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
         zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._cctx, size)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("error setting source size: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "error setting source size: %s" % _zstd_error(zresult)
+            )
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         in_buffer.src = ffi.NULL
         in_buffer.size = 0
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         while True:
             # We should never have output data sitting around after a previous
             # iteration.
             assert out_buffer.pos == 0
 
             # Collect input data.
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = len(reader) - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset : buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input data. Break out of the read loop.
             if not read_result:
                 break
 
             # Feed all read data into the compressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 zresult = lib.ZSTD_compressStream2(
                     self._cctx, out_buffer, in_buffer, lib.ZSTD_e_continue
                 )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd compress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd compress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
             assert out_buffer.pos == 0
 
             # And repeat the loop to collect more data.
             continue
 
         # If we get here, input is exhausted. End the stream and emit what
         # remains.
         while True:
             assert out_buffer.pos == 0
             zresult = lib.ZSTD_compressStream2(
                 self._cctx, out_buffer, in_buffer, lib.ZSTD_e_end
             )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "error ending compression stream: %s" % _zstd_error(zresult)
                 )
 
             if out_buffer.pos:
                 data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                 out_buffer.pos = 0
                 yield data
 
             if zresult == 0:
                 break
 
     read_from = read_to_iter
 
     def frame_progression(self):
         progression = lib.ZSTD_getFrameProgression(self._cctx)
 
         return progression.ingested, progression.consumed, progression.produced
 
 
 class FrameParameters(object):
     def __init__(self, fparams):
         self.content_size = fparams.frameContentSize
         self.window_size = fparams.windowSize
         self.dict_id = fparams.dictID
         self.has_checksum = bool(fparams.checksumFlag)
 
 
 def frame_content_size(data):
     data_buffer = ffi.from_buffer(data)
 
     size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
 
     if size == lib.ZSTD_CONTENTSIZE_ERROR:
         raise ZstdError("error when determining content size")
     elif size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
         return -1
     else:
         return size
 
 
 def frame_header_size(data):
     data_buffer = ffi.from_buffer(data)
 
     zresult = lib.ZSTD_frameHeaderSize(data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
         raise ZstdError(
             "could not determine frame header size: %s" % _zstd_error(zresult)
         )
 
     return zresult
 
 
 def get_frame_parameters(data):
     params = ffi.new("ZSTD_frameHeader *")
 
     data_buffer = ffi.from_buffer(data)
     zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer))
     if lib.ZSTD_isError(zresult):
-        raise ZstdError("cannot get frame parameters: %s" % _zstd_error(zresult))
+        raise ZstdError(
+            "cannot get frame parameters: %s" % _zstd_error(zresult)
+        )
 
     if zresult:
-        raise ZstdError("not enough data for frame parameters; need %d bytes" % zresult)
+        raise ZstdError(
+            "not enough data for frame parameters; need %d bytes" % zresult
+        )
 
     return FrameParameters(params[0])
 
 
 class ZstdCompressionDict(object):
     def __init__(self, data, dict_type=DICT_TYPE_AUTO, k=0, d=0):
         assert isinstance(data, bytes_type)
         self._data = data
         self.k = k
         self.d = d
 
-        if dict_type not in (DICT_TYPE_AUTO, DICT_TYPE_RAWCONTENT, DICT_TYPE_FULLDICT):
+        if dict_type not in (
+            DICT_TYPE_AUTO,
+            DICT_TYPE_RAWCONTENT,
+            DICT_TYPE_FULLDICT,
+        ):
             raise ValueError(
-                "invalid dictionary load mode: %d; must use " "DICT_TYPE_* constants"
+                "invalid dictionary load mode: %d; must use "
+                "DICT_TYPE_* constants"
             )
 
         self._dict_type = dict_type
         self._cdict = None
 
     def __len__(self):
         return len(self._data)
 
     def dict_id(self):
         return int_type(lib.ZDICT_getDictID(self._data, len(self._data)))
 
     def as_bytes(self):
         return self._data
 
     def precompute_compress(self, level=0, compression_params=None):
         if level and compression_params:
-            raise ValueError("must only specify one of level or " "compression_params")
+            raise ValueError(
+                "must only specify one of level or " "compression_params"
+            )
 
         if not level and not compression_params:
             raise ValueError("must specify one of level or compression_params")
 
         if level:
             cparams = lib.ZSTD_getCParams(level, 0, len(self._data))
         else:
             cparams = ffi.new("ZSTD_compressionParameters")
             cparams.chainLog = compression_params.chain_log
             cparams.hashLog = compression_params.hash_log
             cparams.minMatch = compression_params.min_match
             cparams.searchLog = compression_params.search_log
             cparams.strategy = compression_params.compression_strategy
             cparams.targetLength = compression_params.target_length
             cparams.windowLog = compression_params.window_log
 
         cdict = lib.ZSTD_createCDict_advanced(
             self._data,
             len(self._data),
             lib.ZSTD_dlm_byRef,
             self._dict_type,
             cparams,
             lib.ZSTD_defaultCMem,
         )
         if cdict == ffi.NULL:
             raise ZstdError("unable to precompute dictionary")
 
         self._cdict = ffi.gc(
             cdict, lib.ZSTD_freeCDict, size=lib.ZSTD_sizeof_CDict(cdict)
         )
 
     @property
     def _ddict(self):
         ddict = lib.ZSTD_createDDict_advanced(
             self._data,
             len(self._data),
             lib.ZSTD_dlm_byRef,
             self._dict_type,
             lib.ZSTD_defaultCMem,
         )
 
         if ddict == ffi.NULL:
             raise ZstdError("could not create decompression dict")
 
-        ddict = ffi.gc(ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict))
+        ddict = ffi.gc(
+            ddict, lib.ZSTD_freeDDict, size=lib.ZSTD_sizeof_DDict(ddict)
+        )
         self.__dict__["_ddict"] = ddict
 
         return ddict
 
 
 def train_dictionary(
     dict_size,
     samples,
     k=0,
     d=0,
     notifications=0,
     dict_id=0,
     level=0,
     steps=0,
     threads=0,
 ):
     if not isinstance(samples, list):
         raise TypeError("samples must be a list")
 
     if threads < 0:
         threads = _cpu_count()
 
     total_size = sum(map(len, samples))
 
     samples_buffer = new_nonzero("char[]", total_size)
     sample_sizes = new_nonzero("size_t[]", len(samples))
 
     offset = 0
     for i, sample in enumerate(samples):
         if not isinstance(sample, bytes_type):
             raise ValueError("samples must be bytes")
 
         l = len(sample)
         ffi.memmove(samples_buffer + offset, sample, l)
         offset += l
         sample_sizes[i] = l
 
     dict_data = new_nonzero("char[]", dict_size)
 
     dparams = ffi.new("ZDICT_cover_params_t *")[0]
     dparams.k = k
     dparams.d = d
     dparams.steps = steps
     dparams.nbThreads = threads
     dparams.zParams.notificationLevel = notifications
     dparams.zParams.dictID = dict_id
     dparams.zParams.compressionLevel = level
 
     if (
         not dparams.k
         and not dparams.d
         and not dparams.steps
         and not dparams.nbThreads
         and not dparams.zParams.notificationLevel
         and not dparams.zParams.dictID
         and not dparams.zParams.compressionLevel
     ):
         zresult = lib.ZDICT_trainFromBuffer(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
         )
     elif dparams.steps or dparams.nbThreads:
         zresult = lib.ZDICT_optimizeTrainFromBuffer_cover(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
             ffi.addressof(dparams),
         )
     else:
         zresult = lib.ZDICT_trainFromBuffer_cover(
             ffi.addressof(dict_data),
             dict_size,
             ffi.addressof(samples_buffer),
             ffi.addressof(sample_sizes, 0),
             len(samples),
             dparams,
         )
 
     if lib.ZDICT_isError(zresult):
         msg = ffi.string(lib.ZDICT_getErrorName(zresult)).decode("utf-8")
         raise ZstdError("cannot train dict: %s" % msg)
 
     return ZstdCompressionDict(
         ffi.buffer(dict_data, zresult)[:],
         dict_type=DICT_TYPE_FULLDICT,
         k=dparams.k,
         d=dparams.d,
     )
 
 
 class ZstdDecompressionObj(object):
     def __init__(self, decompressor, write_size):
         self._decompressor = decompressor
         self._write_size = write_size
         self._finished = False
 
     def decompress(self, data):
         if self._finished:
             raise ZstdError("cannot use a decompressobj multiple times")
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         data_buffer = ffi.from_buffer(data)
 
         if len(data_buffer) == 0:
             return b""
 
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         chunks = []
 
         while True:
             zresult = lib.ZSTD_decompressStream(
                 self._decompressor._dctx, out_buffer, in_buffer
             )
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd decompressor error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd decompressor error: %s" % _zstd_error(zresult)
+                )
 
             if zresult == 0:
                 self._finished = True
                 self._decompressor = None
 
             if out_buffer.pos:
                 chunks.append(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
 
             if zresult == 0 or (
                 in_buffer.pos == in_buffer.size and out_buffer.pos == 0
             ):
                 break
 
             out_buffer.pos = 0
 
         return b"".join(chunks)
 
     def flush(self, length=0):
         pass
 
 
 class ZstdDecompressionReader(object):
     def __init__(self, decompressor, source, read_size, read_across_frames):
         self._decompressor = decompressor
         self._source = source
         self._read_size = read_size
         self._read_across_frames = bool(read_across_frames)
         self._entered = False
         self._closed = False
         self._bytes_decompressed = 0
         self._finished_input = False
         self._finished_output = False
         self._in_buffer = ffi.new("ZSTD_inBuffer *")
         # Holds a ref to self._in_buffer.src.
         self._source_buffer = None
 
     def __enter__(self):
         if self._entered:
             raise ValueError("cannot __enter__ multiple times")
 
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self._closed = True
         self._source = None
         self._decompressor = None
 
         return False
 
     def readable(self):
         return True
 
     def writable(self):
         return False
 
     def seekable(self):
         return True
 
     def readline(self):
         raise io.UnsupportedOperation()
 
     def readlines(self):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         raise io.UnsupportedOperation()
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def isatty(self):
         return False
 
     def flush(self):
         return None
 
     def close(self):
         self._closed = True
         return None
 
     @property
     def closed(self):
         return self._closed
 
     def tell(self):
         return self._bytes_decompressed
 
     def readall(self):
         chunks = []
 
         while True:
             chunk = self.read(1048576)
             if not chunk:
                 break
 
             chunks.append(chunk)
 
         return b"".join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
 
     def __next__(self):
         raise io.UnsupportedOperation()
 
     next = __next__
 
     def _read_input(self):
         # We have data left over in the input buffer. Use it.
         if self._in_buffer.pos < self._in_buffer.size:
             return
 
         # All input data exhausted. Nothing to do.
         if self._finished_input:
             return
 
         # Else populate the input buffer from our source.
         if hasattr(self._source, "read"):
             data = self._source.read(self._read_size)
 
             if not data:
                 self._finished_input = True
                 return
 
             self._source_buffer = ffi.from_buffer(data)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
         else:
             self._source_buffer = ffi.from_buffer(self._source)
             self._in_buffer.src = self._source_buffer
             self._in_buffer.size = len(self._source_buffer)
             self._in_buffer.pos = 0
 
     def _decompress_into_buffer(self, out_buffer):
         """Decompress available input into an output buffer.
 
         Returns True if data in output buffer should be emitted.
         """
         zresult = lib.ZSTD_decompressStream(
             self._decompressor._dctx, out_buffer, self._in_buffer
         )
 
         if self._in_buffer.pos == self._in_buffer.size:
             self._in_buffer.src = ffi.NULL
             self._in_buffer.pos = 0
             self._in_buffer.size = 0
             self._source_buffer = None
 
             if not hasattr(self._source, "read"):
                 self._finished_input = True
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
 
         # Emit data if there is data AND either:
         # a) output buffer is full (read amount is satisfied)
         # b) we're at end of a frame and not in frame spanning mode
         return out_buffer.pos and (
             out_buffer.pos == out_buffer.size
             or zresult == 0
             and not self._read_across_frames
         )
 
     def read(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if size == -1:
             # This is recursive. But it gets the job done.
             return self.readall()
 
         if self._finished_output or size == 0:
             return b""
 
         # We /could/ call into readinto() here. But that introduces more
         # overhead.
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         self._read_input()
         if self._decompress_into_buffer(out_buffer):
             self._bytes_decompressed += out_buffer.pos
             return out_buffer.pos
 
         while not self._finished_input:
             self._read_input()
             if self._decompress_into_buffer(out_buffer):
                 self._bytes_decompressed += out_buffer.pos
                 return out_buffer.pos
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def read1(self, size=-1):
         if self._closed:
             raise ValueError("stream is closed")
 
         if size < -1:
             raise ValueError("cannot read negative amounts less than -1")
 
         if self._finished_output or size == 0:
             return b""
 
         # -1 returns arbitrary number of bytes.
         if size == -1:
             size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new("char[]", size)
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dst_buffer
         out_buffer.size = size
         out_buffer.pos = 0
 
         # read1() dictates that we can perform at most 1 call to underlying
         # stream to get input. However, we can't satisfy this restriction with
         # decompression because not all input generates output. So we allow
         # multiple read(). But unlike read(), we stop once we have any output.
         while not self._finished_input:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
     def readinto1(self, b):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._finished_output:
             return 0
 
         # TODO use writable=True once we require CFFI >= 1.12.
         dest_buffer = ffi.from_buffer(b)
         ffi.memmove(b, b"", 0)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = dest_buffer
         out_buffer.size = len(dest_buffer)
         out_buffer.pos = 0
 
         while not self._finished_input and not self._finished_output:
             self._read_input()
             self._decompress_into_buffer(out_buffer)
 
             if out_buffer.pos:
                 break
 
         self._bytes_decompressed += out_buffer.pos
         return out_buffer.pos
 
     def seek(self, pos, whence=os.SEEK_SET):
         if self._closed:
             raise ValueError("stream is closed")
 
         read_amount = 0
 
         if whence == os.SEEK_SET:
             if pos < 0:
-                raise ValueError("cannot seek to negative position with SEEK_SET")
+                raise ValueError(
+                    "cannot seek to negative position with SEEK_SET"
+                )
 
             if pos < self._bytes_decompressed:
-                raise ValueError("cannot seek zstd decompression stream " "backwards")
+                raise ValueError(
+                    "cannot seek zstd decompression stream " "backwards"
+                )
 
             read_amount = pos - self._bytes_decompressed
 
         elif whence == os.SEEK_CUR:
             if pos < 0:
-                raise ValueError("cannot seek zstd decompression stream " "backwards")
+                raise ValueError(
+                    "cannot seek zstd decompression stream " "backwards"
+                )
 
             read_amount = pos
         elif whence == os.SEEK_END:
             raise ValueError(
                 "zstd decompression streams cannot be seeked " "with SEEK_END"
             )
 
         while read_amount:
-            result = self.read(min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+            result = self.read(
+                min(read_amount, DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE)
+            )
 
             if not result:
                 break
 
             read_amount -= len(result)
 
         return self._bytes_decompressed
 
 
 class ZstdDecompressionWriter(object):
     def __init__(self, decompressor, writer, write_size, write_return_read):
         decompressor._ensure_dctx()
 
         self._decompressor = decompressor
         self._writer = writer
         self._write_size = write_size
         self._write_return_read = bool(write_return_read)
         self._entered = False
         self._closed = False
 
     def __enter__(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         if self._entered:
             raise ZstdError("cannot __enter__ multiple times")
 
         self._entered = True
 
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
         self.close()
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._decompressor._dctx)
 
     def close(self):
         if self._closed:
             return
 
         try:
             self.flush()
         finally:
             self._closed = True
 
         f = getattr(self._writer, "close", None)
         if f:
             f()
 
     @property
     def closed(self):
         return self._closed
 
     def fileno(self):
         f = getattr(self._writer, "fileno", None)
         if f:
             return f()
         else:
             raise OSError("fileno not available on underlying writer")
 
     def flush(self):
         if self._closed:
             raise ValueError("stream is closed")
 
         f = getattr(self._writer, "flush", None)
         if f:
             return f()
 
     def isatty(self):
         return False
 
     def readable(self):
         return False
 
     def readline(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readlines(self, hint=-1):
         raise io.UnsupportedOperation()
 
     def seek(self, offset, whence=None):
         raise io.UnsupportedOperation()
 
     def seekable(self):
         return False
 
     def tell(self):
         raise io.UnsupportedOperation()
 
     def truncate(self, size=None):
         raise io.UnsupportedOperation()
 
     def writable(self):
         return True
 
     def writelines(self, lines):
         raise io.UnsupportedOperation()
 
     def read(self, size=-1):
         raise io.UnsupportedOperation()
 
     def readall(self):
         raise io.UnsupportedOperation()
 
     def readinto(self, b):
         raise io.UnsupportedOperation()
 
     def write(self, data):
         if self._closed:
             raise ValueError("stream is closed")
 
         total_write = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         data_buffer = ffi.from_buffer(data)
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         dst_buffer = ffi.new("char[]", self._write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         dctx = self._decompressor._dctx
 
         while in_buffer.pos < in_buffer.size:
             zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
-                raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+                raise ZstdError(
+                    "zstd decompress error: %s" % _zstd_error(zresult)
+                )
 
             if out_buffer.pos:
-                self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
+                self._writer.write(
+                    ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+                )
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
         if self._write_return_read:
             return in_buffer.pos
         else:
             return total_write
 
 
 class ZstdDecompressor(object):
     def __init__(self, dict_data=None, max_window_size=0, format=FORMAT_ZSTD1):
         self._dict_data = dict_data
         self._max_window_size = max_window_size
         self._format = format
 
         dctx = lib.ZSTD_createDCtx()
         if dctx == ffi.NULL:
             raise MemoryError()
 
         self._dctx = dctx
 
         # Defer setting up garbage collection until full state is loaded so
         # the memory size is more accurate.
         try:
             self._ensure_dctx()
         finally:
             self._dctx = ffi.gc(
                 dctx, lib.ZSTD_freeDCtx, size=lib.ZSTD_sizeof_DCtx(dctx)
             )
 
     def memory_size(self):
         return lib.ZSTD_sizeof_DCtx(self._dctx)
 
     def decompress(self, data, max_output_size=0):
         self._ensure_dctx()
 
         data_buffer = ffi.from_buffer(data)
 
-        output_size = lib.ZSTD_getFrameContentSize(data_buffer, len(data_buffer))
+        output_size = lib.ZSTD_getFrameContentSize(
+            data_buffer, len(data_buffer)
+        )
 
         if output_size == lib.ZSTD_CONTENTSIZE_ERROR:
             raise ZstdError("error determining content size from frame header")
         elif output_size == 0:
             return b""
         elif output_size == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             if not max_output_size:
-                raise ZstdError("could not determine content size in frame header")
+                raise ZstdError(
+                    "could not determine content size in frame header"
+                )
 
             result_buffer = ffi.new("char[]", max_output_size)
             result_size = max_output_size
             output_size = 0
         else:
             result_buffer = ffi.new("char[]", output_size)
             result_size = output_size
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = result_buffer
         out_buffer.size = result_size
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError("decompression error: %s" % _zstd_error(zresult))
         elif zresult:
-            raise ZstdError("decompression error: did not decompress full frame")
+            raise ZstdError(
+                "decompression error: did not decompress full frame"
+            )
         elif output_size and out_buffer.pos != output_size:
             raise ZstdError(
                 "decompression error: decompressed %d bytes; expected %d"
                 % (zresult, output_size)
             )
 
         return ffi.buffer(result_buffer, out_buffer.pos)[:]
 
     def stream_reader(
         self,
         source,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         read_across_frames=False,
     ):
         self._ensure_dctx()
-        return ZstdDecompressionReader(self, source, read_size, read_across_frames)
+        return ZstdDecompressionReader(
+            self, source, read_size, read_across_frames
+        )
 
     def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if write_size < 1:
             raise ValueError("write_size must be positive")
 
         self._ensure_dctx()
         return ZstdDecompressionObj(self, write_size=write_size)
 
     def read_to_iter(
         self,
         reader,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         skip_bytes=0,
     ):
         if skip_bytes >= read_size:
             raise ValueError("skip_bytes must be smaller than read_size")
 
         if hasattr(reader, "read"):
             have_read = True
         elif hasattr(reader, "__getitem__"):
             have_read = False
             buffer_offset = 0
             size = len(reader)
         else:
             raise ValueError(
                 "must pass an object with a read() method or "
                 "conforms to buffer protocol"
             )
 
         if skip_bytes:
             if have_read:
                 reader.read(skip_bytes)
             else:
                 if skip_bytes > size:
                     raise ValueError("skip_bytes larger than first input chunk")
 
                 buffer_offset = skip_bytes
 
         self._ensure_dctx()
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = len(dst_buffer)
         out_buffer.pos = 0
 
         while True:
             assert out_buffer.pos == 0
 
             if have_read:
                 read_result = reader.read(read_size)
             else:
                 remaining = size - buffer_offset
                 slice_size = min(remaining, read_size)
                 read_result = reader[buffer_offset : buffer_offset + slice_size]
                 buffer_offset += slice_size
 
             # No new input. Break out of read loop.
             if not read_result:
                 break
 
             # Feed all read data into decompressor and emit output until
             # exhausted.
             read_buffer = ffi.from_buffer(read_result)
             in_buffer.src = read_buffer
             in_buffer.size = len(read_buffer)
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
                 assert out_buffer.pos == 0
 
-                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(
+                    self._dctx, out_buffer, in_buffer
+                )
                 if lib.ZSTD_isError(zresult):
-                    raise ZstdError("zstd decompress error: %s" % _zstd_error(zresult))
+                    raise ZstdError(
+                        "zstd decompress error: %s" % _zstd_error(zresult)
+                    )
 
                 if out_buffer.pos:
                     data = ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
                     out_buffer.pos = 0
                     yield data
 
                 if zresult == 0:
                     return
 
             # Repeat loop to collect more input data.
             continue
 
         # If we get here, input is exhausted.
 
     read_from = read_to_iter
 
     def stream_writer(
         self,
         writer,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
         write_return_read=False,
     ):
         if not hasattr(writer, "write"):
             raise ValueError("must pass an object with a write() method")
 
-        return ZstdDecompressionWriter(self, writer, write_size, write_return_read)
+        return ZstdDecompressionWriter(
+            self, writer, write_size, write_return_read
+        )
 
     write_to = stream_writer
 
     def copy_stream(
         self,
         ifh,
         ofh,
         read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
         write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
     ):
         if not hasattr(ifh, "read"):
             raise ValueError("first argument must have a read() method")
         if not hasattr(ofh, "write"):
             raise ValueError("second argument must have a write() method")
 
         self._ensure_dctx()
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         out_buffer = ffi.new("ZSTD_outBuffer *")
 
         dst_buffer = ffi.new("char[]", write_size)
         out_buffer.dst = dst_buffer
         out_buffer.size = write_size
         out_buffer.pos = 0
 
         total_read, total_write = 0, 0
 
         # Read all available input.
         while True:
             data = ifh.read(read_size)
             if not data:
                 break
 
             data_buffer = ffi.from_buffer(data)
             total_read += len(data_buffer)
             in_buffer.src = data_buffer
             in_buffer.size = len(data_buffer)
             in_buffer.pos = 0
 
             # Flush all read data to output.
             while in_buffer.pos < in_buffer.size:
-                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(
+                    self._dctx, out_buffer, in_buffer
+                )
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError(
                         "zstd decompressor error: %s" % _zstd_error(zresult)
                     )
 
                 if out_buffer.pos:
                     ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos))
                     total_write += out_buffer.pos
                     out_buffer.pos = 0
 
             # Continue loop to keep reading.
 
         return total_read, total_write
 
     def decompress_content_dict_chain(self, frames):
         if not isinstance(frames, list):
             raise TypeError("argument must be a list")
 
         if not frames:
             raise ValueError("empty input chain")
 
         # First chunk should not be using a dictionary. We handle it specially.
         chunk = frames[0]
         if not isinstance(chunk, bytes_type):
             raise ValueError("chunk 0 must be bytes")
 
         # All chunks should be zstd frames and should have content size set.
         chunk_buffer = ffi.from_buffer(chunk)
         params = ffi.new("ZSTD_frameHeader *")
-        zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+        zresult = lib.ZSTD_getFrameHeader(
+            params, chunk_buffer, len(chunk_buffer)
+        )
         if lib.ZSTD_isError(zresult):
             raise ValueError("chunk 0 is not a valid zstd frame")
         elif zresult:
             raise ValueError("chunk 0 is too small to contain a zstd frame")
 
         if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
             raise ValueError("chunk 0 missing content size in frame")
 
         self._ensure_dctx(load_dict=False)
 
         last_buffer = ffi.new("char[]", params.frameContentSize)
 
         out_buffer = ffi.new("ZSTD_outBuffer *")
         out_buffer.dst = last_buffer
         out_buffer.size = len(last_buffer)
         out_buffer.pos = 0
 
         in_buffer = ffi.new("ZSTD_inBuffer *")
         in_buffer.src = chunk_buffer
         in_buffer.size = len(chunk_buffer)
         in_buffer.pos = 0
 
         zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("could not decompress chunk 0: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "could not decompress chunk 0: %s" % _zstd_error(zresult)
+            )
         elif zresult:
             raise ZstdError("chunk 0 did not decompress full frame")
 
         # Special case of chain length of 1
         if len(frames) == 1:
             return ffi.buffer(last_buffer, len(last_buffer))[:]
 
         i = 1
         while i < len(frames):
             chunk = frames[i]
             if not isinstance(chunk, bytes_type):
                 raise ValueError("chunk %d must be bytes" % i)
 
             chunk_buffer = ffi.from_buffer(chunk)
-            zresult = lib.ZSTD_getFrameHeader(params, chunk_buffer, len(chunk_buffer))
+            zresult = lib.ZSTD_getFrameHeader(
+                params, chunk_buffer, len(chunk_buffer)
+            )
             if lib.ZSTD_isError(zresult):
                 raise ValueError("chunk %d is not a valid zstd frame" % i)
             elif zresult:
-                raise ValueError("chunk %d is too small to contain a zstd frame" % i)
+                raise ValueError(
+                    "chunk %d is too small to contain a zstd frame" % i
+                )
 
             if params.frameContentSize == lib.ZSTD_CONTENTSIZE_UNKNOWN:
                 raise ValueError("chunk %d missing content size in frame" % i)
 
             dest_buffer = ffi.new("char[]", params.frameContentSize)
 
             out_buffer.dst = dest_buffer
             out_buffer.size = len(dest_buffer)
             out_buffer.pos = 0
 
             in_buffer.src = chunk_buffer
             in_buffer.size = len(chunk_buffer)
             in_buffer.pos = 0
 
-            zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
+            zresult = lib.ZSTD_decompressStream(
+                self._dctx, out_buffer, in_buffer
+            )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "could not decompress chunk %d: %s" % _zstd_error(zresult)
                 )
             elif zresult:
                 raise ZstdError("chunk %d did not decompress full frame" % i)
 
             last_buffer = dest_buffer
             i += 1
 
         return ffi.buffer(last_buffer, len(last_buffer))[:]
 
     def _ensure_dctx(self, load_dict=True):
         lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
 
         if self._max_window_size:
-            zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx, self._max_window_size)
+            zresult = lib.ZSTD_DCtx_setMaxWindowSize(
+                self._dctx, self._max_window_size
+            )
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
                     "unable to set max window size: %s" % _zstd_error(zresult)
                 )
 
         zresult = lib.ZSTD_DCtx_setFormat(self._dctx, self._format)
         if lib.ZSTD_isError(zresult):
-            raise ZstdError("unable to set decoding format: %s" % _zstd_error(zresult))
+            raise ZstdError(
+                "unable to set decoding format: %s" % _zstd_error(zresult)
+            )
 
         if self._dict_data and load_dict:
             zresult = lib.ZSTD_DCtx_refDDict(self._dctx, self._dict_data._ddict)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError(
-                    "unable to reference prepared dictionary: %s" % _zstd_error(zresult)
+                    "unable to reference prepared dictionary: %s"
+                    % _zstd_error(zresult)
                 )

doc/Makefile

@@ -1,48 +1,48 b''
 SOURCES=$(notdir $(wildcard ../mercurial/helptext/*.[0-9].txt))
 MAN=$(SOURCES:%.txt=%)
 HTML=$(SOURCES:%.txt=%.html)
 GENDOC=gendoc.py ../mercurial/commands.py ../mercurial/help.py \
 	../mercurial/helptext/*.txt ../hgext/*.py ../hgext/*/__init__.py
 PREFIX=/usr/local
 MANDIR=$(PREFIX)/share/man
-INSTALL=install -c -m 644
+INSTALL=install -m 644
 PYTHON?=python
 RSTARGS=
 
 export HGENCODING=UTF-8
 
 all: man html
 
 man: $(MAN)
 
 html: $(HTML)
 
 # This logic is duplicated in setup.py:hgbuilddoc()
 common.txt $(SOURCES) $(SOURCES:%.txt=%.gendoc.txt): $(GENDOC)
 	${PYTHON} gendoc.py "$(basename $@)" > $@.tmp
 	mv $@.tmp $@
 
 %: %.txt %.gendoc.txt common.txt
 	$(PYTHON) runrst hgmanpage $(RSTARGS) --halt warning \
 	  --strip-elements-with-class htmlonly $*.txt $*
 
 %.html: %.txt %.gendoc.txt common.txt
 	$(PYTHON) runrst html $(RSTARGS) --halt warning \
 	  --link-stylesheet --stylesheet-path style.css $*.txt $*.html
 
 MANIFEST: man html
 # tracked files are already in the main MANIFEST
 	$(RM) $@
 	for i in $(MAN) $(HTML); do \
 	  echo "doc/$$i" >> $@ ; \
 	done
 
 install: man
 	for i in $(MAN) ; do \
 	  subdir=`echo $$i | sed -n 's/^.*\.\([0-9]\)$$/man\1/p'` ; \
 	  mkdir -p "$(DESTDIR)$(MANDIR)"/$$subdir ; \
 	  $(INSTALL) $$i "$(DESTDIR)$(MANDIR)"/$$subdir ; \
 	done
 
 clean:
 	$(RM) $(MAN) $(HTML) common.txt $(SOURCES) $(SOURCES:%.txt=%.gendoc.txt) MANIFEST