merge: default into stable for release candidate

contrib/automation/README.rst

@@ -0,0 +1,127 b''
+====================
+Mercurial Automation
+====================
+
+This directory contains code and utilities for building and testing Mercurial
+on remote machines.
+
+The ``automation.py`` Script
+============================
+
+``automation.py`` is an executable Python script (requires Python 3.5+)
+that serves as a driver to common automation tasks.
+
+When executed, the script will *bootstrap* a virtualenv in
+``<source-root>/build/venv-automation`` then re-execute itself using
+that virtualenv. So there is no need for the caller to have a virtualenv
+explicitly activated. This virtualenv will be populated with various
+dependencies (as defined by the ``requirements.txt`` file).
+
+To see what you can do with this script, simply run it::
+
+   $ ./automation.py
+
+Local State
+===========
+
+By default, local state required to interact with remote servers is stored
+in the ``~/.hgautomation`` directory.
+
+We attempt to limit persistent state to this directory. Even when
+performing tasks that may have side-effects, we try to limit those
+side-effects so they don't impact the local system. e.g. when we SSH
+into a remote machine, we create a temporary directory for the SSH
+config so the user's known hosts file isn't updated.
+
+AWS Integration
+===============
+
+Various automation tasks integrate with AWS to provide access to
+resources such as EC2 instances for generic compute.
+
+This obviously requires an AWS account and credentials to work.
+
+We use the ``boto3`` library for interacting with AWS APIs. We do not employ
+any special functionality for telling ``boto3`` where to find AWS credentials. See
+https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+for how ``boto3`` works. Once you have configured your environment such
+that ``boto3`` can find credentials, interaction with AWS should *just work*.
+
+.. hint::
+
+   Typically you have a ``~/.aws/credentials`` file containing AWS
+   credentials. If you manage multiple credentials, you can override which
+   *profile* to use at run-time by setting the ``AWS_PROFILE`` environment
+   variable.
+
+Resource Management
+-------------------
+
+Depending on the task being performed, various AWS services will be accessed.
+This of course requires AWS credentials with permissions to access these
+services.
+
+The following AWS services can be accessed by automation tasks:
+
+* EC2
+* IAM
+* Simple Systems Manager (SSM)
+
+Various resources will also be created as part of performing various tasks.
+This also requires various permissions.
+
+The following AWS resources can be created by automation tasks:
+
+* EC2 key pairs
+* EC2 security groups
+* EC2 instances
+* IAM roles and instance profiles
+* SSM command invocations
+
+When possible, we prefix resource names with ``hg-`` so they can easily
+be identified as belonging to Mercurial.
+
+.. important::
+
+   We currently assume that AWS accounts utilized by *us* are single
+   tenancy. Attempts to have discrete users of ``automation.py`` (including
+   sharing credentials across machines) using the same AWS account can result
+   in them interfering with each other and things breaking.
+
+Cost of Operation
+-----------------
+
+``automation.py`` tries to be frugal with regards to utilization of remote
+resources. Persistent remote resources are minimized in order to keep costs
+in check. For example, EC2 instances are often ephemeral and only live as long
+as the operation being performed.
+
+Under normal operation, recurring costs are limited to:
+
+* Storage costs for AMI / EBS snapshots. This should be just a few pennies
+  per month.
+
+When running EC2 instances, you'll be billed accordingly. By default, we
+use *small* instances, like ``t3.medium``. This instance type costs ~$0.07 per
+hour.
+
+.. note::
+
+   When running Windows EC2 instances, AWS bills at the full hourly cost, even
+   if the instance doesn't run for a full hour (per-second billing doesn't
+   apply to Windows AMIs).
+
+Managing Remote Resources
+-------------------------
+
+Occassionally, there may be an error purging a temporary resource. Or you
+may wish to forcefully purge remote state. Commands can be invoked to manually
+purge remote resources.
+
+To terminate all EC2 instances that we manage::
+
+   $ automation.py terminate-ec2-instances
+
+To purge all EC2 resources that we manage::
+
+   $ automation.py purge-ec2-resources

contrib/automation/automation.py

@@ -0,0 +1,70 b''
+#!/usr/bin/env python3
+#
+# automation.py - Perform tasks on remote machines
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+import os
+import pathlib
+import subprocess
+import sys
+import venv
+
+
+HERE = pathlib.Path(os.path.abspath(__file__)).parent
+REQUIREMENTS_TXT = HERE / 'requirements.txt'
+SOURCE_DIR = HERE.parent.parent
+VENV = SOURCE_DIR / 'build' / 'venv-automation'
+
+
+def bootstrap():
+    venv_created = not VENV.exists()
+
+    VENV.parent.mkdir(exist_ok=True)
+
+    venv.create(VENV, with_pip=True)
+
+    if os.name == 'nt':
+        venv_bin = VENV / 'Scripts'
+        pip = venv_bin / 'pip.exe'
+        python = venv_bin / 'python.exe'
+    else:
+        venv_bin = VENV / 'bin'
+        pip = venv_bin / 'pip'
+        python = venv_bin / 'python'
+
+    args = [str(pip), 'install', '-r', str(REQUIREMENTS_TXT),
+            '--disable-pip-version-check']
+
+    if not venv_created:
+        args.append('-q')
+
+    subprocess.run(args, check=True)
+
+    os.environ['HGAUTOMATION_BOOTSTRAPPED'] = '1'
+    os.environ['PATH'] = '%s%s%s' % (
+        venv_bin, os.pathsep, os.environ['PATH'])
+
+    subprocess.run([str(python), __file__] + sys.argv[1:], check=True)
+
+
+def run():
+    import hgautomation.cli as cli
+
+    # Need to strip off main Python executable.
+    cli.main()
+
+
+if __name__ == '__main__':
+    try:
+        if 'HGAUTOMATION_BOOTSTRAPPED' not in os.environ:
+            bootstrap()
+        else:
+            run()
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+    except KeyboardInterrupt:
+        sys.exit(1)

contrib/automation/hgautomation/__init__.py

@@ -0,0 +1,59 b''
+# __init__.py - High-level automation interfaces
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import pathlib
+import secrets
+
+from .aws import (
+    AWSConnection,
+)
+
+
+class HGAutomation:
+    """High-level interface for Mercurial automation.
+
+    Holds global state, provides access to other primitives, etc.
+    """
+
+    def __init__(self, state_path: pathlib.Path):
+        self.state_path = state_path
+
+        state_path.mkdir(exist_ok=True)
+
+    def default_password(self):
+        """Obtain the default password to use for remote machines.
+
+        A new password will be generated if one is not stored.
+        """
+        p = self.state_path / 'default-password'
+
+        try:
+            with p.open('r', encoding='ascii') as fh:
+                data = fh.read().strip()
+
+                if data:
+                    return data
+
+        except FileNotFoundError:
+            pass
+
+        password = secrets.token_urlsafe(24)
+
+        with p.open('w', encoding='ascii') as fh:
+            fh.write(password)
+            fh.write('\n')
+
+        p.chmod(0o0600)
+
+        return password
+
+    def aws_connection(self, region: str):
+        """Obtain an AWSConnection instance bound to a specific region."""
+
+        return AWSConnection(self, region)

contrib/automation/hgautomation/aws.py

@@ -0,0 +1,879 b''
+# aws.py - Automation code for Amazon Web Services
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import contextlib
+import copy
+import hashlib
+import json
+import os
+import pathlib
+import subprocess
+import time
+
+import boto3
+import botocore.exceptions
+
+from .winrm import (
+    run_powershell,
+    wait_for_winrm,
+)
+
+
+SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent
+
+INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_ROOT / 'contrib' /
+                                'install-windows-dependencies.ps1')
+
+
+KEY_PAIRS = {
+    'automation',
+}
+
+
+SECURITY_GROUPS = {
+    'windows-dev-1': {
+        'description': 'Mercurial Windows instances that perform build automation',
+        'ingress': [
+            {
+                'FromPort': 22,
+                'ToPort': 22,
+                'IpProtocol': 'tcp',
+                'IpRanges': [
+                    {
+                        'CidrIp': '0.0.0.0/0',
+                        'Description': 'SSH from entire Internet',
+                    },
+                ],
+            },
+            {
+                'FromPort': 3389,
+                'ToPort': 3389,
+                'IpProtocol': 'tcp',
+                'IpRanges': [
+                    {
+                        'CidrIp': '0.0.0.0/0',
+                        'Description': 'RDP from entire Internet',
+                    },
+                ],
+
+            },
+            {
+                'FromPort': 5985,
+                'ToPort': 5986,
+                'IpProtocol': 'tcp',
+                'IpRanges': [
+                    {
+                        'CidrIp': '0.0.0.0/0',
+                        'Description': 'PowerShell Remoting (Windows Remote Management)',
+                    },
+                ],
+            }
+        ],
+    },
+}
+
+
+IAM_ROLES = {
+    'ephemeral-ec2-role-1': {
+        'description': 'Mercurial temporary EC2 instances',
+        'policy_arns': [
+            'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
+        ],
+    },
+}
+
+
+ASSUME_ROLE_POLICY_DOCUMENT = '''
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+'''.strip()
+
+
+IAM_INSTANCE_PROFILES = {
+    'ephemeral-ec2-1': {
+        'roles': [
+            'ephemeral-ec2-role-1',
+        ],
+    }
+}
+
+
+# User Data for Windows EC2 instance. Mainly used to set the password
+# and configure WinRM.
+# Inspired by the User Data script used by Packer
+# (from https://www.packer.io/intro/getting-started/build-image.html).
+WINDOWS_USER_DATA = r'''
+<powershell>
+
+# TODO enable this once we figure out what is failing.
+#$ErrorActionPreference = "stop"
+
+# Set administrator password
+net user Administrator "%s"
+wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE
+
+# First, make sure WinRM can't be connected to
+netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block
+
+# Delete any existing WinRM listeners
+winrm delete winrm/config/listener?Address=*+Transport=HTTP  2>$Null
+winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null
+
+# Create a new WinRM listener and configure
+winrm create winrm/config/listener?Address=*+Transport=HTTP
+winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
+winrm set winrm/config '@{MaxTimeoutms="7200000"}'
+winrm set winrm/config/service '@{AllowUnencrypted="true"}'
+winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}'
+winrm set winrm/config/service/auth '@{Basic="true"}'
+winrm set winrm/config/client/auth '@{Basic="true"}'
+
+# Configure UAC to allow privilege elevation in remote shells
+$Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System'
+$Setting = 'LocalAccountTokenFilterPolicy'
+Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
+
+# Configure and restart the WinRM Service; Enable the required firewall exception
+Stop-Service -Name WinRM
+Set-Service -Name WinRM -StartupType Automatic
+netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any
+Start-Service -Name WinRM
+
+# Disable firewall on private network interfaces so prompts don't appear.
+Set-NetFirewallProfile -Name private -Enabled false
+</powershell>
+'''.lstrip()
+
+
+WINDOWS_BOOTSTRAP_POWERSHELL = '''
+Write-Output "installing PowerShell dependencies"
+Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
+Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
+Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0
+
+Write-Output "installing OpenSSL server"
+Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+# Various tools will attempt to use older versions of .NET. So we enable
+# the feature that provides them so it doesn't have to be auto-enabled
+# later.
+Write-Output "enabling .NET Framework feature"
+Install-WindowsFeature -Name Net-Framework-Core
+'''
+
+
+class AWSConnection:
+    """Manages the state of a connection with AWS."""
+
+    def __init__(self, automation, region: str):
+        self.automation = automation
+        self.local_state_path = automation.state_path
+
+        self.prefix = 'hg-'
+
+        self.session = boto3.session.Session(region_name=region)
+        self.ec2client = self.session.client('ec2')
+        self.ec2resource = self.session.resource('ec2')
+        self.iamclient = self.session.client('iam')
+        self.iamresource = self.session.resource('iam')
+
+        ensure_key_pairs(automation.state_path, self.ec2resource)
+
+        self.security_groups = ensure_security_groups(self.ec2resource)
+        ensure_iam_state(self.iamresource)
+
+    def key_pair_path_private(self, name):
+        """Path to a key pair private key file."""
+        return self.local_state_path / 'keys' / ('keypair-%s' % name)
+
+    def key_pair_path_public(self, name):
+        return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
+
+
+def rsa_key_fingerprint(p: pathlib.Path):
+    """Compute the fingerprint of an RSA private key."""
+
+    # TODO use rsa package.
+    res = subprocess.run(
+        ['openssl', 'pkcs8', '-in', str(p), '-nocrypt', '-topk8',
+         '-outform', 'DER'],
+        capture_output=True,
+        check=True)
+
+    sha1 = hashlib.sha1(res.stdout).hexdigest()
+    return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2]))
+
+
+def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'):
+    remote_existing = {}
+
+    for kpi in ec2resource.key_pairs.all():
+        if kpi.name.startswith(prefix):
+            remote_existing[kpi.name[len(prefix):]] = kpi.key_fingerprint
+
+    # Validate that we have these keys locally.
+    key_path = state_path / 'keys'
+    key_path.mkdir(exist_ok=True, mode=0o700)
+
+    def remove_remote(name):
+        print('deleting key pair %s' % name)
+        key = ec2resource.KeyPair(name)
+        key.delete()
+
+    def remove_local(name):
+        pub_full = key_path / ('keypair-%s.pub' % name)
+        priv_full = key_path / ('keypair-%s' % name)
+
+        print('removing %s' % pub_full)
+        pub_full.unlink()
+        print('removing %s' % priv_full)
+        priv_full.unlink()
+
+    local_existing = {}
+
+    for f in sorted(os.listdir(key_path)):
+        if not f.startswith('keypair-') or not f.endswith('.pub'):
+            continue
+
+        name = f[len('keypair-'):-len('.pub')]
+
+        pub_full = key_path / f
+        priv_full = key_path / ('keypair-%s' % name)
+
+        with open(pub_full, 'r', encoding='ascii') as fh:
+            data = fh.read()
+
+        if not data.startswith('ssh-rsa '):
+            print('unexpected format for key pair file: %s; removing' %
+                  pub_full)
+            pub_full.unlink()
+            priv_full.unlink()
+            continue
+
+        local_existing[name] = rsa_key_fingerprint(priv_full)
+
+    for name in sorted(set(remote_existing) | set(local_existing)):
+        if name not in local_existing:
+            actual = '%s%s' % (prefix, name)
+            print('remote key %s does not exist locally' % name)
+            remove_remote(actual)
+            del remote_existing[name]
+
+        elif name not in remote_existing:
+            print('local key %s does not exist remotely' % name)
+            remove_local(name)
+            del local_existing[name]
+
+        elif remote_existing[name] != local_existing[name]:
+            print('key fingerprint mismatch for %s; '
+                  'removing from local and remote' % name)
+            remove_local(name)
+            remove_remote('%s%s' % (prefix, name))
+            del local_existing[name]
+            del remote_existing[name]
+
+    missing = KEY_PAIRS - set(remote_existing)
+
+    for name in sorted(missing):
+        actual = '%s%s' % (prefix, name)
+        print('creating key pair %s' % actual)
+
+        priv_full = key_path / ('keypair-%s' % name)
+        pub_full = key_path / ('keypair-%s.pub' % name)
+
+        kp = ec2resource.create_key_pair(KeyName=actual)
+
+        with priv_full.open('w', encoding='ascii') as fh:
+            fh.write(kp.key_material)
+            fh.write('\n')
+
+        priv_full.chmod(0o0600)
+
+        # SSH public key can be extracted via `ssh-keygen`.
+        with pub_full.open('w', encoding='ascii') as fh:
+            subprocess.run(
+                ['ssh-keygen', '-y', '-f', str(priv_full)],
+                stdout=fh,
+                check=True)
+
+        pub_full.chmod(0o0600)
+
+
+def delete_instance_profile(profile):
+    for role in profile.roles:
+        print('removing role %s from instance profile %s' % (role.name,
+                                                             profile.name))
+        profile.remove_role(RoleName=role.name)
+
+    print('deleting instance profile %s' % profile.name)
+    profile.delete()
+
+
+def ensure_iam_state(iamresource, prefix='hg-'):
+    """Ensure IAM state is in sync with our canonical definition."""
+
+    remote_profiles = {}
+
+    for profile in iamresource.instance_profiles.all():
+        if profile.name.startswith(prefix):
+            remote_profiles[profile.name[len(prefix):]] = profile
+
+    for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)):
+        delete_instance_profile(remote_profiles[name])
+        del remote_profiles[name]
+
+    remote_roles = {}
+
+    for role in iamresource.roles.all():
+        if role.name.startswith(prefix):
+            remote_roles[role.name[len(prefix):]] = role
+
+    for name in sorted(set(remote_roles) - set(IAM_ROLES)):
+        role = remote_roles[name]
+
+        print('removing role %s' % role.name)
+        role.delete()
+        del remote_roles[name]
+
+    # We've purged remote state that doesn't belong. Create missing
+    # instance profiles and roles.
+    for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
+        actual = '%s%s' % (prefix, name)
+        print('creating IAM instance profile %s' % actual)
+
+        profile = iamresource.create_instance_profile(
+            InstanceProfileName=actual)
+        remote_profiles[name] = profile
+
+    for name in sorted(set(IAM_ROLES) - set(remote_roles)):
+        entry = IAM_ROLES[name]
+
+        actual = '%s%s' % (prefix, name)
+        print('creating IAM role %s' % actual)
+
+        role = iamresource.create_role(
+            RoleName=actual,
+            Description=entry['description'],
+            AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
+        )
+
+        remote_roles[name] = role
+
+        for arn in entry['policy_arns']:
+            print('attaching policy %s to %s' % (arn, role.name))
+            role.attach_policy(PolicyArn=arn)
+
+    # Now reconcile state of profiles.
+    for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):
+        profile = remote_profiles[name]
+        wanted = {'%s%s' % (prefix, role) for role in meta['roles']}
+        have = {role.name for role in profile.roles}
+
+        for role in sorted(have - wanted):
+            print('removing role %s from %s' % (role, profile.name))
+            profile.remove_role(RoleName=role)
+
+        for role in sorted(wanted - have):
+            print('adding role %s to %s' % (role, profile.name))
+            profile.add_role(RoleName=role)
+
+
+def find_windows_server_2019_image(ec2resource):
+    """Find the Amazon published Windows Server 2019 base image."""
+
+    images = ec2resource.images.filter(
+        Filters=[
+            {
+                'Name': 'owner-alias',
+                'Values': ['amazon'],
+            },
+            {
+                'Name': 'state',
+                'Values': ['available'],
+            },
+            {
+                'Name': 'image-type',
+                'Values': ['machine'],
+            },
+            {
+                'Name': 'name',
+                'Values': ['Windows_Server-2019-English-Full-Base-2019.02.13'],
+            },
+        ])
+
+    for image in images:
+        return image
+
+    raise Exception('unable to find Windows Server 2019 image')
+
+
+def ensure_security_groups(ec2resource, prefix='hg-'):
+    """Ensure all necessary Mercurial security groups are present.
+
+    All security groups are prefixed with ``hg-`` by default. Any security
+    groups having this prefix but aren't in our list are deleted.
+    """
+    existing = {}
+
+    for group in ec2resource.security_groups.all():
+        if group.group_name.startswith(prefix):
+            existing[group.group_name[len(prefix):]] = group
+
+    purge = set(existing) - set(SECURITY_GROUPS)
+
+    for name in sorted(purge):
+        group = existing[name]
+        print('removing legacy security group: %s' % group.group_name)
+        group.delete()
+
+    security_groups = {}
+
+    for name, group in sorted(SECURITY_GROUPS.items()):
+        if name in existing:
+            security_groups[name] = existing[name]
+            continue
+
+        actual = '%s%s' % (prefix, name)
+        print('adding security group %s' % actual)
+
+        group_res = ec2resource.create_security_group(
+            Description=group['description'],
+            GroupName=actual,
+        )
+
+        group_res.authorize_ingress(
+            IpPermissions=group['ingress'],
+        )
+
+        security_groups[name] = group_res
+
+    return security_groups
+
+
+def terminate_ec2_instances(ec2resource, prefix='hg-'):
+    """Terminate all EC2 instances managed by us."""
+    waiting = []
+
+    for instance in ec2resource.instances.all():
+        if instance.state['Name'] == 'terminated':
+            continue
+
+        for tag in instance.tags or []:
+            if tag['Key'] == 'Name' and tag['Value'].startswith(prefix):
+                print('terminating %s' % instance.id)
+                instance.terminate()
+                waiting.append(instance)
+
+    for instance in waiting:
+        instance.wait_until_terminated()
+
+
+def remove_resources(c, prefix='hg-'):
+    """Purge all of our resources in this EC2 region."""
+    ec2resource = c.ec2resource
+    iamresource = c.iamresource
+
+    terminate_ec2_instances(ec2resource, prefix=prefix)
+
+    for image in ec2resource.images.all():
+        if image.name.startswith(prefix):
+            remove_ami(ec2resource, image)
+
+    for group in ec2resource.security_groups.all():
+        if group.group_name.startswith(prefix):
+            print('removing security group %s' % group.group_name)
+            group.delete()
+
+    for profile in iamresource.instance_profiles.all():
+        if profile.name.startswith(prefix):
+            delete_instance_profile(profile)
+
+    for role in iamresource.roles.all():
+        if role.name.startswith(prefix):
+            print('removing role %s' % role.name)
+            role.delete()
+
+
+def wait_for_ip_addresses(instances):
+    """Wait for the public IP addresses of an iterable of instances."""
+    for instance in instances:
+        while True:
+            if not instance.public_ip_address:
+                time.sleep(2)
+                instance.reload()
+                continue
+
+            print('public IP address for %s: %s' % (
+                instance.id, instance.public_ip_address))
+            break
+
+
+def remove_ami(ec2resource, image):
+    """Remove an AMI and its underlying snapshots."""
+    snapshots = []
+
+    for device in image.block_device_mappings:
+        if 'Ebs' in device:
+            snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId']))
+
+    print('deregistering %s' % image.id)
+    image.deregister()
+
+    for snapshot in snapshots:
+        print('deleting snapshot %s' % snapshot.id)
+        snapshot.delete()
+
+
+def wait_for_ssm(ssmclient, instances):
+    """Wait for SSM to come online for an iterable of instance IDs."""
+    while True:
+        res = ssmclient.describe_instance_information(
+            Filters=[
+                {
+                    'Key': 'InstanceIds',
+                    'Values': [i.id for i in instances],
+                },
+            ],
+        )
+
+        available = len(res['InstanceInformationList'])
+        wanted = len(instances)
+
+        print('%d/%d instances available in SSM' % (available, wanted))
+
+        if available == wanted:
+            return
+
+        time.sleep(2)
+
+
+def run_ssm_command(ssmclient, instances, document_name, parameters):
+    """Run a PowerShell script on an EC2 instance."""
+
+    res = ssmclient.send_command(
+        InstanceIds=[i.id for i in instances],
+        DocumentName=document_name,
+        Parameters=parameters,
+        CloudWatchOutputConfig={
+            'CloudWatchOutputEnabled': True,
+        },
+    )
+
+    command_id = res['Command']['CommandId']
+
+    for instance in instances:
+        while True:
+            try:
+                res = ssmclient.get_command_invocation(
+                    CommandId=command_id,
+                    InstanceId=instance.id,
+                )
+            except botocore.exceptions.ClientError as e:
+                if e.response['Error']['Code'] == 'InvocationDoesNotExist':
+                    print('could not find SSM command invocation; waiting')
+                    time.sleep(1)
+                    continue
+                else:
+                    raise
+
+            if res['Status'] == 'Success':
+                break
+            elif res['Status'] in ('Pending', 'InProgress', 'Delayed'):
+                time.sleep(2)
+            else:
+                raise Exception('command failed on %s: %s' % (
+                    instance.id, res['Status']))
+
+
[e-mail]
+def temporary_ec2_instances(ec2resource, config):
+    """Create temporary EC2 instances.
+
+    This is a proxy to ``ec2client.run_instances(**config)`` that takes care of
+    managing the lifecycle of the instances.
+
+    When the context manager exits, the instances are terminated.
+
+    The context manager evaluates to the list of data structures
+    describing each created instance. The instances may not be available
+    for work immediately: it is up to the caller to wait for the instance
+    to start responding.
+    """
+
+    ids = None
+
+    try:
+        res = ec2resource.create_instances(**config)
+
+        ids = [i.id for i in res]
+        print('started instances: %s' % ' '.join(ids))
+
+        yield res
+    finally:
+        if ids:
+            print('terminating instances: %s' % ' '.join(ids))
+            for instance in res:
+                instance.terminate()
+            print('terminated %d instances' % len(ids))
+
+
[e-mail]
+def create_temp_windows_ec2_instances(c: AWSConnection, config):
+    """Create temporary Windows EC2 instances.
+
+    This is a higher-level wrapper around ``create_temp_ec2_instances()`` that
+    configures the Windows instance for Windows Remote Management. The emitted
+    instances will have a ``winrm_client`` attribute containing a
+    ``pypsrp.client.Client`` instance bound to the instance.
+    """
+    if 'IamInstanceProfile' in config:
+        raise ValueError('IamInstanceProfile cannot be provided in config')
+    if 'UserData' in config:
+        raise ValueError('UserData cannot be provided in config')
+
+    password = c.automation.default_password()
+
+    config = copy.deepcopy(config)
+    config['IamInstanceProfile'] = {
+        'Name': 'hg-ephemeral-ec2-1',
+    }
+    config.setdefault('TagSpecifications', []).append({
+        'ResourceType': 'instance',
+        'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}],
+    })
+    config['UserData'] = WINDOWS_USER_DATA % password
+
+    with temporary_ec2_instances(c.ec2resource, config) as instances:
+        wait_for_ip_addresses(instances)
+
+        print('waiting for Windows Remote Management service...')
+
+        for instance in instances:
+            client = wait_for_winrm(instance.public_ip_address, 'Administrator', password)
+            print('established WinRM connection to %s' % instance.id)
+            instance.winrm_client = client
+
+        yield instances
+
+
+def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'):
+    """Ensure Windows Development AMI is available and up-to-date.
+
+    If necessary, a modern AMI will be built by starting a temporary EC2
+    instance and bootstrapping it.
+
+    Obsolete AMIs will be deleted so there is only a single AMI having the
+    desired name.
+
+    Returns an ``ec2.Image`` of either an existing AMI or a newly-built
+    one.
+    """
+    ec2client = c.ec2client
+    ec2resource = c.ec2resource
+    ssmclient = c.session.client('ssm')
+
+    name = '%s%s' % (prefix, 'windows-dev')
+
+    config = {
+        'BlockDeviceMappings': [
+            {
+                'DeviceName': '/dev/sda1',
+                'Ebs': {
+                    'DeleteOnTermination': True,
+                    'VolumeSize': 32,
+                    'VolumeType': 'gp2',
+                },
+            }
+        ],
+        'ImageId': find_windows_server_2019_image(ec2resource).id,
+        'InstanceInitiatedShutdownBehavior': 'stop',
+        'InstanceType': 't3.medium',
+        'KeyName': '%sautomation' % prefix,
+        'MaxCount': 1,
+        'MinCount': 1,
+        'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
+    }
+
+    commands = [
+        # Need to start the service so sshd_config is generated.
+        'Start-Service sshd',
+        'Write-Output "modifying sshd_config"',
+        r'$content = Get-Content C:\ProgramData\ssh\sshd_config',
+        '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""',
+        r'$content | Set-Content C:\ProgramData\ssh\sshd_config',
+        'Import-Module OpenSSHUtils',
+        r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false',
+        'Restart-Service sshd',
+        'Write-Output "installing OpenSSL client"',
+        'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0',
+        'Set-Service -Name sshd -StartupType "Automatic"',
+        'Write-Output "OpenSSH server running"',
+    ]
+
+    with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh:
+        commands.extend(l.rstrip() for l in fh)
+
+    # Disable Windows Defender when bootstrapping because it just slows
+    # things down.
+    commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
+    commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
+
+    # Compute a deterministic fingerprint to determine whether image needs
+    # to be regenerated.
+    fingerprint = {
+        'instance_config': config,
+        'user_data': WINDOWS_USER_DATA,
+        'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
+        'bootstrap_commands': commands,
+    }
+
+    fingerprint = json.dumps(fingerprint, sort_keys=True)
+    fingerprint = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
+
+    # Find existing AMIs with this name and delete the ones that are invalid.
+    # Store a reference to a good image so it can be returned one the
+    # image state is reconciled.
+    images = ec2resource.images.filter(
+        Filters=[{'Name': 'name', 'Values': [name]}])
+
+    existing_image = None
+
+    for image in images:
+        if image.tags is None:
+            print('image %s for %s lacks required tags; removing' % (
+                image.id, image.name))
+            remove_ami(ec2resource, image)
+        else:
+            tags = {t['Key']: t['Value'] for t in image.tags}
+
+            if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
+                existing_image = image
+            else:
+                print('image %s for %s has wrong fingerprint; removing' % (
+                      image.id, image.name))
+                remove_ami(ec2resource, image)
+
+    if existing_image:
+        return existing_image
+
+    print('no suitable Windows development image found; creating one...')
+
+    with create_temp_windows_ec2_instances(c, config) as instances:
+        assert len(instances) == 1
+        instance = instances[0]
+
+        wait_for_ssm(ssmclient, [instance])
+
+        # On first boot, install various Windows updates.
+        # We would ideally use PowerShell Remoting for this. However, there are
+        # trust issues that make it difficult to invoke Windows Update
+        # remotely. So we use SSM, which has a mechanism for running Windows
+        # Update.
+        print('installing Windows features...')
+        run_ssm_command(
+            ssmclient,
+            [instance],
+            'AWS-RunPowerShellScript',
+            {
+                'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),
+            },
+        )
+
+        # Reboot so all updates are fully applied.
+        print('rebooting instance %s' % instance.id)
+        ec2client.reboot_instances(InstanceIds=[instance.id])
+
+        time.sleep(15)
+
+        print('waiting for Windows Remote Management to come back...')
+        client = wait_for_winrm(instance.public_ip_address, 'Administrator',
+                                c.automation.default_password())
+        print('established WinRM connection to %s' % instance.id)
+        instance.winrm_client = client
+
+        print('bootstrapping instance...')
+        run_powershell(instance.winrm_client, '\n'.join(commands))
+
+        print('bootstrap completed; stopping %s to create image' % instance.id)
+        instance.stop()
+
+        ec2client.get_waiter('instance_stopped').wait(
+            InstanceIds=[instance.id],
+            WaiterConfig={
+                'Delay': 5,
+            })
+        print('%s is stopped' % instance.id)
+
+        image = instance.create_image(
+            Name=name,
+            Description='Mercurial Windows development environment',
+        )
+
+        image.create_tags(Tags=[
+            {
+                'Key': 'HGIMAGEFINGERPRINT',
+                'Value': fingerprint,
+            },
+        ])
+
+        print('waiting for image %s' % image.id)
+
+        ec2client.get_waiter('image_available').wait(
+            ImageIds=[image.id],
+        )
+
+        print('image %s available as %s' % (image.id, image.name))
+
+        return image
+
+
[e-mail]
+def temporary_windows_dev_instances(c: AWSConnection, image, instance_type,
+                                    prefix='hg-', disable_antivirus=False):
+    """Create a temporary Windows development EC2 instance.
+
+    Context manager resolves to the list of ``EC2.Instance`` that were created.
+    """
+    config = {
+        'BlockDeviceMappings': [
+            {
+                'DeviceName': '/dev/sda1',
+                'Ebs': {
+                    'DeleteOnTermination': True,
+                    'VolumeSize': 32,
+                    'VolumeType': 'gp2',
+                },
+            }
+        ],
+        'ImageId': image.id,
+        'InstanceInitiatedShutdownBehavior': 'stop',
+        'InstanceType': instance_type,
+        'KeyName': '%sautomation' % prefix,
+        'MaxCount': 1,
+        'MinCount': 1,
+        'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
+    }
+
+    with create_temp_windows_ec2_instances(c, config) as instances:
+        if disable_antivirus:
+            for instance in instances:
+                run_powershell(
+                    instance.winrm_client,
+                    'Set-MpPreference -DisableRealtimeMonitoring $true')
+
+        yield instances

contrib/automation/hgautomation/cli.py

@@ -0,0 +1,273 b''
+# cli.py - Command line interface for automation
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import argparse
+import os
+import pathlib
+
+from . import (
+    aws,
+    HGAutomation,
+    windows,
+)
+
+
+SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent
+DIST_PATH = SOURCE_ROOT / 'dist'
+
+
+def bootstrap_windows_dev(hga: HGAutomation, aws_region):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+    print('Windows development AMI available as %s' % image.id)
+
+
+def build_inno(hga: HGAutomation, aws_region, arch, revision, version):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+    DIST_PATH.mkdir(exist_ok=True)
+
+    with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
+        instance = insts[0]
+
+        windows.synchronize_hg(SOURCE_ROOT, revision, instance)
+
+        for a in arch:
+            windows.build_inno_installer(instance.winrm_client, a,
+                                         DIST_PATH,
+                                         version=version)
+
+
+def build_wix(hga: HGAutomation, aws_region, arch, revision, version):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+    DIST_PATH.mkdir(exist_ok=True)
+
+    with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
+        instance = insts[0]
+
+        windows.synchronize_hg(SOURCE_ROOT, revision, instance)
+
+        for a in arch:
+            windows.build_wix_installer(instance.winrm_client, a,
+                                        DIST_PATH, version=version)
+
+
+def build_windows_wheel(hga: HGAutomation, aws_region, arch, revision):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+    DIST_PATH.mkdir(exist_ok=True)
+
+    with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
+        instance = insts[0]
+
+        windows.synchronize_hg(SOURCE_ROOT, revision, instance)
+
+        for a in arch:
+            windows.build_wheel(instance.winrm_client, a, DIST_PATH)
+
+
+def build_all_windows_packages(hga: HGAutomation, aws_region, revision):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+    DIST_PATH.mkdir(exist_ok=True)
+
+    with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
+        instance = insts[0]
+
+        winrm_client = instance.winrm_client
+
+        windows.synchronize_hg(SOURCE_ROOT, revision, instance)
+
+        for arch in ('x86', 'x64'):
+            windows.purge_hg(winrm_client)
+            windows.build_wheel(winrm_client, arch, DIST_PATH)
+            windows.purge_hg(winrm_client)
+            windows.build_inno_installer(winrm_client, arch, DIST_PATH)
+            windows.purge_hg(winrm_client)
+            windows.build_wix_installer(winrm_client, arch, DIST_PATH)
+
+
+def terminate_ec2_instances(hga: HGAutomation, aws_region):
+    c = hga.aws_connection(aws_region)
+    aws.terminate_ec2_instances(c.ec2resource)
+
+
+def purge_ec2_resources(hga: HGAutomation, aws_region):
+    c = hga.aws_connection(aws_region)
+    aws.remove_resources(c)
+
+
+def run_tests_windows(hga: HGAutomation, aws_region, instance_type,
+                      python_version, arch, test_flags):
+    c = hga.aws_connection(aws_region)
+    image = aws.ensure_windows_dev_ami(c)
+
+    with aws.temporary_windows_dev_instances(c, image, instance_type,
+                                             disable_antivirus=True) as insts:
+        instance = insts[0]
+
+        windows.synchronize_hg(SOURCE_ROOT, '.', instance)
+        windows.run_tests(instance.winrm_client, python_version, arch,
+                          test_flags)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        '--state-path',
+        default='~/.hgautomation',
+        help='Path for local state files',
+    )
+    parser.add_argument(
+        '--aws-region',
+        help='AWS region to use',
+        default='us-west-1',
+    )
+
+    subparsers = parser.add_subparsers()
+
+    sp = subparsers.add_parser(
+        'bootstrap-windows-dev',
+        help='Bootstrap the Windows development environment',
+    )
+    sp.set_defaults(func=bootstrap_windows_dev)
+
+    sp = subparsers.add_parser(
+        'build-all-windows-packages',
+        help='Build all Windows packages',
+    )
+    sp.add_argument(
+        '--revision',
+        help='Mercurial revision to build',
+        default='.',
+    )
+    sp.set_defaults(func=build_all_windows_packages)
+
+    sp = subparsers.add_parser(
+        'build-inno',
+        help='Build Inno Setup installer(s)',
+    )
+    sp.add_argument(
+        '--arch',
+        help='Architecture to build for',
+        choices={'x86', 'x64'},
+        nargs='*',
+        default=['x64'],
+    )
+    sp.add_argument(
+        '--revision',
+        help='Mercurial revision to build',
+        default='.',
+    )
+    sp.add_argument(
+        '--version',
+        help='Mercurial version string to use in installer',
+    )
+    sp.set_defaults(func=build_inno)
+
+    sp = subparsers.add_parser(
+        'build-windows-wheel',
+        help='Build Windows wheel(s)',
+    )
+    sp.add_argument(
+        '--arch',
+        help='Architecture to build for',
+        choices={'x86', 'x64'},
+        nargs='*',
+        default=['x64'],
+    )
+    sp.add_argument(
+        '--revision',
+        help='Mercurial revision to build',
+        default='.',
+    )
+    sp.set_defaults(func=build_windows_wheel)
+
+    sp = subparsers.add_parser(
+        'build-wix',
+        help='Build WiX installer(s)'
+    )
+    sp.add_argument(
+        '--arch',
+        help='Architecture to build for',
+        choices={'x86', 'x64'},
+        nargs='*',
+        default=['x64'],
+    )
+    sp.add_argument(
+        '--revision',
+        help='Mercurial revision to build',
+        default='.',
+    )
+    sp.add_argument(
+        '--version',
+        help='Mercurial version string to use in installer',
+    )
+    sp.set_defaults(func=build_wix)
+
+    sp = subparsers.add_parser(
+        'terminate-ec2-instances',
+        help='Terminate all active EC2 instances managed by us',
+    )
+    sp.set_defaults(func=terminate_ec2_instances)
+
+    sp = subparsers.add_parser(
+        'purge-ec2-resources',
+        help='Purge all EC2 resources managed by us',
+    )
+    sp.set_defaults(func=purge_ec2_resources)
+
+    sp = subparsers.add_parser(
+        'run-tests-windows',
+        help='Run tests on Windows',
+    )
+    sp.add_argument(
+        '--instance-type',
+        help='EC2 instance type to use',
+        default='t3.medium',
+    )
+    sp.add_argument(
+        '--python-version',
+        help='Python version to use',
+        choices={'2.7', '3.5', '3.6', '3.7', '3.8'},
+        default='2.7',
+    )
+    sp.add_argument(
+        '--arch',
+        help='Architecture to test',
+        choices={'x86', 'x64'},
+        default='x64',
+    )
+    sp.add_argument(
+        '--test-flags',
+        help='Extra command line flags to pass to run-tests.py',
+    )
+    sp.set_defaults(func=run_tests_windows)
+
+    return parser
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+
+    local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
+    automation = HGAutomation(local_state_path)
+
+    if not hasattr(args, 'func'):
+        parser.print_help()
+        return
+
+    kwargs = dict(vars(args))
+    del kwargs['func']
+    del kwargs['state_path']
+
+    args.func(automation, **kwargs)

contrib/automation/hgautomation/windows.py

@@ -0,0 +1,287 b''
+# windows.py - Automation specific to Windows
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import os
+import pathlib
+import re
+import subprocess
+import tempfile
+
+from .winrm import (
+    run_powershell,
+)
+
+
+# PowerShell commands to activate a Visual Studio 2008 environment.
+# This is essentially a port of vcvarsall.bat to PowerShell.
+ACTIVATE_VC9_AMD64 = r'''
+Write-Output "activating Visual Studio 2008 environment for AMD64"
+$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
+$Env:VCINSTALLDIR = "${root}\VC\"
+$Env:WindowsSdkDir = "${root}\WinSDK\"
+$Env:PATH = "${root}\VC\Bin\amd64;${root}\WinSDK\Bin\x64;${root}\WinSDK\Bin;$Env:PATH"
+$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:PATH"
+$Env:LIB = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIB"
+$Env:LIBPATH = "${root}\VC\Lib\amd64;${root}\WinSDK\Lib\x64;$Env:LIBPATH"
+'''.lstrip()
+
+ACTIVATE_VC9_X86 = r'''
+Write-Output "activating Visual Studio 2008 environment for x86"
+$root = "$env:LOCALAPPDATA\Programs\Common\Microsoft\Visual C++ for Python\9.0"
+$Env:VCINSTALLDIR = "${root}\VC\"
+$Env:WindowsSdkDir = "${root}\WinSDK\"
+$Env:PATH = "${root}\VC\Bin;${root}\WinSDK\Bin;$Env:PATH"
+$Env:INCLUDE = "${root}\VC\Include;${root}\WinSDK\Include;$Env:INCLUDE"
+$Env:LIB = "${root}\VC\Lib;${root}\WinSDK\Lib;$Env:LIB"
+$Env:LIBPATH = "${root}\VC\lib;${root}\WinSDK\Lib:$Env:LIBPATH"
+'''.lstrip()
+
+HG_PURGE = r'''
+$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
+Set-Location C:\hgdev\src
+hg.exe --config extensions.purge= purge --all
+if ($LASTEXITCODE -ne 0) {
+    throw "process exited non-0: $LASTEXITCODE"
+}
+Write-Output "purged Mercurial repo"
+'''
+
+HG_UPDATE_CLEAN = r'''
+$Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
+Set-Location C:\hgdev\src
+hg.exe --config extensions.purge= purge --all
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+hg.exe update -C {revision}
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+hg.exe log -r .
+Write-Output "updated Mercurial working directory to {revision}"
+'''.lstrip()
+
+BUILD_INNO = r'''
+Set-Location C:\hgdev\src
+$python = "C:\hgdev\python27-{arch}\python.exe"
+C:\hgdev\python37-x64\python.exe contrib\packaging\inno\build.py --python $python
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+'''.lstrip()
+
+BUILD_WHEEL = r'''
+Set-Location C:\hgdev\src
+C:\hgdev\python27-{arch}\Scripts\pip.exe wheel --wheel-dir dist .
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+'''
+
+BUILD_WIX = r'''
+Set-Location C:\hgdev\src
+$python = "C:\hgdev\python27-{arch}\python.exe"
+C:\hgdev\python37-x64\python.exe contrib\packaging\wix\build.py --python $python {extra_args}
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+'''
+
+RUN_TESTS = r'''
+C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
+if ($LASTEXITCODE -ne 0) {{
+    throw "process exited non-0: $LASTEXITCODE"
+}}
+'''
+
+
+def get_vc_prefix(arch):
+    if arch == 'x86':
+        return ACTIVATE_VC9_X86
+    elif arch == 'x64':
+        return ACTIVATE_VC9_AMD64
+    else:
+        raise ValueError('illegal arch: %s; must be x86 or x64' % arch)
+
+
+def fix_authorized_keys_permissions(winrm_client, path):
+    commands = [
+        '$ErrorActionPreference = "Stop"',
+        'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
+        r'icacls %s /remove:g "NT Service\sshd"' % path,
+    ]
+
+    run_powershell(winrm_client, '\n'.join(commands))
+
+
+def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
+    """Synchronize local Mercurial repo to remote EC2 instance."""
+
+    winrm_client = ec2_instance.winrm_client
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_dir = pathlib.Path(temp_dir)
+
+        ssh_dir = temp_dir / '.ssh'
+        ssh_dir.mkdir()
+        ssh_dir.chmod(0o0700)
+
+        # Generate SSH key to use for communication.
+        subprocess.run([
+            'ssh-keygen', '-t', 'rsa', '-b', '4096', '-N', '',
+            '-f', str(ssh_dir / 'id_rsa')],
+            check=True, capture_output=True)
+
+        # Add it to ~/.ssh/authorized_keys on remote.
+        # This assumes the file doesn't already exist.
+        authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
+        winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
+        winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
+        fix_authorized_keys_permissions(winrm_client, authorized_keys)
+
+        public_ip = ec2_instance.public_ip_address
+
+        ssh_config = temp_dir / '.ssh' / 'config'
+
+        with open(ssh_config, 'w', encoding='utf-8') as fh:
+            fh.write('Host %s\n' % public_ip)
+            fh.write('  User Administrator\n')
+            fh.write('  StrictHostKeyChecking no\n')
+            fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
+            fh.write('  IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
+
+        env = dict(os.environ)
+        env['HGPLAIN'] = '1'
+        env['HGENCODING'] = 'utf-8'
+
+        hg_bin = hg_repo / 'hg'
+
+        res = subprocess.run(
+            ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
+            cwd=str(hg_repo), env=env, check=True, capture_output=True)
+
+        full_revision = res.stdout.decode('ascii')
+
+        args = [
+            'python2.7', hg_bin,
+            '--config', 'ui.ssh=ssh -F %s' % ssh_config,
+            '--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
+            'push', '-r', full_revision, 'ssh://%s/c:/hgdev/src' % public_ip,
+        ]
+
+        subprocess.run(args, cwd=str(hg_repo), env=env, check=True)
+
+        run_powershell(winrm_client,
+                       HG_UPDATE_CLEAN.format(revision=full_revision))
+
+        # TODO detect dirty local working directory and synchronize accordingly.
+
+
+def purge_hg(winrm_client):
+    """Purge the Mercurial source repository on an EC2 instance."""
+    run_powershell(winrm_client, HG_PURGE)
+
+
+def find_latest_dist(winrm_client, pattern):
+    """Find path to newest file in dist/ directory matching a pattern."""
+
+    res = winrm_client.execute_ps(
+        r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
+        '| Sort-Object LastWriteTime -Descending '
+        '| Select-Object -First 1\n'
+        '$v.name' % pattern
+    )
+    return res[0]
+
+
+def copy_latest_dist(winrm_client, pattern, dest_path):
+    """Copy latest file matching pattern in dist/ directory.
+
+    Given a WinRM client and a file pattern, find the latest file on the remote
+    matching that pattern and copy it to the ``dest_path`` directory on the
+    local machine.
+    """
+    latest = find_latest_dist(winrm_client, pattern)
+    source = r'C:\hgdev\src\dist\%s' % latest
+    dest = dest_path / latest
+    print('copying %s to %s' % (source, dest))
+    winrm_client.fetch(source, str(dest))
+
+
+def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path,
+                         version=None):
+    """Build the Inno Setup installer on a remote machine.
+
+    Using a WinRM client, remote commands are executed to build
+    a Mercurial Inno Setup installer.
+    """
+    print('building Inno Setup installer for %s' % arch)
+
+    extra_args = []
+    if version:
+        extra_args.extend(['--version', version])
+
+    ps = get_vc_prefix(arch) + BUILD_INNO.format(arch=arch,
+                                                 extra_args=' '.join(extra_args))
+    run_powershell(winrm_client, ps)
+    copy_latest_dist(winrm_client, '*.exe', dest_path)
+
+
+def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path):
+    """Build Python wheels on a remote machine.
+
+    Using a WinRM client, remote commands are executed to build a Python wheel
+    for Mercurial.
+    """
+    print('Building Windows wheel for %s' % arch)
+    ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch)
+    run_powershell(winrm_client, ps)
+    copy_latest_dist(winrm_client, '*.whl', dest_path)
+
+
+def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path,
+                        version=None):
+    """Build the WiX installer on a remote machine.
+
+    Using a WinRM client, remote commands are executed to build a WiX installer.
+    """
+    print('Building WiX installer for %s' % arch)
+    extra_args = []
+    if version:
+        extra_args.extend(['--version', version])
+
+    ps = get_vc_prefix(arch) + BUILD_WIX.format(arch=arch,
+                                                extra_args=' '.join(extra_args))
+    run_powershell(winrm_client, ps)
+    copy_latest_dist(winrm_client, '*.msi', dest_path)
+
+
+def run_tests(winrm_client, python_version, arch, test_flags=''):
+    """Run tests on a remote Windows machine.
+
+    ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
+    ``arch`` is ``x86`` or ``x64``.
+    ``test_flags`` is a str representing extra arguments to pass to
+    ``run-tests.py``.
+    """
+    if not re.match(r'\d\.\d', python_version):
+        raise ValueError(r'python_version must be \d.\d; got %s' %
+                         python_version)
+
+    if arch not in ('x86', 'x64'):
+        raise ValueError('arch must be x86 or x64; got %s' % arch)
+
+    python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
+
+    ps = RUN_TESTS.format(
+        python_path=python_path,
+        test_flags=test_flags or '',
+    )
+
+    run_powershell(winrm_client, ps)

contrib/automation/hgautomation/winrm.py

@@ -0,0 +1,82 b''
+# winrm.py - Interact with Windows Remote Management (WinRM)
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import logging
+import pprint
+import time
+
+from pypsrp.client import (
+    Client,
+)
+from pypsrp.powershell import (
+    PowerShell,
+    PSInvocationState,
+    RunspacePool,
+)
+import requests.exceptions
+
+
+logger = logging.getLogger(__name__)
+
+
+def wait_for_winrm(host, username, password, timeout=120, ssl=False):
+    """Wait for the Windows Remoting (WinRM) service to become available.
+
+    Returns a ``psrpclient.Client`` instance.
+    """
+
+    end_time = time.time() + timeout
+
+    while True:
+        try:
+            client = Client(host, username=username, password=password,
+                            ssl=ssl, connection_timeout=5)
+            client.execute_cmd('echo "hello world"')
+            return client
+        except requests.exceptions.ConnectionError:
+            if time.time() >= end_time:
+                raise
+
+            time.sleep(1)
+
+
+def format_object(o):
+    if isinstance(o, str):
+        return o
+
+    try:
+        o = str(o)
+    except TypeError:
+        o = pprint.pformat(o.extended_properties)
+
+    return o
+
+
+def run_powershell(client, script):
+    with RunspacePool(client.wsman) as pool:
+        ps = PowerShell(pool)
+        ps.add_script(script)
+
+        ps.begin_invoke()
+
+        while ps.state == PSInvocationState.RUNNING:
+            ps.poll_invoke()
+            for o in ps.output:
+                print(format_object(o))
+
+            ps.output[:] = []
+
+        ps.end_invoke()
+
+        for o in ps.output:
+            print(format_object(o))
+
+        if ps.state == PSInvocationState.FAILED:
+            raise Exception('PowerShell execution failed: %s' %
+                            ' '.join(map(format_object, ps.streams.error)))

contrib/automation/requirements.txt

@@ -0,0 +1,119 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile -U --generate-hashes --output-file contrib/automation/requirements.txt contrib/automation/requirements.txt.in
+#
+asn1crypto==0.24.0 \
+    --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
+    --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \
+    # via cryptography
+boto3==1.9.111 \
+    --hash=sha256:06414c75d1f62af7d04fd652b38d1e4fd3cfd6b35bad978466af88e2aaecd00d \
+    --hash=sha256:f3b77dff382374773d02411fa47ee408f4f503aeebd837fd9dc9ed8635bc5e8e
+botocore==1.12.111 \
+    --hash=sha256:6af473c52d5e3e7ff82de5334e9fee96b2d5ec2df5d78bc00cd9937e2573a7a8 \
+    --hash=sha256:9f5123c7be704b17aeacae99b5842ab17bda1f799dd29134de8c70e0a50a45d7 \
+    # via boto3, s3transfer
+certifi==2019.3.9 \
+    --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \
+    --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \
+    # via requests
+cffi==1.12.2 \
+    --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
+    --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
+    --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \
+    --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \
+    --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \
+    --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \
+    --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \
+    --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \
+    --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \
+    --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \
+    --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \
+    --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \
+    --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \
+    --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \
+    --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \
+    --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \
+    --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \
+    --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \
+    --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \
+    --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \
+    --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \
+    --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \
+    --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \
+    --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \
+    --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \
+    --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \
+    --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \
+    --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 \
+    # via cryptography
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+cryptography==2.6.1 \
+    --hash=sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1 \
+    --hash=sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705 \
+    --hash=sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6 \
+    --hash=sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1 \
+    --hash=sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8 \
+    --hash=sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151 \
+    --hash=sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d \
+    --hash=sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659 \
+    --hash=sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537 \
+    --hash=sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e \
+    --hash=sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb \
+    --hash=sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c \
+    --hash=sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9 \
+    --hash=sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5 \
+    --hash=sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad \
+    --hash=sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a \
+    --hash=sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460 \
+    --hash=sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd \
+    --hash=sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6 \
+    # via pypsrp
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6 \
+    # via botocore
+idna==2.8 \
+    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
+    # via requests
+jmespath==0.9.4 \
+    --hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \
+    --hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c \
+    # via boto3, botocore
+ntlm-auth==1.2.0 \
+    --hash=sha256:7bc02a3fbdfee7275d3dc20fce8028ed8eb6d32364637f28be9e9ae9160c6d5c \
+    --hash=sha256:9b13eaf88f16a831637d75236a93d60c0049536715aafbf8190ba58a590b023e \
+    # via pypsrp
+pycparser==2.19 \
+    --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
+    # via cffi
+pypsrp==0.3.1 \
+    --hash=sha256:309853380fe086090a03cc6662a778ee69b1cae355ae4a932859034fd76e9d0b \
+    --hash=sha256:90f946254f547dc3493cea8493c819ab87e152a755797c93aa2668678ba8ae85
+python-dateutil==2.8.0 \
+    --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
+    --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
+    # via botocore
+requests==2.21.0 \
+    --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
+    --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \
+    # via pypsrp
+s3transfer==0.2.0 \
+    --hash=sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e \
+    --hash=sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021 \
+    # via boto3
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via cryptography, pypsrp, python-dateutil
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via botocore, requests

contrib/automation/requirements.txt.in

@@ -0,0 +1,2 b''
+boto3
+pypsrp

contrib/install-windows-dependencies.ps1

@@ -0,0 +1,200 b''
+# install-dependencies.ps1 - Install Windows dependencies for building Mercurial
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# This script can be used to bootstrap a Mercurial build environment on
+# Windows.
+#
+# The script makes a lot of assumptions about how things should work.
+# For example, the install location of Python is hardcoded to c:\hgdev\*.
+#
+# The script should be executed from a PowerShell with elevated privileges
+# if you don't want to see a UAC prompt for various installers.
+#
+# The script is tested on Windows 10 and Windows Server 2019 (in EC2).
+
+$VS_BUILD_TOOLS_URL = "https://download.visualstudio.microsoft.com/download/pr/a1603c02-8a66-4b83-b821-811e3610a7c4/aa2db8bb39e0cbd23e9940d8951e0bc3/vs_buildtools.exe"
+$VS_BUILD_TOOLS_SHA256 = "911E292B8E6E5F46CBC17003BDCD2D27A70E616E8D5E6E69D5D489A605CAA139"
+
+$VC9_PYTHON_URL = "https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
+$VC9_PYTHON_SHA256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf"
+
+$PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.16/python-2.7.16.amd64.msi"
+$PYTHON27_x64_SHA256 = "7c0f45993019152d46041a7db4b947b919558fdb7a8f67bcd0535bc98d42b603"
+$PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.16/python-2.7.16.msi"
+$PYTHON27_X86_SHA256 = "d57dc3e1ba490aee856c28b4915d09e3f49442461e46e481bc6b2d18207831d7"
+
+$PYTHON35_x86_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4.exe"
+$PYTHON35_x86_SHA256 = "F27C2D67FD9688E4970F3BFF799BB9D722A0D6C2C13B04848E1F7D620B524B0E"
+$PYTHON35_x64_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4-amd64.exe"
+$PYTHON35_x64_SHA256 = "9B7741CC32357573A77D2EE64987717E527628C38FD7EAF3E2AACA853D45A1EE"
+
+$PYTHON36_x86_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8.exe"
+$PYTHON36_x86_SHA256 = "89871D432BC06E4630D7B64CB1A8451E53C80E68DE29029976B12AAD7DBFA5A0"
+$PYTHON36_x64_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
+$PYTHON36_x64_SHA256 = "96088A58B7C43BC83B84E6B67F15E8706C614023DD64F9A5A14E81FF824ADADC"
+
+$PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.2/python-3.7.2.exe"
+$PYTHON37_x86_SHA256 = "8BACE330FB409E428B04EEEE083DD9CA7F6C754366D07E23B3853891D8F8C3D0"
+$PYTHON37_x64_URL = "https://www.python.org/ftp/python/3.7.2/python-3.7.2-amd64.exe"
+$PYTHON37_x64_SHA256 = "0FE2A696F5A3E481FED795EF6896ED99157BCEF273EF3C4A96F2905CBDB3AA13"
+
+$PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.0/python-3.8.0a2.exe"
+$PYTHON38_x86_SHA256 = "013A7DDD317679FE51223DE627688CFCB2F0F1128FD25A987F846AEB476D3FEF"
+$PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.0/python-3.8.0a2-amd64.exe"
+$PYTHON38_X64_SHA256 = "560BC6D1A76BCD6D544AC650709F3892956890753CDCF9CE67E3D7302D76FB41"
+
+# PIP 19.0.3.
+$PIP_URL = "https://github.com/pypa/get-pip/raw/fee32c376da1ff6496a798986d7939cd51e1644f/get-pip.py"
+$PIP_SHA256 = "efe99298f3fbb1f56201ce6b81d2658067d2f7d7dfc2d412e0d3cacc9a397c61"
+
+$VIRTUALENV_URL = "https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/virtualenv-16.4.3.tar.gz"
+$VIRTUALENV_SHA256 = "984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39"
+
+$INNO_SETUP_URL = "http://files.jrsoftware.org/is/5/innosetup-5.6.1-unicode.exe"
+$INNO_SETUP_SHA256 = "27D49E9BC769E9D1B214C153011978DB90DC01C2ACD1DDCD9ED7B3FE3B96B538"
+
+$MINGW_BIN_URL = "https://osdn.net/frs/redir.php?m=constant&f=mingw%2F68260%2Fmingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip"
+$MINGW_BIN_SHA256 = "2AB8EFD7C7D1FC8EAF8B2FA4DA4EEF8F3E47768284C021599BC7435839A046DF"
+
+$MERCURIAL_WHEEL_FILENAME = "mercurial-4.9-cp27-cp27m-win_amd64.whl"
+$MERCURIAL_WHEEL_URL = "https://files.pythonhosted.org/packages/fe/e8/b872d53dfbbf986bdc46af0b30f580b227fb59bddd2587152a55e205b0cc/$MERCURIAL_WHEEL_FILENAME"
+$MERCURIAL_WHEEL_SHA256 = "218cc2e7c3f1d535007febbb03351663897edf27df0e57d6842e3b686492b429"
+
+# Writing progress slows down downloads substantially. So disable it.
+$progressPreference = 'silentlyContinue'
+
+function Secure-Download($url, $path, $sha256) {
+    if (Test-Path -Path $path) {
+        Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
+
+        if ($hash.Hash -eq $sha256) {
+            Write-Output "SHA256 of $path verified as $sha256"
+            return
+        }
+
+        Write-Output "hash mismatch on $path; downloading again"
+    }
+
+    Write-Output "downloading $url to $path"
+    Invoke-WebRequest -Uri $url -OutFile $path
+    Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
+
+    if ($hash.Hash -ne $sha256) {
+        Remove-Item -Path $path
+        throw "hash mismatch when downloading $url; got $($hash.Hash), expected $sha256"
+    }
+}
+
+function Invoke-Process($path, $arguments) {
+    $p = Start-Process -FilePath $path -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
+
+    if ($p.ExitCode -ne 0) {
+        throw "process exited non-0: $($p.ExitCode)"
+    }
+}
+
+function Install-Python3($name, $installer, $dest, $pip) {
+    Write-Output "installing $name"
+
+    # We hit this when running the script as part of Simple Systems Manager in
+    # EC2. The Python 3 installer doesn't seem to like per-user installs
+    # when running as the SYSTEM user. So enable global installs if executed in
+    # this mode.
+    if ($env:USERPROFILE -eq "C:\Windows\system32\config\systemprofile") {
+        Write-Output "running with SYSTEM account; installing for all users"
+        $allusers = "1"
+    }
+    else {
+        $allusers = "0"
+    }
+
+    Invoke-Process $installer "/quiet TargetDir=${dest} InstallAllUsers=${allusers} AssociateFiles=0 CompileAll=0 PrependPath=0 Include_doc=0 Include_launcher=0 InstallLauncherAllUsers=0 Include_pip=0 Include_test=0"
+    Invoke-Process ${dest}\python.exe $pip
+}
+
+function Install-Dependencies($prefix) {
+    if (!(Test-Path -Path $prefix\assets)) {
+        New-Item -Path $prefix\assets -ItemType Directory
+    }
+
+    $pip = "${prefix}\assets\get-pip.py"
+
+    Secure-Download $VC9_PYTHON_URL ${prefix}\assets\VCForPython27.msi $VC9_PYTHON_SHA256
+    Secure-Download $PYTHON27_x86_URL ${prefix}\assets\python27-x86.msi $PYTHON27_x86_SHA256
+    Secure-Download $PYTHON27_x64_URL ${prefix}\assets\python27-x64.msi $PYTHON27_x64_SHA256
+    Secure-Download $PYTHON35_x86_URL ${prefix}\assets\python35-x86.exe $PYTHON35_x86_SHA256
+    Secure-Download $PYTHON35_x64_URL ${prefix}\assets\python35-x64.exe $PYTHON35_x64_SHA256
+    Secure-Download $PYTHON36_x86_URL ${prefix}\assets\python36-x86.exe $PYTHON36_x86_SHA256
+    Secure-Download $PYTHON36_x64_URL ${prefix}\assets\python36-x64.exe $PYTHON36_x64_SHA256
+    Secure-Download $PYTHON37_x86_URL ${prefix}\assets\python37-x86.exe $PYTHON37_x86_SHA256
+    Secure-Download $PYTHON37_x64_URL ${prefix}\assets\python37-x64.exe $PYTHON37_x64_SHA256
+    Secure-Download $PYTHON38_x86_URL ${prefix}\assets\python38-x86.exe $PYTHON38_x86_SHA256
+    Secure-Download $PYTHON38_x64_URL ${prefix}\assets\python38-x64.exe $PYTHON38_x64_SHA256
+    Secure-Download $PIP_URL ${pip} $PIP_SHA256
+    Secure-Download $VIRTUALENV_URL ${prefix}\assets\virtualenv.tar.gz $VIRTUALENV_SHA256
+    Secure-Download $VS_BUILD_TOOLS_URL ${prefix}\assets\vs_buildtools.exe $VS_BUILD_TOOLS_SHA256
+    Secure-Download $INNO_SETUP_URL ${prefix}\assets\InnoSetup.exe $INNO_SETUP_SHA256
+    Secure-Download $MINGW_BIN_URL ${prefix}\assets\mingw-get-bin.zip $MINGW_BIN_SHA256
+    Secure-Download $MERCURIAL_WHEEL_URL ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME} $MERCURIAL_WHEEL_SHA256
+
+    Write-Output "installing Python 2.7 32-bit"
+    Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x86.msi /l* ${prefix}\assets\python27-x86.log /q TARGETDIR=${prefix}\python27-x86 ALLUSERS="
+    Invoke-Process ${prefix}\python27-x86\python.exe ${prefix}\assets\get-pip.py
+    Invoke-Process ${prefix}\python27-x86\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
+
+    Write-Output "installing Python 2.7 64-bit"
+    Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x64.msi /l* ${prefix}\assets\python27-x64.log /q TARGETDIR=${prefix}\python27-x64 ALLUSERS="
+    Invoke-Process ${prefix}\python27-x64\python.exe ${prefix}\assets\get-pip.py
+    Invoke-Process ${prefix}\python27-x64\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
+
+    Install-Python3 "Python 3.5 32-bit" ${prefix}\assets\python35-x86.exe ${prefix}\python35-x86 ${pip}
+    Install-Python3 "Python 3.5 64-bit" ${prefix}\assets\python35-x64.exe ${prefix}\python35-x64 ${pip}
+    Install-Python3 "Python 3.6 32-bit" ${prefix}\assets\python36-x86.exe ${prefix}\python36-x86 ${pip}
+    Install-Python3 "Python 3.6 64-bit" ${prefix}\assets\python36-x64.exe ${prefix}\python36-x64 ${pip}
+    Install-Python3 "Python 3.7 32-bit" ${prefix}\assets\python37-x86.exe ${prefix}\python37-x86 ${pip}
+    Install-Python3 "Python 3.7 64-bit" ${prefix}\assets\python37-x64.exe ${prefix}\python37-x64 ${pip}
+    Install-Python3 "Python 3.8 32-bit" ${prefix}\assets\python38-x86.exe ${prefix}\python38-x86 ${pip}
+    Install-Python3 "Python 3.8 64-bit" ${prefix}\assets\python38-x64.exe ${prefix}\python38-x64 ${pip}
+
+    Write-Output "installing Visual Studio 2017 Build Tools and SDKs"
+    Invoke-Process ${prefix}\assets\vs_buildtools.exe "--quiet --wait --norestart --nocache --channelUri https://aka.ms/vs/15/release/channel --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.VC.140"
+
+    Write-Output "installing Visual C++ 9.0 for Python 2.7"
+    Invoke-Process msiexec.exe "/i ${prefix}\assets\VCForPython27.msi /l* ${prefix}\assets\VCForPython27.log /q"
+
+    Write-Output "installing Inno Setup"
+    Invoke-Process ${prefix}\assets\InnoSetup.exe "/SP- /VERYSILENT /SUPPRESSMSGBOXES"
+
+    Write-Output "extracting MinGW base archive"
+    Expand-Archive -Path ${prefix}\assets\mingw-get-bin.zip -DestinationPath "${prefix}\MinGW" -Force
+
+    Write-Output "updating MinGW package catalogs"
+    Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "update"
+
+    Write-Output "installing MinGW packages"
+    Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "install msys-base msys-coreutils msys-diffutils msys-unzip"
+
+    # Construct a virtualenv useful for bootstrapping. It conveniently contains a
+    # Mercurial install.
+    Write-Output "creating bootstrap virtualenv with Mercurial"
+    Invoke-Process "$prefix\python27-x64\Scripts\virtualenv.exe" "${prefix}\venv-bootstrap"
+    Invoke-Process "${prefix}\venv-bootstrap\Scripts\pip.exe" "install ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME}"
+}
+
+function Clone-Mercurial-Repo($prefix, $repo_url, $dest) {
+    Write-Output "cloning $repo_url to $dest"
+    # TODO Figure out why CA verification isn't working in EC2 and remove
+    # --insecure.
+    Invoke-Process "${prefix}\venv-bootstrap\Scripts\hg.exe" "clone --insecure $repo_url $dest"
+
+    # Mark repo as non-publishing by default for convenience.
+    Add-Content -Path "$dest\.hg\hgrc" -Value "`n[phases]`npublish = false"
+}
+
+$prefix = "c:\hgdev"
+Install-Dependencies $prefix
+Clone-Mercurial-Repo $prefix "https://www.mercurial-scm.org/repo/hg" $prefix\src

contrib/packaging/hgpackaging/downloads.py

@@ -0,0 +1,175 b''
+# downloads.py - Code for downloading dependencies.
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import gzip
+import hashlib
+import pathlib
+import urllib.request
+
+
+DOWNLOADS = {
+    'gettext': {
+        'url': 'https://versaweb.dl.sourceforge.net/project/gnuwin32/gettext/0.14.4/gettext-0.14.4-bin.zip',
+        'size': 1606131,
+        'sha256': '60b9ef26bc5cceef036f0424e542106cf158352b2677f43a01affd6d82a1d641',
+        'version': '0.14.4',
+    },
+    'gettext-dep': {
+        'url': 'https://versaweb.dl.sourceforge.net/project/gnuwin32/gettext/0.14.4/gettext-0.14.4-dep.zip',
+        'size': 715086,
+        'sha256': '411f94974492fd2ecf52590cb05b1023530aec67e64154a88b1e4ebcd9c28588',
+    },
+    'py2exe': {
+        'url': 'https://versaweb.dl.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.zip',
+        'size': 149687,
+        'sha256': '6bd383312e7d33eef2e43a5f236f9445e4f3e0f6b16333c6f183ed445c44ddbd',
+        'version': '0.6.9',
+    },
+    # The VC9 CRT merge modules aren't readily available on most systems because
+    # they are only installed as part of a full Visual Studio 2008 install.
+    # While we could potentially extract them from a Visual Studio 2008
+    # installer, it is easier to just fetch them from a known URL.
+    'vc9-crt-x86-msm': {
+        'url': 'https://github.com/indygreg/vc90-merge-modules/raw/9232f8f0b2135df619bf7946eaa176b4ac35ccff/Microsoft_VC90_CRT_x86.msm',
+        'size': 615424,
+        'sha256': '837e887ef31b332feb58156f429389de345cb94504228bb9a523c25a9dd3d75e',
+    },
+    'vc9-crt-x86-msm-policy': {
+        'url': 'https://github.com/indygreg/vc90-merge-modules/raw/9232f8f0b2135df619bf7946eaa176b4ac35ccff/policy_9_0_Microsoft_VC90_CRT_x86.msm',
+        'size': 71168,
+        'sha256': '3fbcf92e3801a0757f36c5e8d304e134a68d5cafd197a6df7734ae3e8825c940',
+    },
+    'vc9-crt-x64-msm': {
+        'url': 'https://github.com/indygreg/vc90-merge-modules/raw/9232f8f0b2135df619bf7946eaa176b4ac35ccff/Microsoft_VC90_CRT_x86_x64.msm',
+        'size': 662528,
+        'sha256': '50d9639b5ad4844a2285269c7551bf5157ec636e32396ddcc6f7ec5bce487a7c',
+    },
+    'vc9-crt-x64-msm-policy': {
+        'url': 'https://github.com/indygreg/vc90-merge-modules/raw/9232f8f0b2135df619bf7946eaa176b4ac35ccff/policy_9_0_Microsoft_VC90_CRT_x86_x64.msm',
+        'size': 71168,
+        'sha256': '0550ea1929b21239134ad3a678c944ba0f05f11087117b6cf0833e7110686486',
+    },
+    'virtualenv': {
+        'url': 'https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/virtualenv-16.4.3.tar.gz',
+        'size': 3713208,
+        'sha256': '984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39',
+        'version': '16.4.3',
+    },
+    'wix': {
+        'url': 'https://github.com/wixtoolset/wix3/releases/download/wix3111rtm/wix311-binaries.zip',
+        'size': 34358269,
+        'sha256': '37f0a533b0978a454efb5dc3bd3598becf9660aaf4287e55bf68ca6b527d051d',
+        'version': '3.11.1',
+    },
+}
+
+
+def hash_path(p: pathlib.Path):
+    h = hashlib.sha256()
+
+    with p.open('rb') as fh:
+        while True:
+            chunk = fh.read(65536)
+            if not chunk:
+                break
+
+            h.update(chunk)
+
+    return h.hexdigest()
+
+
+class IntegrityError(Exception):
+    """Represents an integrity error when downloading a URL."""
+
+
+def secure_download_stream(url, size, sha256):
+    """Securely download a URL to a stream of chunks.
+
+    If the integrity of the download fails, an IntegrityError is
+    raised.
+    """
+    h = hashlib.sha256()
+    length = 0
+
+    with urllib.request.urlopen(url) as fh:
+        if not url.endswith('.gz') and fh.info().get('Content-Encoding') == 'gzip':
+            fh = gzip.GzipFile(fileobj=fh)
+
+        while True:
+            chunk = fh.read(65536)
+            if not chunk:
+                break
+
+            h.update(chunk)
+            length += len(chunk)
+
+            yield chunk
+
+    digest = h.hexdigest()
+
+    if length != size:
+        raise IntegrityError('size mismatch on %s: wanted %d; got %d' % (
+            url, size, length))
+
+    if digest != sha256:
+        raise IntegrityError('sha256 mismatch on %s: wanted %s; got %s' % (
+            url, sha256, digest))
+
+
+def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):
+    """Download a URL to a filesystem path, possibly with verification."""
+
+    # We download to a temporary file and rename at the end so there's
+    # no chance of the final file being partially written or containing
+    # bad data.
+    print('downloading %s to %s' % (url, path))
+
+    if path.exists():
+        good = True
+
+        if path.stat().st_size != size:
+            print('existing file size is wrong; removing')
+            good = False
+
+        if good:
+            if hash_path(path) != sha256:
+                print('existing file hash is wrong; removing')
+                good = False
+
+        if good:
+            print('%s exists and passes integrity checks' % path)
+            return
+
+        path.unlink()
+
+    tmp = path.with_name('%s.tmp' % path.name)
+
+    try:
+        with tmp.open('wb') as fh:
+            for chunk in secure_download_stream(url, size, sha256):
+                fh.write(chunk)
+    except IntegrityError:
+        tmp.unlink()
+        raise
+
+    tmp.rename(path)
+    print('successfully downloaded %s' % url)
+
+
+def download_entry(name: dict, dest_path: pathlib.Path, local_name=None) -> pathlib.Path:
+    entry = DOWNLOADS[name]
+
+    url = entry['url']
+
+    local_name = local_name or url[url.rindex('/') + 1:]
+
+    local_path = dest_path / local_name
+    download_to_path(url, local_path, entry['size'], entry['sha256'])
+
+    return local_path, entry

contrib/packaging/hgpackaging/inno.py

@@ -0,0 +1,78 b''
+# inno.py - Inno Setup functionality.
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import os
+import pathlib
+import shutil
+import subprocess
+
+from .py2exe import (
+    build_py2exe,
+)
+from .util import (
+    find_vc_runtime_files,
+)
+
+
+EXTRA_PACKAGES = {
+    'dulwich',
+    'keyring',
+    'pygments',
+    'win32ctypes',
+}
+
+
+def build(source_dir: pathlib.Path, build_dir: pathlib.Path,
+          python_exe: pathlib.Path, iscc_exe: pathlib.Path,
+          version=None):
+    """Build the Inno installer.
+
+    Build files will be placed in ``build_dir``.
+
+    py2exe's setup.py doesn't use setuptools. It doesn't have modern logic
+    for finding the Python 2.7 toolchain. So, we require the environment
+    to already be configured with an active toolchain.
+    """
+    if not iscc_exe.exists():
+        raise Exception('%s does not exist' % iscc_exe)
+
+    vc_x64 = r'\x64' in os.environ.get('LIB', '')
+
+    requirements_txt = (source_dir / 'contrib' / 'packaging' /
+                        'inno' / 'requirements.txt')
+
+    build_py2exe(source_dir, build_dir, python_exe, 'inno',
+                 requirements_txt, extra_packages=EXTRA_PACKAGES)
+
+    # hg.exe depends on VC9 runtime DLLs. Copy those into place.
+    for f in find_vc_runtime_files(vc_x64):
+        if f.name.endswith('.manifest'):
+            basename = 'Microsoft.VC90.CRT.manifest'
+        else:
+            basename = f.name
+
+        dest_path = source_dir / 'dist' / basename
+
+        print('copying %s to %s' % (f, dest_path))
+        shutil.copyfile(f, dest_path)
+
+    print('creating installer')
+
+    args = [str(iscc_exe)]
+
+    if vc_x64:
+        args.append('/dARCH=x64')
+
+    if version:
+        args.append('/dVERSION=%s' % version)
+
+    args.append('/Odist')
+    args.append('contrib/packaging/inno/mercurial.iss')
+
+    subprocess.run(args, cwd=str(source_dir), check=True)

contrib/packaging/hgpackaging/py2exe.py

@@ -0,0 +1,150 b''
+# py2exe.py - Functionality for performing py2exe builds.
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import os
+import pathlib
+import subprocess
+
+from .downloads import (
+    download_entry,
+)
+from .util import (
+    extract_tar_to_directory,
+    extract_zip_to_directory,
+    python_exe_info,
+)
+
+
+def build_py2exe(source_dir: pathlib.Path, build_dir: pathlib.Path,
+                 python_exe: pathlib.Path, build_name: str,
+                 venv_requirements_txt: pathlib.Path,
+                 extra_packages=None, extra_excludes=None,
+                 extra_dll_excludes=None,
+                 extra_packages_script=None):
+    """Build Mercurial with py2exe.
+
+    Build files will be placed in ``build_dir``.
+
+    py2exe's setup.py doesn't use setuptools. It doesn't have modern logic
+    for finding the Python 2.7 toolchain. So, we require the environment
+    to already be configured with an active toolchain.
+    """
+    if 'VCINSTALLDIR' not in os.environ:
+        raise Exception('not running from a Visual C++ build environment; '
+                        'execute the "Visual C++ <version> Command Prompt" '
+                        'application shortcut or a vcsvarsall.bat file')
+
+    # Identity x86/x64 and validate the environment matches the Python
+    # architecture.
+    vc_x64 = r'\x64' in os.environ['LIB']
+
+    py_info = python_exe_info(python_exe)
+
+    if vc_x64:
+        if py_info['arch'] != '64bit':
+            raise Exception('architecture mismatch: Visual C++ environment '
+                            'is configured for 64-bit but Python is 32-bit')
+    else:
+        if py_info['arch'] != '32bit':
+            raise Exception('architecture mismatch: Visual C++ environment '
+                            'is configured for 32-bit but Python is 64-bit')
+
+    if py_info['py3']:
+        raise Exception('Only Python 2 is currently supported')
+
+    build_dir.mkdir(exist_ok=True)
+
+    gettext_pkg, gettext_entry = download_entry('gettext', build_dir)
+    gettext_dep_pkg = download_entry('gettext-dep', build_dir)[0]
+    virtualenv_pkg, virtualenv_entry = download_entry('virtualenv', build_dir)
+    py2exe_pkg, py2exe_entry = download_entry('py2exe', build_dir)
+
+    venv_path = build_dir / ('venv-%s-%s' % (build_name,
+                                             'x64' if vc_x64 else 'x86'))
+
+    gettext_root = build_dir / (
+        'gettext-win-%s' % gettext_entry['version'])
+
+    if not gettext_root.exists():
+        extract_zip_to_directory(gettext_pkg, gettext_root)
+        extract_zip_to_directory(gettext_dep_pkg, gettext_root)
+
+    # This assumes Python 2. We don't need virtualenv on Python 3.
+    virtualenv_src_path = build_dir / (
+        'virtualenv-%s' % virtualenv_entry['version'])
+    virtualenv_py = virtualenv_src_path / 'virtualenv.py'
+
+    if not virtualenv_src_path.exists():
+        extract_tar_to_directory(virtualenv_pkg, build_dir)
+
+    py2exe_source_path = build_dir / ('py2exe-%s' % py2exe_entry['version'])
+
+    if not py2exe_source_path.exists():
+        extract_zip_to_directory(py2exe_pkg, build_dir)
+
+    if not venv_path.exists():
+        print('creating virtualenv with dependencies')
+        subprocess.run(
+            [str(python_exe), str(virtualenv_py), str(venv_path)],
+            check=True)
+
+    venv_python = venv_path / 'Scripts' / 'python.exe'
+    venv_pip = venv_path / 'Scripts' / 'pip.exe'
+
+    subprocess.run([str(venv_pip), 'install', '-r', str(venv_requirements_txt)],
+                   check=True)
+
+    # Force distutils to use VC++ settings from environment, which was
+    # validated above.
+    env = dict(os.environ)
+    env['DISTUTILS_USE_SDK'] = '1'
+    env['MSSdk'] = '1'
+
+    if extra_packages_script:
+        more_packages = set(subprocess.check_output(
+            extra_packages_script,
+            cwd=build_dir).split(b'\0')[-1].strip().decode('utf-8').splitlines())
+        if more_packages:
+            if not extra_packages:
+                extra_packages = more_packages
+            else:
+                extra_packages |= more_packages
+
+    if extra_packages:
+        env['HG_PY2EXE_EXTRA_PACKAGES'] = ' '.join(sorted(extra_packages))
+        hgext3rd_extras = sorted(
+            e for e in extra_packages if e.startswith('hgext3rd.'))
+        if hgext3rd_extras:
+            env['HG_PY2EXE_EXTRA_INSTALL_PACKAGES'] = ' '.join(hgext3rd_extras)
+    if extra_excludes:
+        env['HG_PY2EXE_EXTRA_EXCLUDES'] = ' '.join(sorted(extra_excludes))
+    if extra_dll_excludes:
+        env['HG_PY2EXE_EXTRA_DLL_EXCLUDES'] = ' '.join(
+            sorted(extra_dll_excludes))
+
+    py2exe_py_path = venv_path / 'Lib' / 'site-packages' / 'py2exe'
+    if not py2exe_py_path.exists():
+        print('building py2exe')
+        subprocess.run([str(venv_python), 'setup.py', 'install'],
+                       cwd=py2exe_source_path,
+                       env=env,
+                       check=True)
+
+    # Register location of msgfmt and other binaries.
+    env['PATH'] = '%s%s%s' % (
+        env['PATH'], os.pathsep, str(gettext_root / 'bin'))
+
+    print('building Mercurial')
+    subprocess.run(
+        [str(venv_python), 'setup.py',
+         'py2exe',
+         'build_doc', '--html'],
+        cwd=str(source_dir),
+        env=env,
+        check=True)

contrib/packaging/hgpackaging/util.py

@@ -0,0 +1,155 b''
+# util.py - Common packaging utility code.
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import distutils.version
+import getpass
+import os
+import pathlib
+import subprocess
+import tarfile
+import zipfile
+
+
+def extract_tar_to_directory(source: pathlib.Path, dest: pathlib.Path):
+    with tarfile.open(source, 'r') as tf:
+        tf.extractall(dest)
+
+
+def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):
+    with zipfile.ZipFile(source, 'r') as zf:
+        zf.extractall(dest)
+
+
+def find_vc_runtime_files(x64=False):
+    """Finds Visual C++ Runtime DLLs to include in distribution."""
+    winsxs = pathlib.Path(os.environ['SYSTEMROOT']) / 'WinSxS'
+
+    prefix = 'amd64' if x64 else 'x86'
+
+    candidates = sorted(p for p in os.listdir(winsxs)
+                  if p.lower().startswith('%s_microsoft.vc90.crt_' % prefix))
+
+    for p in candidates:
+        print('found candidate VC runtime: %s' % p)
+
+    # Take the newest version.
+    version = candidates[-1]
+
+    d = winsxs / version
+
+    return [
+        d / 'msvcm90.dll',
+        d / 'msvcp90.dll',
+        d / 'msvcr90.dll',
+        winsxs / 'Manifests' / ('%s.manifest' % version),
+    ]
+
+
+def windows_10_sdk_info():
+    """Resolves information about the Windows 10 SDK."""
+
+    base = pathlib.Path(os.environ['ProgramFiles(x86)']) / 'Windows Kits' / '10'
+
+    if not base.is_dir():
+        raise Exception('unable to find Windows 10 SDK at %s' % base)
+
+    # Find the latest version.
+    bin_base = base / 'bin'
+
+    versions = [v for v in os.listdir(bin_base) if v.startswith('10.')]
+    version = sorted(versions, reverse=True)[0]
+
+    bin_version = bin_base / version
+
+    return {
+        'root': base,
+        'version': version,
+        'bin_root': bin_version,
+        'bin_x86': bin_version / 'x86',
+        'bin_x64': bin_version / 'x64'
+    }
+
+
+def find_signtool():
+    """Find signtool.exe from the Windows SDK."""
+    sdk = windows_10_sdk_info()
+
+    for key in ('bin_x64', 'bin_x86'):
+        p = sdk[key] / 'signtool.exe'
+
+        if p.exists():
+            return p
+
+    raise Exception('could not find signtool.exe in Windows 10 SDK')
+
+
+def sign_with_signtool(file_path, description, subject_name=None,
+                       cert_path=None, cert_password=None,
+                       timestamp_url=None):
+    """Digitally sign a file with signtool.exe.
+
+    ``file_path`` is file to sign.
+    ``description`` is text that goes in the signature.
+
+    The signing certificate can be specified by ``cert_path`` or
+    ``subject_name``. These correspond to the ``/f`` and ``/n`` arguments
+    to signtool.exe, respectively.
+
+    The certificate password can be specified via ``cert_password``. If
+    not provided, you will be prompted for the password.
+
+    ``timestamp_url`` is the URL of a RFC 3161 timestamp server (``/tr``
+    argument to signtool.exe).
+    """
+    if cert_path and subject_name:
+        raise ValueError('cannot specify both cert_path and subject_name')
+
+    while cert_path and not cert_password:
+        cert_password = getpass.getpass('password for %s: ' % cert_path)
+
+    args = [
+        str(find_signtool()), 'sign',
+        '/v',
+        '/fd', 'sha256',
+        '/d', description,
+    ]
+
+    if cert_path:
+        args.extend(['/f', str(cert_path), '/p', cert_password])
+    elif subject_name:
+        args.extend(['/n', subject_name])
+
+    if timestamp_url:
+        args.extend(['/tr', timestamp_url, '/td', 'sha256'])
+
+    args.append(str(file_path))
+
+    print('signing %s' % file_path)
+    subprocess.run(args, check=True)
+
+
+PRINT_PYTHON_INFO = '''
+import platform; print("%s:%s" % (platform.architecture()[0], platform.python_version()))
+'''.strip()
+
+
+def python_exe_info(python_exe: pathlib.Path):
+    """Obtain information about a Python executable."""
+
+    res = subprocess.check_output([str(python_exe), '-c', PRINT_PYTHON_INFO])
+
+    arch, version = res.decode('utf-8').split(':')
+
+    version = distutils.version.LooseVersion(version)
+
+    return {
+        'arch': arch,
+        'version': version,
+        'py3': version >= distutils.version.LooseVersion('3'),
+    }

contrib/packaging/hgpackaging/wix.py

@@ -0,0 +1,327 b''
+# wix.py - WiX installer functionality
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+import os
+import pathlib
+import re
+import subprocess
+import tempfile
+import typing
+import xml.dom.minidom
+
+from .downloads import (
+    download_entry,
+)
+from .py2exe import (
+    build_py2exe,
+)
+from .util import (
+    extract_zip_to_directory,
+    sign_with_signtool,
+)
+
+
+SUPPORT_WXS = [
+    ('contrib.wxs', r'contrib'),
+    ('dist.wxs', r'dist'),
+    ('doc.wxs', r'doc'),
+    ('help.wxs', r'mercurial\help'),
+    ('i18n.wxs', r'i18n'),
+    ('locale.wxs', r'mercurial\locale'),
+    ('templates.wxs', r'mercurial\templates'),
+]
+
+
+EXTRA_PACKAGES = {
+    'distutils',
+    'pygments',
+}
+
+
+def find_version(source_dir: pathlib.Path):
+    version_py = source_dir / 'mercurial' / '__version__.py'
+
+    with version_py.open('r', encoding='utf-8') as fh:
+        source = fh.read().strip()
+
+    m = re.search('version = b"(.*)"', source)
+    return m.group(1)
+
+
+def normalize_version(version):
+    """Normalize Mercurial version string so WiX accepts it.
+
+    Version strings have to be numeric X.Y.Z.
+    """
+
+    if '+' in version:
+        version, extra = version.split('+', 1)
+    else:
+        extra = None
+
+    # 4.9rc0
+    if version[:-1].endswith('rc'):
+        version = version[:-3]
+
+    versions = [int(v) for v in version.split('.')]
+    while len(versions) < 3:
+        versions.append(0)
+
+    major, minor, build = versions[:3]
+
+    if extra:
+        # <commit count>-<hash>+<date>
+        build = int(extra.split('-')[0])
+
+    return '.'.join('%d' % x for x in (major, minor, build))
+
+
+def ensure_vc90_merge_modules(build_dir):
+    x86 = (
+        download_entry('vc9-crt-x86-msm', build_dir,
+                       local_name='microsoft.vcxx.crt.x86_msm.msm')[0],
+        download_entry('vc9-crt-x86-msm-policy', build_dir,
+                       local_name='policy.x.xx.microsoft.vcxx.crt.x86_msm.msm')[0]
+    )
+
+    x64 = (
+        download_entry('vc9-crt-x64-msm', build_dir,
+                       local_name='microsoft.vcxx.crt.x64_msm.msm')[0],
+        download_entry('vc9-crt-x64-msm-policy', build_dir,
+                       local_name='policy.x.xx.microsoft.vcxx.crt.x64_msm.msm')[0]
+    )
+    return {
+        'x86': x86,
+        'x64': x64,
+    }
+
+
+def run_candle(wix, cwd, wxs, source_dir, defines=None):
+    args = [
+        str(wix / 'candle.exe'),
+        '-nologo',
+        str(wxs),
+        '-dSourceDir=%s' % source_dir,
+    ]
+
+    if defines:
+        args.extend('-d%s=%s' % define for define in sorted(defines.items()))
+
+    subprocess.run(args, cwd=str(cwd), check=True)
+
+
+def make_post_build_signing_fn(name, subject_name=None, cert_path=None,
+                               cert_password=None, timestamp_url=None):
+    """Create a callable that will use signtool to sign hg.exe."""
+
+    def post_build_sign(source_dir, build_dir, dist_dir, version):
+        description = '%s %s' % (name, version)
+
+        sign_with_signtool(dist_dir / 'hg.exe', description,
+                           subject_name=subject_name, cert_path=cert_path,
+                           cert_password=cert_password,
+                           timestamp_url=timestamp_url)
+
+    return post_build_sign
+
+
+LIBRARIES_XML = '''
+<?xml version="1.0" encoding="utf-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+
+  <?include {wix_dir}/guids.wxi ?>
+  <?include {wix_dir}/defines.wxi ?>
+
+  <Fragment>
+    <DirectoryRef Id="INSTALLDIR" FileSource="$(var.SourceDir)">
+      <Directory Id="libdir" Name="lib" FileSource="$(var.SourceDir)/lib">
+        <Component Id="libOutput" Guid="$(var.lib.guid)" Win64='$(var.IsX64)'>
+        </Component>
+      </Directory>
+    </DirectoryRef>
+  </Fragment>
+</Wix>
+'''.lstrip()
+
+
+def make_libraries_xml(wix_dir: pathlib.Path, dist_dir: pathlib.Path):
+    """Make XML data for library components WXS."""
+    # We can't use ElementTree because it doesn't handle the
+    # <?include ?> directives.
+    doc = xml.dom.minidom.parseString(
+        LIBRARIES_XML.format(wix_dir=str(wix_dir)))
+
+    component = doc.getElementsByTagName('Component')[0]
+
+    f = doc.createElement('File')
+    f.setAttribute('Name', 'library.zip')
+    f.setAttribute('KeyPath', 'yes')
+    component.appendChild(f)
+
+    lib_dir = dist_dir / 'lib'
+
+    for p in sorted(lib_dir.iterdir()):
+        if not p.name.endswith(('.dll', '.pyd')):
+            continue
+
+        f = doc.createElement('File')
+        f.setAttribute('Name', p.name)
+        component.appendChild(f)
+
+    return doc.toprettyxml()
+
+
+def build_installer(source_dir: pathlib.Path, python_exe: pathlib.Path,
+                    msi_name='mercurial', version=None, post_build_fn=None,
+                    extra_packages_script=None,
+                    extra_wxs:typing.Optional[typing.Dict[str,str]]=None,
+                    extra_features:typing.Optional[typing.List[str]]=None):
+    """Build a WiX MSI installer.
+
+    ``source_dir`` is the path to the Mercurial source tree to use.
+    ``arch`` is the target architecture. either ``x86`` or ``x64``.
+    ``python_exe`` is the path to the Python executable to use/bundle.
+    ``version`` is the Mercurial version string. If not defined,
+    ``mercurial/__version__.py`` will be consulted.
+    ``post_build_fn`` is a callable that will be called after building
+    Mercurial but before invoking WiX. It can be used to e.g. facilitate
+    signing. It is passed the paths to the Mercurial source, build, and
+    dist directories and the resolved Mercurial version.
+    ``extra_packages_script`` is a command to be run to inject extra packages
+    into the py2exe binary. It should stage packages into the virtualenv and
+    print a null byte followed by a newline-separated list of packages that
+    should be included in the exe.
+    ``extra_wxs`` is a dict of {wxs_name: working_dir_for_wxs_build}.
+    ``extra_features`` is a list of additional named Features to include in
+    the build. These must match Feature names in one of the wxs scripts.
+    """
+    arch = 'x64' if r'\x64' in os.environ.get('LIB', '') else 'x86'
+
+    hg_build_dir = source_dir / 'build'
+    dist_dir = source_dir / 'dist'
+    wix_dir = source_dir / 'contrib' / 'packaging' / 'wix'
+
+    requirements_txt = wix_dir / 'requirements.txt'
+
+    build_py2exe(source_dir, hg_build_dir,
+                 python_exe, 'wix', requirements_txt,
+                 extra_packages=EXTRA_PACKAGES,
+                 extra_packages_script=extra_packages_script)
+
+    version = version or normalize_version(find_version(source_dir))
+    print('using version string: %s' % version)
+
+    if post_build_fn:
+        post_build_fn(source_dir, hg_build_dir, dist_dir, version)
+
+    build_dir = hg_build_dir / ('wix-%s' % arch)
+
+    build_dir.mkdir(exist_ok=True)
+
+    wix_pkg, wix_entry = download_entry('wix', hg_build_dir)
+    wix_path = hg_build_dir / ('wix-%s' % wix_entry['version'])
+
+    if not wix_path.exists():
+        extract_zip_to_directory(wix_pkg, wix_path)
+
+    ensure_vc90_merge_modules(hg_build_dir)
+
+    source_build_rel = pathlib.Path(os.path.relpath(source_dir, build_dir))
+
+    defines = {'Platform': arch}
+
+    for wxs, rel_path in SUPPORT_WXS:
+        wxs = wix_dir / wxs
+        wxs_source_dir = source_dir / rel_path
+        run_candle(wix_path, build_dir, wxs, wxs_source_dir, defines=defines)
+
+    for source, rel_path in sorted((extra_wxs or {}).items()):
+        run_candle(wix_path, build_dir, source, rel_path, defines=defines)
+
+    # candle.exe doesn't like when we have an open handle on the file.
+    # So use TemporaryDirectory() instead of NamedTemporaryFile().
+    with tempfile.TemporaryDirectory() as td:
+        td = pathlib.Path(td)
+
+        tf = td / 'library.wxs'
+        with tf.open('w') as fh:
+            fh.write(make_libraries_xml(wix_dir, dist_dir))
+
+        run_candle(wix_path, build_dir, tf, dist_dir, defines=defines)
+
+    source = wix_dir / 'mercurial.wxs'
+    defines['Version'] = version
+    defines['Comments'] = 'Installs Mercurial version %s' % version
+    defines['VCRedistSrcDir'] = str(hg_build_dir)
+    if extra_features:
+        assert all(';' not in f for f in extra_features)
+        defines['MercurialExtraFeatures'] = ';'.join(extra_features)
+
+    run_candle(wix_path, build_dir, source, source_build_rel, defines=defines)
+
+    msi_path = source_dir / 'dist' / (
+        '%s-%s-%s.msi' % (msi_name, version, arch))
+
+    args = [
+        str(wix_path / 'light.exe'),
+        '-nologo',
+        '-ext', 'WixUIExtension',
+        '-sw1076',
+        '-spdb',
+        '-o', str(msi_path),
+    ]
+
+    for source, rel_path in SUPPORT_WXS:
+        assert source.endswith('.wxs')
+        args.append(str(build_dir / ('%s.wixobj' % source[:-4])))
+
+    for source, rel_path in sorted((extra_wxs or {}).items()):
+        assert source.endswith('.wxs')
+        source = os.path.basename(source)
+        args.append(str(build_dir / ('%s.wixobj' % source[:-4])))
+
+    args.extend([
+        str(build_dir / 'library.wixobj'),
+        str(build_dir / 'mercurial.wixobj'),
+    ])
+
+    subprocess.run(args, cwd=str(source_dir), check=True)
+
+    print('%s created' % msi_path)
+
+    return {
+        'msi_path': msi_path,
+    }
+
+
+def build_signed_installer(source_dir: pathlib.Path, python_exe: pathlib.Path,
+                           name: str, version=None, subject_name=None,
+                           cert_path=None, cert_password=None,
+                           timestamp_url=None, extra_packages_script=None,
+                           extra_wxs=None, extra_features=None):
+    """Build an installer with signed executables."""
+
+    post_build_fn = make_post_build_signing_fn(
+        name,
+        subject_name=subject_name,
+        cert_path=cert_path,
+        cert_password=cert_password,
+        timestamp_url=timestamp_url)
+
+    info = build_installer(source_dir, python_exe=python_exe,
+                           msi_name=name.lower(), version=version,
+                           post_build_fn=post_build_fn,
+                           extra_packages_script=extra_packages_script,
+                           extra_wxs=extra_wxs, extra_features=extra_features)
+
+    description = '%s %s' % (name, version)
+
+    sign_with_signtool(info['msi_path'], description,
+                       subject_name=subject_name, cert_path=cert_path,
+                       cert_password=cert_password, timestamp_url=timestamp_url)

contrib/packaging/inno/build.py

@@ -0,0 +1,51 b''
+#!/usr/bin/env python3
+# build.py - Inno installer build script.
+#
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# This script automates the building of the Inno MSI installer for Mercurial.
+
+# no-check-code because Python 3 native.
+
+import argparse
+import os
+import pathlib
+import sys
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--python',
+                        required=True,
+                        help='path to python.exe to use')
+    parser.add_argument('--iscc',
+                        help='path to iscc.exe to use')
+    parser.add_argument('--version',
+                        help='Mercurial version string to use '
+                             '(detected from __version__.py if not defined')
+
+    args = parser.parse_args()
+
+    if not os.path.isabs(args.python):
+        raise Exception('--python arg must be an absolute path')
+
+    if args.iscc:
+        iscc = pathlib.Path(args.iscc)
+    else:
+        iscc = (pathlib.Path(os.environ['ProgramFiles(x86)']) / 'Inno Setup 5' /
+            'ISCC.exe')
+
+    here = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
+    source_dir = here.parent.parent.parent
+    build_dir = source_dir / 'build'
+
+    sys.path.insert(0, str(source_dir / 'contrib' / 'packaging'))
+
+    from hgpackaging.inno import build
+
+    build(source_dir, build_dir, pathlib.Path(args.python), iscc,
+          version=args.version)

contrib/packaging/inno/modpath.iss

@@ -0,0 +1,219 b''
+// ----------------------------------------------------------------------------
+//
+// Inno Setup Ver:	5.4.2
+// Script Version:	1.4.2
+// Author:			Jared Breland <[e-mail]>
+// Homepage:		http://www.legroom.net/software
+// License:			GNU Lesser General Public License (LGPL), version 3
+//						http://www.gnu.org/licenses/lgpl.html
+//
+// Script Function:
+//	Allow modification of environmental path directly from Inno Setup installers
+//
+// Instructions:
+//	Copy modpath.iss to the same directory as your setup script
+//
+//	Add this statement to your [Setup] section
+//		ChangesEnvironment=true
+//
+//	Add this statement to your [Tasks] section
+//	You can change the Description or Flags
+//	You can change the Name, but it must match the ModPathName setting below
+//		Name: modifypath; Description: &Add application directory to your environmental path; Flags: unchecked
+//
+//	Add the following to the end of your [Code] section
+//	ModPathName defines the name of the task defined above
+//	ModPathType defines whether the 'user' or 'system' path will be modified;
+//		this will default to user if anything other than system is set
+//	setArrayLength must specify the total number of dirs to be added
+//	Result[0] contains first directory, Result[1] contains second, etc.
+//		const
+//			ModPathName = 'modifypath';
+//			ModPathType = 'user';
+//
+//		function ModPathDir(): TArrayOfString;
+//		begin
+//			setArrayLength(Result, 1);
+//			Result[0] := ExpandConstant('{app}');
+//		end;
+//		#include "modpath.iss"
+// ----------------------------------------------------------------------------
+
+procedure ModPath();
+var
+	oldpath:	String;
+	newpath:	String;
+	updatepath:	Boolean;
+	pathArr:	TArrayOfString;
+	aExecFile:	String;
+	aExecArr:	TArrayOfString;
+	i, d:		Integer;
+	pathdir:	TArrayOfString;
+	regroot:	Integer;
+	regpath:	String;
+
+begin
+	// Get constants from main script and adjust behavior accordingly
+	// ModPathType MUST be 'system' or 'user'; force 'user' if invalid
+	if ModPathType = 'system' then begin
+		regroot := HKEY_LOCAL_MACHINE;
+		regpath := 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment';
+	end else begin
+		regroot := HKEY_CURRENT_USER;
+		regpath := 'Environment';
+	end;
+
+	// Get array of new directories and act on each individually
+	pathdir := ModPathDir();
+	for d := 0 to GetArrayLength(pathdir)-1 do begin
+		updatepath := true;
+
+		// Modify WinNT path
+		if UsingWinNT() = true then begin
+
+			// Get current path, split into an array
+			RegQueryStringValue(regroot, regpath, 'Path', oldpath);
+			oldpath := oldpath + ';';
+			i := 0;
+
+			while (Pos(';', oldpath) > 0) do begin
+				SetArrayLength(pathArr, i+1);
+				pathArr[i] := Copy(oldpath, 0, Pos(';', oldpath)-1);
+				oldpath := Copy(oldpath, Pos(';', oldpath)+1, Length(oldpath));
+				i := i + 1;
+
+				// Check if current directory matches app dir
+				if pathdir[d] = pathArr[i-1] then begin
+					// if uninstalling, remove dir from path
+					if IsUninstaller() = true then begin
+						continue;
+					// if installing, flag that dir already exists in path
+					end else begin
+						updatepath := false;
+					end;
+				end;
+
+				// Add current directory to new path
+				if i = 1 then begin
+					newpath := pathArr[i-1];
+				end else begin
+					newpath := newpath + ';' + pathArr[i-1];
+				end;
+			end;
+
+			// Append app dir to path if not already included
+			if (IsUninstaller() = false) AND (updatepath = true) then
+				newpath := newpath + ';' + pathdir[d];
+
+			// Write new path
+			RegWriteStringValue(regroot, regpath, 'Path', newpath);
+
+		// Modify Win9x path
+		end else begin
+
+			// Convert to shortened dirname
+			pathdir[d] := GetShortName(pathdir[d]);
+
+			// If autoexec.bat exists, check if app dir already exists in path
+			aExecFile := 'C:\AUTOEXEC.BAT';
+			if FileExists(aExecFile) then begin
+				LoadStringsFromFile(aExecFile, aExecArr);
+				for i := 0 to GetArrayLength(aExecArr)-1 do begin
+					if IsUninstaller() = false then begin
+						// If app dir already exists while installing, skip add
+						if (Pos(pathdir[d], aExecArr[i]) > 0) then
+							updatepath := false;
+							break;
+					end else begin
+						// If app dir exists and = what we originally set, then delete at uninstall
+						if aExecArr[i] = 'SET PATH=%PATH%;' + pathdir[d] then
+							aExecArr[i] := '';
+					end;
+				end;
+			end;
+
+			// If app dir not found, or autoexec.bat didn't exist, then (create and) append to current path
+			if (IsUninstaller() = false) AND (updatepath = true) then begin
+				SaveStringToFile(aExecFile, #13#10 + 'SET PATH=%PATH%;' + pathdir[d], True);
+
+			// If uninstalling, write the full autoexec out
+			end else begin
+				SaveStringsToFile(aExecFile, aExecArr, False);
+			end;
+		end;
+	end;
+end;
+
+// Split a string into an array using passed delimeter
+procedure MPExplode(var Dest: TArrayOfString; Text: String; Separator: String);
+var
+	i: Integer;
+begin
+	i := 0;
+	repeat
+		SetArrayLength(Dest, i+1);
+		if Pos(Separator,Text) > 0 then	begin
+			Dest[i] := Copy(Text, 1, Pos(Separator, Text)-1);
+			Text := Copy(Text, Pos(Separator,Text) + Length(Separator), Length(Text));
+			i := i + 1;
+		end else begin
+			 Dest[i] := Text;
+			 Text := '';
+		end;
+	until Length(Text)=0;
+end;
+
+
+procedure CurStepChanged(CurStep: TSetupStep);
+var
+	taskname:	String;
+begin
+	taskname := ModPathName;
+	if CurStep = ssPostInstall then
+		if IsTaskSelected(taskname) then
+			ModPath();
+end;
+
+procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
+var
+	aSelectedTasks:	TArrayOfString;
+	i:				Integer;
+	taskname:		String;
+	regpath:		String;
+	regstring:		String;
+	appid:			String;
+begin
+	// only run during actual uninstall
+	if CurUninstallStep = usUninstall then begin
+		// get list of selected tasks saved in registry at install time
+		appid := '{#emit SetupSetting("AppId")}';
+		if appid = '' then appid := '{#emit SetupSetting("AppName")}';
+		regpath := ExpandConstant('Software\Microsoft\Windows\CurrentVersion\Uninstall\'+appid+'_is1');
+		RegQueryStringValue(HKLM, regpath, 'Inno Setup: Selected Tasks', regstring);
+		if regstring = '' then RegQueryStringValue(HKCU, regpath, 'Inno Setup: Selected Tasks', regstring);
+
+		// check each task; if matches modpath taskname, trigger patch removal
+		if regstring <> '' then begin
+			taskname := ModPathName;
+			MPExplode(aSelectedTasks, regstring, ',');
+			if GetArrayLength(aSelectedTasks) > 0 then begin
+				for i := 0 to GetArrayLength(aSelectedTasks)-1 do begin
+					if comparetext(aSelectedTasks[i], taskname) = 0 then
+						ModPath();
+				end;
+			end;
+		end;
+	end;
+end;
+
+function NeedRestart(): Boolean;
+var
+	taskname:	String;
+begin
+	taskname := ModPathName;
+	if IsTaskSelected(taskname) and not UsingWinNT() then begin
+		Result := True;
+	end else begin
+		Result := False;
+	end;
+end;

contrib/packaging/inno/requirements.txt

@@ -0,0 +1,38 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes contrib/packaging/inno/requirements.txt.in -o contrib/packaging/inno/requirements.txt
+#
+certifi==2018.11.29 \
+    --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
+    --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \
+    # via dulwich
+configparser==3.7.3 \
+    --hash=sha256:27594cf4fc279f321974061ac69164aaebd2749af962ac8686b20503ac0bcf2d \
+    --hash=sha256:9d51fe0a382f05b6b117c5e601fc219fede4a8c71703324af3f7d883aef476a3 \
+    # via entrypoints
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
+dulwich==0.19.11 \
+    --hash=sha256:afbe070f6899357e33f63f3f3696e601731fef66c64a489dea1bc9f539f4a725
+entrypoints==0.3 \
+    --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \
+    --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \
+    # via keyring
+keyring==18.0.0 \
+    --hash=sha256:12833d2b05d2055e0e25931184af9cd6a738f320a2264853cabbd8a3a0f0b65d \
+    --hash=sha256:ca33f5ccc542b9ffaa196ee9a33488069e5e7eac77d5b81969f8a3ce74d0230c
+pygments==2.3.1 \
+    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
+    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d
+pywin32-ctypes==0.2.0 \
+    --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
+    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98 \
+    # via keyring
+urllib3==1.24.1 \
+    --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
+    --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
+    # via dulwich

contrib/packaging/inno/requirements.txt.in

@@ -0,0 +1,4 b''
+docutils
+dulwich
+keyring
+pygments

contrib/packaging/wix/build.py

@@ -0,0 +1,84 b''
+#!/usr/bin/env python3
+# Copyright 2019 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+"""Code to build Mercurial WiX installer."""
+
+import argparse
+import os
+import pathlib
+import sys
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--name',
+                        help='Application name',
+                        default='Mercurial')
+    parser.add_argument('--python',
+                        help='Path to Python executable to use',
+                        required=True)
+    parser.add_argument('--sign-sn',
+                        help='Subject name (or fragment thereof) of certificate '
+                             'to use for signing')
+    parser.add_argument('--sign-cert',
+                        help='Path to certificate to use for signing')
+    parser.add_argument('--sign-password',
+                        help='Password for signing certificate')
+    parser.add_argument('--sign-timestamp-url',
+                        help='URL of timestamp server to use for signing')
+    parser.add_argument('--version',
+                        help='Version string to use')
+    parser.add_argument('--extra-packages-script',
+                        help=('Script to execute to include extra packages in '
+                              'py2exe binary.'))
+    parser.add_argument('--extra-wxs',
+                        help='CSV of path_to_wxs_file=working_dir_for_wxs_file')
+    parser.add_argument('--extra-features',
+                        help=('CSV of extra feature names to include '
+                              'in the installer from the extra wxs files'))
+
+    args = parser.parse_args()
+
+    here = pathlib.Path(os.path.abspath(os.path.dirname(__file__)))
+    source_dir = here.parent.parent.parent
+
+    sys.path.insert(0, str(source_dir / 'contrib' / 'packaging'))
+
+    from hgpackaging.wix import (
+        build_installer,
+        build_signed_installer,
+    )
+
+    fn = build_installer
+    kwargs = {
+        'source_dir': source_dir,
+        'python_exe': pathlib.Path(args.python),
+        'version': args.version,
+    }
+
+    if not os.path.isabs(args.python):
+        raise Exception('--python arg must be an absolute path')
+
+    if args.extra_packages_script:
+        kwargs['extra_packages_script'] = args.extra_packages_script
+    if args.extra_wxs:
+        kwargs['extra_wxs'] = dict(
+            thing.split("=") for thing in args.extra_wxs.split(','))
+    if args.extra_features:
+        kwargs['extra_features'] = args.extra_features.split(',')
+
+    if args.sign_sn or args.sign_cert:
+        fn = build_signed_installer
+        kwargs['name'] = args.name
+        kwargs['subject_name'] = args.sign_sn
+        kwargs['cert_path'] = args.sign_cert
+        kwargs['cert_password'] = args.sign_password
+        kwargs['timestamp_url'] = args.sign_timestamp_url
+
+    fn(**kwargs)

contrib/packaging/wix/requirements.txt

@@ -0,0 +1,13 b''
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes contrib/packaging/wix/requirements.txt.in -o contrib/packaging/wix/requirements.txt
+#
+docutils==0.14 \
+    --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
+    --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
+    --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
+pygments==2.3.1 \
+    --hash=sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a \
+    --hash=sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d

contrib/packaging/wix/requirements.txt.in

@@ -0,0 +1,2 b''
+docutils
+pygments

Makefile

@@ -5,7 +5,7 b''
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
-PYTHON=python
+PYTHON?=python
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=

contrib/base-revsets.txt

@@ -47,3 +47,6 b' parents(20000)'
 # The one below is used by rebase
 (children(ancestor(tip~5, tip)) and ::(tip~5))::
 heads(commonancestors(last(head(), 2)))
+heads(-10000:-1)
+roots(-10000:-1)
+only(max(head()), min(head()))

contrib/bdiff-torture.py

@@ -25,7 +25,7 b' def reducetest(a, b):'
 
         try:
             test1(a, b)
-        except Exception as inst:
+        except Exception:
             reductions += 1
             tries = 0
             a = a2

contrib/check-code.py

@@ -40,6 +40,8 b' try:'
 except ImportError:
     re2 = None
 
+import testparseutil
+
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
@@ -231,8 +233,10 b' utestfilters = ['
     (r"( +)(#([^!][^\n]*\S)?)", repcomment),
 ]
 
-pypats = [
+# common patterns to check *.py
+commonpypats = [
   [
+    (r'\\$', 'Use () to wrap long lines in Python, not \\'),
     (r'^\s*def\s*\w+\s*\(.*,\s*\(',
      "tuple parameter unpacking not available in Python 3+"),
     (r'lambda\s*\(.*,.*\)',
@@ -261,7 +265,6 b' pypats = ['
         # a pass at the same indent level, which is bogus
         r'(?P=indent)pass[ \t\n#]'
       ), 'omit superfluous pass'),
-    (r'.{81}', "line too long"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
 #    (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=',
@@ -299,7 +302,6 b' pypats = ['
      "wrong whitespace around ="),
     (r'\([^()]*( =[^=]|[^<>!=]= )',
      "no whitespace around = for named parameters"),
-    (r'raise Exception', "don't raise generic exceptions"),
     (r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
      "don't use old-style two-argument raise, use Exception(message)"),
     (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
@@ -315,21 +317,12 b' pypats = ['
      "use opener.read() instead"),
     (r'opener\([^)]*\).write\(',
      "use opener.write() instead"),
-    (r'[\s\(](open|file)\([^)]*\)\.read\(',
-     "use util.readfile() instead"),
-    (r'[\s\(](open|file)\([^)]*\)\.write\(',
-     "use util.writefile() instead"),
-    (r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
-     "always assign an opened file to a variable, and close it afterwards"),
-    (r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
-     "always assign an opened file to a variable, and close it afterwards"),
     (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
     (r'\.debug\(\_', "don't mark debug messages for translation"),
     (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
     (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
     (r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
      'legacy exception syntax; use "as" instead of ","'),
-    (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
     (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
     (r'\bdef\s+__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
     (r'os\.path\.join\(.*, *(""|\'\')\)',
@@ -339,7 +332,6 b' pypats = ['
     (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
     (r'\butil\.Abort\b', "directly use error.Abort"),
     (r'^@(\w*\.)?cachefunc', "module-level @cachefunc is risky, please avoid"),
-    (r'^import atexit', "don't use atexit, use ui.atexit"),
     (r'^import Queue', "don't use Queue, use pycompat.queue.Queue + "
                        "pycompat.queue.Empty"),
     (r'^import cStringIO', "don't use cStringIO.StringIO, use util.stringio"),
@@ -358,6 +350,34 b' pypats = ['
      "don't convert rev to node before passing to revision(nodeorrev)"),
     (r'platform\.system\(\)', "don't use platform.system(), use pycompat"),
 
+  ],
+  # warnings
+  [
+  ]
+]
+
+# patterns to check normal *.py files
+pypats = [
+  [
+    # Ideally, these should be placed in "commonpypats" for
+    # consistency of coding rules in Mercurial source tree.
+    # But on the other hand, these are not so seriously required for
+    # python code fragments embedded in test scripts. Fixing test
+    # scripts for these patterns requires many changes, and has less
+    # profit than effort.
+    (r'.{81}', "line too long"),
+    (r'raise Exception', "don't raise generic exceptions"),
+    (r'[\s\(](open|file)\([^)]*\)\.read\(',
+     "use util.readfile() instead"),
+    (r'[\s\(](open|file)\([^)]*\)\.write\(',
+     "use util.writefile() instead"),
+    (r'^[\s\(]*(open(er)?|file)\([^)]*\)(?!\.close\(\))',
+     "always assign an opened file to a variable, and close it afterwards"),
+    (r'[\s\(](open|file)\([^)]*\)\.(?!close\(\))',
+     "always assign an opened file to a variable, and close it afterwards"),
+    (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
+    (r'^import atexit', "don't use atexit, use ui.atexit"),
+
     # rules depending on implementation of repquote()
     (r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
      'string join across lines with no space'),
@@ -376,21 +396,35 b' pypats = ['
            # because _preparepats forcibly adds "\n" into [^...],
            # even though this regexp wants match it against "\n")''',
      "missing _() in ui message (use () to hide false-positives)"),
-  ],
+  ] + commonpypats[0],
   # warnings
   [
     # rules depending on implementation of repquote()
     (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
-  ]
+  ] + commonpypats[1]
 ]
 
-pyfilters = [
+# patterns to check *.py for embedded ones in test script
+embeddedpypats = [
+  [
+  ] + commonpypats[0],
+  # warnings
+  [
+  ] + commonpypats[1]
+]
+
+# common filters to convert *.py
+commonpyfilters = [
     (r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""", reppython),
 ]
 
+# filters to convert normal *.py files
+pyfilters = [
+] + commonpyfilters
+
 # non-filter patterns
 pynfpats = [
     [
@@ -403,6 +437,10 b' pynfpats = ['
     [],
 ]
 
+# filters to convert *.py for embedded ones in test script
+embeddedpyfilters = [
+] + commonpyfilters
+
 # extension non-filter patterns
 pyextnfpats = [
     [(r'^"""\n?[A-Z]', "don't capitalize docstring title")],
@@ -414,7 +452,7 b' txtfilters = []'
 
 txtpats = [
   [
-    ('\s$', 'trailing whitespace'),
+    (r'\s$', 'trailing whitespace'),
     ('.. note::[ \n][^\n]', 'add two newlines after note::')
   ],
   []
@@ -537,9 +575,17 b' checks = ['
      allfilesfilters, allfilespats),
 ]
 
+# (desc,
+#  func to pick up embedded code fragments,
+#  list of patterns to convert target files
+#  list of patterns to detect errors/warnings)
+embeddedchecks = [
+    ('embedded python',
+     testparseutil.pyembedded, embeddedpyfilters, embeddedpypats)
+]
+
 def _preparepats():
-    for c in checks:
-        failandwarn = c[-1]
+    def preparefailandwarn(failandwarn):
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
@@ -553,10 +599,19 b' def _preparepats():'
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
-        filters = c[3]
+
+    def preparefilters(filters):
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
 
+    for cs in (checks, embeddedchecks):
+        for c in cs:
+            failandwarn = c[-1]
+            preparefailandwarn(failandwarn)
+
+            filters = c[-2]
+            preparefilters(filters)
+
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
@@ -604,13 +659,12 b' def checkfile(f, logfunc=_defaultlogger.'
 
     return True if no error is found, False otherwise.
     """
-    blamecache = None
     result = True
 
     try:
         with opentext(f) as fp:
             try:
-                pre = post = fp.read()
+                pre = fp.read()
             except UnicodeDecodeError as e:
                 print("%s while reading %s" % (e, f))
                 return result
@@ -618,11 +672,12 b' def checkfile(f, logfunc=_defaultlogger.'
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
 
+    # context information shared while single checkfile() invocation
+    context = {'blamecache': None}
+
     for name, match, magic, filters, pats in checks:
-        post = pre # discard filtering result of previous check
         if debug:
             print(name, f)
-        fc = 0
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print("Skipping %s for %s it doesn't match %s" % (
@@ -637,6 +692,74 b' def checkfile(f, logfunc=_defaultlogger.'
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip" # skip checking this file
+
+        fc = _checkfiledata(name, f, pre, filters, pats, context,
+                            logfunc, maxerr, warnings, blame, debug, lineno)
+        if fc:
+            result = False
+
+    if f.endswith('.t') and "no-" "check-code" not in pre:
+        if debug:
+            print("Checking embedded code in %s" % (f))
+
+        prelines = pre.splitlines()
+        embeddederros = []
+        for name, embedded, filters, pats in embeddedchecks:
+            # "reset curmax at each repetition" treats maxerr as "max
+            # nubmer of errors in an actual file per entry of
+            # (embedded)checks"
+            curmaxerr = maxerr
+
+            for found in embedded(f, prelines, embeddederros):
+                filename, starts, ends, code = found
+                fc = _checkfiledata(name, f, code, filters, pats, context,
+                                    logfunc, curmaxerr, warnings, blame, debug,
+                                    lineno, offset=starts - 1)
+                if fc:
+                    result = False
+                    if curmaxerr:
+                        if fc >= curmaxerr:
+                            break
+                        curmaxerr -= fc
+
+    return result
+
+def _checkfiledata(name, f, filedata, filters, pats, context,
+                   logfunc, maxerr, warnings, blame, debug, lineno,
+                   offset=None):
+    """Execute actual error check for file data
+
+    :name: of the checking category
+    :f: filepath
+    :filedata: content of a file
+    :filters: to be applied before checking
+    :pats: to detect errors
+    :context: a dict of information shared while single checkfile() invocation
+              Valid keys: 'blamecache'.
+    :logfunc: function used to report error
+              logfunc(filename, linenumber, linecontent, errormessage)
+    :maxerr: number of error to display before aborting, or False to
+             report all errors
+    :warnings: whether warning level checks should be applied
+    :blame: whether blame information should be displayed at error reporting
+    :debug: whether debug information should be displayed
+    :lineno: whether lineno should be displayed at error reporting
+    :offset: line number offset of 'filedata' in 'f' for checking
+             an embedded code fragment, or None (offset=0 is different
+             from offset=None)
+
+    returns number of detected errors.
+    """
+    blamecache = context['blamecache']
+    if offset is None:
+        lineoffset = 0
+    else:
+        lineoffset = offset
+
+    fc = 0
+    pre = post = filedata
+
+    if True: # TODO: get rid of this redundant 'if' block
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0]) # nerr elements are errors
@@ -679,20 +802,30 b' def checkfile(f, logfunc=_defaultlogger.'
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print("Skipping %s for %s:%s (ignore pattern)" % (
-                            name, f, n))
+                            name, f, (n + lineoffset)))
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
-                    if not blamecache:
+                    if blamecache is None:
                         blamecache = getblame(f)
-                    if n < len(blamecache):
-                        bl, bu, br = blamecache[n]
-                        if bl == l:
+                        context['blamecache'] = blamecache
+                    if (n + lineoffset) < len(blamecache):
+                        bl, bu, br = blamecache[(n + lineoffset)]
+                        if offset is None and bl == l:
                             bd = '%s@%s' % (bu, br)
+                        elif offset is not None and bl.endswith(l):
+                            # "offset is not None" means "checking
+                            # embedded code fragment". In this case,
+                            # "l" does not have information about the
+                            # beginning of an *original* line in the
+                            # file (e.g. '  > ').
+                            # Therefore, use "str.endswith()", and
+                            # show "maybe" for a little loose
+                            # examination.
+                            bd = '%s@%s, maybe' % (bu, br)
 
-                errors.append((f, lineno and n + 1, l, msg, bd))
-                result = False
+                errors.append((f, lineno and (n + lineoffset + 1), l, msg, bd))
 
         errors.sort()
         for e in errors:
@@ -702,7 +835,7 b' def checkfile(f, logfunc=_defaultlogger.'
                 print(" (too many errors, giving up)")
                 break
 
-    return result
+    return fc
 
 def main():
     parser = optparse.OptionParser("%prog [options] [files | -]")

contrib/check-commit

@@ -47,7 +47,7 b' errors = ['
      "adds a function with foo_bar naming"),
 ]
 
-word = re.compile('\S')
+word = re.compile(r'\S')
 def nonempty(first, second):
     if word.search(first):
         return first

contrib/check-config.py

@@ -25,7 +25,7 b" configre = re.compile(br'''"
         (?:default=)?(?P<default>\S+?))?
     \)''', re.VERBOSE | re.MULTILINE)
 
-configwithre = re.compile(b'''
+configwithre = re.compile(br'''
     ui\.config(?P<ctype>with)\(
         # First argument is callback function. This doesn't parse robustly
         # if it is e.g. a function call.
@@ -61,10 +61,10 b' def main(args):'
             linenum += 1
 
             # check topic-like bits
-            m = re.match(b'\s*``(\S+)``', l)
+            m = re.match(br'\s*``(\S+)``', l)
             if m:
                 prevname = m.group(1)
-            if re.match(b'^\s*-+$', l):
+            if re.match(br'^\s*-+$', l):
                 sect = prevname
                 prevname = b''
 

contrib/check-py3-compat.py

@@ -14,6 +14,7 b' import importlib'
 import os
 import sys
 import traceback
+import warnings
 
 def check_compat_py2(f):
     """Check Python 3 compatibility for a file with Python 2"""
@@ -45,7 +46,7 b' def check_compat_py3(f):'
         content = fh.read()
 
     try:
-        ast.parse(content)
+        ast.parse(content, filename=f)
     except SyntaxError as e:
         print('%s: invalid syntax: %s' % (f, e))
         return
@@ -91,6 +92,11 b" if __name__ == '__main__':"
         fn = check_compat_py3
 
     for f in sys.argv[1:]:
-        fn(f)
+        with warnings.catch_warnings(record=True) as warns:
+            fn(f)
+
+        for w in warns:
+            print(warnings.formatwarning(w.message, w.category,
+                                         w.filename, w.lineno).rstrip())
 
     sys.exit(0)

contrib/chg/hgclient.c

@@ -84,8 +84,9 b' static void initcontext(context_t *ctx)'
 
 static void enlargecontext(context_t *ctx, size_t newsize)
 {
-	if (newsize <= ctx->maxdatasize)
+	if (newsize <= ctx->maxdatasize) {
 		return;
+	}
 
 	newsize = defaultdatasize *
 	          ((newsize + defaultdatasize - 1) / defaultdatasize);
@@ -117,22 +118,25 b' static void readchannel(hgclient_t *hgc)'
 
 	uint32_t datasize_n;
 	rsize = recv(hgc->sockfd, &datasize_n, sizeof(datasize_n), 0);
-	if (rsize != sizeof(datasize_n))
+	if (rsize != sizeof(datasize_n)) {
 		abortmsg("failed to read data size");
+	}
 
 	/* datasize denotes the maximum size to write if input request */
 	hgc->ctx.datasize = ntohl(datasize_n);
 	enlargecontext(&hgc->ctx, hgc->ctx.datasize);
 
-	if (isupper(hgc->ctx.ch) && hgc->ctx.ch != 'S')
+	if (isupper(hgc->ctx.ch) && hgc->ctx.ch != 'S') {
 		return; /* assumes input request */
+	}
 
 	size_t cursize = 0;
 	while (cursize < hgc->ctx.datasize) {
 		rsize = recv(hgc->sockfd, hgc->ctx.data + cursize,
 		             hgc->ctx.datasize - cursize, 0);
-		if (rsize < 1)
+		if (rsize < 1) {
 			abortmsg("failed to read data block");
+		}
 		cursize += rsize;
 	}
 }
@@ -143,8 +147,9 b' static void sendall(int sockfd, const vo'
 	const char *const endp = p + datasize;
 	while (p < endp) {
 		ssize_t r = send(sockfd, p, endp - p, 0);
-		if (r < 0)
+		if (r < 0) {
 			abortmsgerrno("cannot communicate");
+		}
 		p += r;
 	}
 }
@@ -186,8 +191,9 b' static void packcmdargs(context_t *ctx, '
 		ctx->datasize += n;
 	}
 
-	if (ctx->datasize > 0)
+	if (ctx->datasize > 0) {
 		--ctx->datasize; /* strip last '\0' */
+	}
 }
 
 /* Extract '\0'-separated list of args to new buffer, terminated by NULL */
@@ -205,8 +211,9 b' static const char **unpackcmdargsnul(con'
 		args[nargs] = s;
 		nargs++;
 		s = memchr(s, '\0', e - s);
-		if (!s)
+		if (!s) {
 			break;
+		}
 		s++;
 	}
 	args[nargs] = NULL;
@@ -225,8 +232,9 b' static void handlereadrequest(hgclient_t'
 static void handlereadlinerequest(hgclient_t *hgc)
 {
 	context_t *ctx = &hgc->ctx;
-	if (!fgets(ctx->data, ctx->datasize, stdin))
+	if (!fgets(ctx->data, ctx->datasize, stdin)) {
 		ctx->data[0] = '\0';
+	}
 	ctx->datasize = strlen(ctx->data);
 	writeblock(hgc);
 }
@@ -239,8 +247,9 b' static void handlesystemrequest(hgclient'
 	ctx->data[ctx->datasize] = '\0'; /* terminate last string */
 
 	const char **args = unpackcmdargsnul(ctx);
-	if (!args[0] || !args[1] || !args[2])
+	if (!args[0] || !args[1] || !args[2]) {
 		abortmsg("missing type or command or cwd in system request");
+	}
 	if (strcmp(args[0], "system") == 0) {
 		debugmsg("run '%s' at '%s'", args[1], args[2]);
 		int32_t r = runshellcmd(args[1], args + 3, args[2]);
@@ -252,8 +261,9 b' static void handlesystemrequest(hgclient'
 		writeblock(hgc);
 	} else if (strcmp(args[0], "pager") == 0) {
 		setuppager(args[1], args + 3);
-		if (hgc->capflags & CAP_ATTACHIO)
+		if (hgc->capflags & CAP_ATTACHIO) {
 			attachio(hgc);
+		}
 		/* unblock the server */
 		static const char emptycmd[] = "\n";
 		sendall(hgc->sockfd, emptycmd, sizeof(emptycmd) - 1);
@@ -296,9 +306,10 b' static void handleresponse(hgclient_t *h'
 			handlesystemrequest(hgc);
 			break;
 		default:
-			if (isupper(ctx->ch))
+			if (isupper(ctx->ch)) {
 				abortmsg("cannot handle response (ch = %c)",
 				         ctx->ch);
+			}
 		}
 	}
 }
@@ -308,8 +319,9 b' static unsigned int parsecapabilities(co'
 	unsigned int flags = 0;
 	while (s < e) {
 		const char *t = strchr(s, ' ');
-		if (!t || t > e)
+		if (!t || t > e) {
 			t = e;
+		}
 		const cappair_t *cap;
 		for (cap = captable; cap->flag; ++cap) {
 			size_t n = t - s;
@@ -346,11 +358,13 b' static void readhello(hgclient_t *hgc)'
 	const char *const dataend = ctx->data + ctx->datasize;
 	while (s < dataend) {
 		const char *t = strchr(s, ':');
-		if (!t || t[1] != ' ')
+		if (!t || t[1] != ' ') {
 			break;
+		}
 		const char *u = strchr(t + 2, '\n');
-		if (!u)
+		if (!u) {
 			u = dataend;
+		}
 		if (strncmp(s, "capabilities:", t - s + 1) == 0) {
 			hgc->capflags = parsecapabilities(t + 2, u);
 		} else if (strncmp(s, "pgid:", t - s + 1) == 0) {
@@ -367,8 +381,9 b' static void updateprocname(hgclient_t *h'
 {
 	int r = snprintf(hgc->ctx.data, hgc->ctx.maxdatasize, "chg[worker/%d]",
 	                 (int)getpid());
-	if (r < 0 || (size_t)r >= hgc->ctx.maxdatasize)
+	if (r < 0 || (size_t)r >= hgc->ctx.maxdatasize) {
 		abortmsg("insufficient buffer to write procname (r = %d)", r);
+	}
 	hgc->ctx.datasize = (size_t)r;
 	writeblockrequest(hgc, "setprocname");
 }
@@ -380,8 +395,9 b' static void attachio(hgclient_t *hgc)'
 	sendall(hgc->sockfd, chcmd, sizeof(chcmd) - 1);
 	readchannel(hgc);
 	context_t *ctx = &hgc->ctx;
-	if (ctx->ch != 'I')
+	if (ctx->ch != 'I') {
 		abortmsg("unexpected response for attachio (ch = %c)", ctx->ch);
+	}
 
 	static const int fds[3] = {STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO};
 	struct msghdr msgh;
@@ -399,23 +415,27 b' static void attachio(hgclient_t *hgc)'
 	memcpy(CMSG_DATA(cmsg), fds, sizeof(fds));
 	msgh.msg_controllen = cmsg->cmsg_len;
 	ssize_t r = sendmsg(hgc->sockfd, &msgh, 0);
-	if (r < 0)
+	if (r < 0) {
 		abortmsgerrno("sendmsg failed");
+	}
 
 	handleresponse(hgc);
 	int32_t n;
-	if (ctx->datasize != sizeof(n))
+	if (ctx->datasize != sizeof(n)) {
 		abortmsg("unexpected size of attachio result");
+	}
 	memcpy(&n, ctx->data, sizeof(n));
 	n = ntohl(n);
-	if (n != sizeof(fds) / sizeof(fds[0]))
+	if (n != sizeof(fds) / sizeof(fds[0])) {
 		abortmsg("failed to send fds (n = %d)", n);
+	}
 }
 
 static void chdirtocwd(hgclient_t *hgc)
 {
-	if (!getcwd(hgc->ctx.data, hgc->ctx.maxdatasize))
+	if (!getcwd(hgc->ctx.data, hgc->ctx.maxdatasize)) {
 		abortmsgerrno("failed to getcwd");
+	}
 	hgc->ctx.datasize = strlen(hgc->ctx.data);
 	writeblockrequest(hgc, "chdir");
 }
@@ -440,8 +460,9 b' static void forwardumask(hgclient_t *hgc'
 hgclient_t *hgc_open(const char *sockname)
 {
 	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
-	if (fd < 0)
+	if (fd < 0) {
 		abortmsgerrno("cannot create socket");
+	}
 
 	/* don't keep fd on fork(), so that it can be closed when the parent
 	 * process get terminated. */
@@ -456,34 +477,39 b' hgclient_t *hgc_open(const char *socknam'
 	{
 		const char *split = strrchr(sockname, '/');
 		if (split && split != sockname) {
-			if (split[1] == '\0')
+			if (split[1] == '\0') {
 				abortmsg("sockname cannot end with a slash");
+			}
 			size_t len = split - sockname;
 			char sockdir[len + 1];
 			memcpy(sockdir, sockname, len);
 			sockdir[len] = '\0';
 
 			bakfd = open(".", O_DIRECTORY);
-			if (bakfd == -1)
+			if (bakfd == -1) {
 				abortmsgerrno("cannot open cwd");
+			}
 
 			int r = chdir(sockdir);
-			if (r != 0)
+			if (r != 0) {
 				abortmsgerrno("cannot chdir %s", sockdir);
+			}
 
 			basename = split + 1;
 		}
 	}
-	if (strlen(basename) >= sizeof(addr.sun_path))
+	if (strlen(basename) >= sizeof(addr.sun_path)) {
 		abortmsg("sockname is too long: %s", basename);
+	}
 	strncpy(addr.sun_path, basename, sizeof(addr.sun_path));
 	addr.sun_path[sizeof(addr.sun_path) - 1] = '\0';
 
 	/* real connect */
 	int r = connect(fd, (struct sockaddr *)&addr, sizeof(addr));
 	if (r < 0) {
-		if (errno != ENOENT && errno != ECONNREFUSED)
+		if (errno != ENOENT && errno != ECONNREFUSED) {
 			abortmsgerrno("cannot connect to %s", sockname);
+		}
 	}
 	if (bakfd != -1) {
 		fchdirx(bakfd);
@@ -501,16 +527,21 b' hgclient_t *hgc_open(const char *socknam'
 	initcontext(&hgc->ctx);
 
 	readhello(hgc);
-	if (!(hgc->capflags & CAP_RUNCOMMAND))
+	if (!(hgc->capflags & CAP_RUNCOMMAND)) {
 		abortmsg("insufficient capability: runcommand");
-	if (hgc->capflags & CAP_SETPROCNAME)
+	}
+	if (hgc->capflags & CAP_SETPROCNAME) {
 		updateprocname(hgc);
-	if (hgc->capflags & CAP_ATTACHIO)
+	}
+	if (hgc->capflags & CAP_ATTACHIO) {
 		attachio(hgc);
-	if (hgc->capflags & CAP_CHDIR)
+	}
+	if (hgc->capflags & CAP_CHDIR) {
 		chdirtocwd(hgc);
-	if (hgc->capflags & CAP_SETUMASK2)
+	}
+	if (hgc->capflags & CAP_SETUMASK2) {
 		forwardumask(hgc);
+	}
 
 	return hgc;
 }
@@ -555,16 +586,18 b' const char **hgc_validate(hgclient_t *hg'
                           size_t argsize)
 {
 	assert(hgc);
-	if (!(hgc->capflags & CAP_VALIDATE))
+	if (!(hgc->capflags & CAP_VALIDATE)) {
 		return NULL;
+	}
 
 	packcmdargs(&hgc->ctx, args, argsize);
 	writeblockrequest(hgc, "validate");
 	handleresponse(hgc);
 
 	/* the server returns '\0' if it can handle our request */
-	if (hgc->ctx.datasize <= 1)
+	if (hgc->ctx.datasize <= 1) {
 		return NULL;
+	}
 
 	/* make sure the buffer is '\0' terminated */
 	enlargecontext(&hgc->ctx, hgc->ctx.datasize + 1);
@@ -599,8 +632,9 b' int hgc_runcommand(hgclient_t *hgc, cons'
 void hgc_attachio(hgclient_t *hgc)
 {
 	assert(hgc);
-	if (!(hgc->capflags & CAP_ATTACHIO))
+	if (!(hgc->capflags & CAP_ATTACHIO)) {
 		return;
+	}
 	attachio(hgc);
 }
 
@@ -613,8 +647,9 b' void hgc_attachio(hgclient_t *hgc)'
 void hgc_setenv(hgclient_t *hgc, const char *const envp[])
 {
 	assert(hgc && envp);
-	if (!(hgc->capflags & CAP_SETENV))
+	if (!(hgc->capflags & CAP_SETENV)) {
 		return;
+	}
 	packcmdargs(&hgc->ctx, envp, /*argsize*/ -1);
 	writeblockrequest(hgc, "setenv");
 }

contrib/chg/procutil.c

@@ -25,8 +25,9 b' static pid_t peerpid = 0;'
 static void forwardsignal(int sig)
 {
 	assert(peerpid > 0);
-	if (kill(peerpid, sig) < 0)
+	if (kill(peerpid, sig) < 0) {
 		abortmsgerrno("cannot kill %d", peerpid);
+	}
 	debugmsg("forward signal %d", sig);
 }
 
@@ -34,8 +35,9 b' static void forwardsignaltogroup(int sig'
 {
 	/* prefer kill(-pgid, sig), fallback to pid if pgid is invalid */
 	pid_t killpid = peerpgid > 1 ? -peerpgid : peerpid;
-	if (kill(killpid, sig) < 0)
+	if (kill(killpid, sig) < 0) {
 		abortmsgerrno("cannot kill %d", killpid);
+	}
 	debugmsg("forward signal %d to %d", sig, killpid);
 }
 
@@ -43,28 +45,36 b' static void handlestopsignal(int sig)'
 {
 	sigset_t unblockset, oldset;
 	struct sigaction sa, oldsa;
-	if (sigemptyset(&unblockset) < 0)
+	if (sigemptyset(&unblockset) < 0) {
 		goto error;
-	if (sigaddset(&unblockset, sig) < 0)
+	}
+	if (sigaddset(&unblockset, sig) < 0) {
 		goto error;
+	}
 	memset(&sa, 0, sizeof(sa));
 	sa.sa_handler = SIG_DFL;
 	sa.sa_flags = SA_RESTART;
-	if (sigemptyset(&sa.sa_mask) < 0)
+	if (sigemptyset(&sa.sa_mask) < 0) {
 		goto error;
+	}
 
 	forwardsignal(sig);
-	if (raise(sig) < 0) /* resend to self */
+	if (raise(sig) < 0) { /* resend to self */
 		goto error;
-	if (sigaction(sig, &sa, &oldsa) < 0)
+	}
+	if (sigaction(sig, &sa, &oldsa) < 0) {
 		goto error;
-	if (sigprocmask(SIG_UNBLOCK, &unblockset, &oldset) < 0)
+	}
+	if (sigprocmask(SIG_UNBLOCK, &unblockset, &oldset) < 0) {
 		goto error;
+	}
 	/* resent signal will be handled before sigprocmask() returns */
-	if (sigprocmask(SIG_SETMASK, &oldset, NULL) < 0)
+	if (sigprocmask(SIG_SETMASK, &oldset, NULL) < 0) {
 		goto error;
-	if (sigaction(sig, &oldsa, NULL) < 0)
+	}
+	if (sigaction(sig, &oldsa, NULL) < 0) {
 		goto error;
+	}
 	return;
 
 error:
@@ -73,19 +83,22 b' error:'
 
 static void handlechildsignal(int sig UNUSED_)
 {
-	if (peerpid == 0 || pagerpid == 0)
+	if (peerpid == 0 || pagerpid == 0) {
 		return;
+	}
 	/* if pager exits, notify the server with SIGPIPE immediately.
 	 * otherwise the server won't get SIGPIPE if it does not write
 	 * anything. (issue5278) */
-	if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid)
+	if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid) {
 		kill(peerpid, SIGPIPE);
+	}
 }
 
 void setupsignalhandler(pid_t pid, pid_t pgid)
 {
-	if (pid <= 0)
+	if (pid <= 0) {
 		return;
+	}
 	peerpid = pid;
 	peerpgid = (pgid <= 1 ? 0 : pgid);
 
@@ -98,42 +111,52 b' void setupsignalhandler(pid_t pid, pid_t'
 	 * - SIGINT: usually generated by the terminal */
 	sa.sa_handler = forwardsignaltogroup;
 	sa.sa_flags = SA_RESTART;
-	if (sigemptyset(&sa.sa_mask) < 0)
+	if (sigemptyset(&sa.sa_mask) < 0) {
+		goto error;
+	}
+	if (sigaction(SIGHUP, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGHUP, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGINT, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGINT, &sa, NULL) < 0)
-		goto error;
+	}
 
 	/* terminate frontend by double SIGTERM in case of server freeze */
 	sa.sa_handler = forwardsignal;
 	sa.sa_flags |= SA_RESETHAND;
-	if (sigaction(SIGTERM, &sa, NULL) < 0)
+	if (sigaction(SIGTERM, &sa, NULL) < 0) {
 		goto error;
+	}
 
 	/* notify the worker about window resize events */
 	sa.sa_flags = SA_RESTART;
-	if (sigaction(SIGWINCH, &sa, NULL) < 0)
+	if (sigaction(SIGWINCH, &sa, NULL) < 0) {
 		goto error;
+	}
 	/* forward user-defined signals */
-	if (sigaction(SIGUSR1, &sa, NULL) < 0)
+	if (sigaction(SIGUSR1, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGUSR2, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGUSR2, &sa, NULL) < 0) {
 		goto error;
+	}
 	/* propagate job control requests to worker */
 	sa.sa_handler = forwardsignal;
 	sa.sa_flags = SA_RESTART;
-	if (sigaction(SIGCONT, &sa, NULL) < 0)
+	if (sigaction(SIGCONT, &sa, NULL) < 0) {
 		goto error;
+	}
 	sa.sa_handler = handlestopsignal;
 	sa.sa_flags = SA_RESTART;
-	if (sigaction(SIGTSTP, &sa, NULL) < 0)
+	if (sigaction(SIGTSTP, &sa, NULL) < 0) {
 		goto error;
+	}
 	/* get notified when pager exits */
 	sa.sa_handler = handlechildsignal;
 	sa.sa_flags = SA_RESTART;
-	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+	if (sigaction(SIGCHLD, &sa, NULL) < 0) {
 		goto error;
+	}
 
 	return;
 
@@ -147,26 +170,34 b' void restoresignalhandler(void)'
 	memset(&sa, 0, sizeof(sa));
 	sa.sa_handler = SIG_DFL;
 	sa.sa_flags = SA_RESTART;
-	if (sigemptyset(&sa.sa_mask) < 0)
+	if (sigemptyset(&sa.sa_mask) < 0) {
 		goto error;
+	}
 
-	if (sigaction(SIGHUP, &sa, NULL) < 0)
+	if (sigaction(SIGHUP, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGTERM, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGTERM, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGWINCH, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGWINCH, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGCONT, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGCONT, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGTSTP, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGTSTP, &sa, NULL) < 0) {
 		goto error;
-	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+	}
+	if (sigaction(SIGCHLD, &sa, NULL) < 0) {
 		goto error;
+	}
 
 	/* ignore Ctrl+C while shutting down to make pager exits cleanly */
 	sa.sa_handler = SIG_IGN;
-	if (sigaction(SIGINT, &sa, NULL) < 0)
+	if (sigaction(SIGINT, &sa, NULL) < 0) {
 		goto error;
+	}
 
 	peerpid = 0;
 	return;
@@ -180,22 +211,27 b' error:'
 pid_t setuppager(const char *pagercmd, const char *envp[])
 {
 	assert(pagerpid == 0);
-	if (!pagercmd)
+	if (!pagercmd) {
 		return 0;
+	}
 
 	int pipefds[2];
-	if (pipe(pipefds) < 0)
+	if (pipe(pipefds) < 0) {
 		return 0;
+	}
 	pid_t pid = fork();
-	if (pid < 0)
+	if (pid < 0) {
 		goto error;
+	}
 	if (pid > 0) {
 		close(pipefds[0]);
-		if (dup2(pipefds[1], fileno(stdout)) < 0)
+		if (dup2(pipefds[1], fileno(stdout)) < 0) {
 			goto error;
+		}
 		if (isatty(fileno(stderr))) {
-			if (dup2(pipefds[1], fileno(stderr)) < 0)
+			if (dup2(pipefds[1], fileno(stderr)) < 0) {
 				goto error;
+			}
 		}
 		close(pipefds[1]);
 		pagerpid = pid;
@@ -222,16 +258,18 b' error:'
 
 void waitpager(void)
 {
-	if (pagerpid == 0)
+	if (pagerpid == 0) {
 		return;
+	}
 
 	/* close output streams to notify the pager its input ends */
 	fclose(stdout);
 	fclose(stderr);
 	while (1) {
 		pid_t ret = waitpid(pagerpid, NULL, 0);
-		if (ret == -1 && errno == EINTR)
+		if (ret == -1 && errno == EINTR) {
 			continue;
+		}
 		break;
 	}
 }

contrib/chg/util.c

@@ -25,8 +25,9 b' static int colorenabled = 0;'
 
 static inline void fsetcolor(FILE *fp, const char *code)
 {
-	if (!colorenabled)
+	if (!colorenabled) {
 		return;
+	}
 	fprintf(fp, "\033[%sm", code);
 }
 
@@ -35,8 +36,9 b' static void vabortmsgerrno(int no, const'
 	fsetcolor(stderr, "1;31");
 	fputs("chg: abort: ", stderr);
 	vfprintf(stderr, fmt, args);
-	if (no != 0)
+	if (no != 0) {
 		fprintf(stderr, " (errno = %d, %s)", no, strerror(no));
+	}
 	fsetcolor(stderr, "");
 	fputc('\n', stderr);
 	exit(255);
@@ -82,8 +84,9 b' void enabledebugmsg(void)'
 
 void debugmsg(const char *fmt, ...)
 {
-	if (!debugmsgenabled)
+	if (!debugmsgenabled) {
 		return;
+	}
 
 	va_list args;
 	va_start(args, fmt);
@@ -98,32 +101,37 b' void debugmsg(const char *fmt, ...)'
 void fchdirx(int dirfd)
 {
 	int r = fchdir(dirfd);
-	if (r == -1)
+	if (r == -1) {
 		abortmsgerrno("failed to fchdir");
+	}
 }
 
 void fsetcloexec(int fd)
 {
 	int flags = fcntl(fd, F_GETFD);
-	if (flags < 0)
+	if (flags < 0) {
 		abortmsgerrno("cannot get flags of fd %d", fd);
-	if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0)
+	}
+	if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) {
 		abortmsgerrno("cannot set flags of fd %d", fd);
+	}
 }
 
 void *mallocx(size_t size)
 {
 	void *result = malloc(size);
-	if (!result)
+	if (!result) {
 		abortmsg("failed to malloc");
+	}
 	return result;
 }
 
 void *reallocx(void *ptr, size_t size)
 {
 	void *result = realloc(ptr, size);
-	if (!result)
+	if (!result) {
 		abortmsg("failed to realloc");
+	}
 	return result;
 }
 
@@ -144,30 +152,37 b' int runshellcmd(const char *cmd, const c'
 	memset(&newsa, 0, sizeof(newsa));
 	newsa.sa_handler = SIG_IGN;
 	newsa.sa_flags = 0;
-	if (sigemptyset(&newsa.sa_mask) < 0)
+	if (sigemptyset(&newsa.sa_mask) < 0) {
 		goto done;
-	if (sigaction(SIGINT, &newsa, &oldsaint) < 0)
+	}
+	if (sigaction(SIGINT, &newsa, &oldsaint) < 0) {
 		goto done;
+	}
 	doneflags |= F_SIGINT;
-	if (sigaction(SIGQUIT, &newsa, &oldsaquit) < 0)
+	if (sigaction(SIGQUIT, &newsa, &oldsaquit) < 0) {
 		goto done;
+	}
 	doneflags |= F_SIGQUIT;
 
-	if (sigaddset(&newsa.sa_mask, SIGCHLD) < 0)
+	if (sigaddset(&newsa.sa_mask, SIGCHLD) < 0) {
 		goto done;
-	if (sigprocmask(SIG_BLOCK, &newsa.sa_mask, &oldmask) < 0)
+	}
+	if (sigprocmask(SIG_BLOCK, &newsa.sa_mask, &oldmask) < 0) {
 		goto done;
+	}
 	doneflags |= F_SIGMASK;
 
 	pid_t pid = fork();
-	if (pid < 0)
+	if (pid < 0) {
 		goto done;
+	}
 	if (pid == 0) {
 		sigaction(SIGINT, &oldsaint, NULL);
 		sigaction(SIGQUIT, &oldsaquit, NULL);
 		sigprocmask(SIG_SETMASK, &oldmask, NULL);
-		if (cwd && chdir(cwd) < 0)
+		if (cwd && chdir(cwd) < 0) {
 			_exit(127);
+		}
 		const char *argv[] = {"sh", "-c", cmd, NULL};
 		if (envp) {
 			execve("/bin/sh", (char **)argv, (char **)envp);
@@ -176,25 +191,32 b' int runshellcmd(const char *cmd, const c'
 		}
 		_exit(127);
 	} else {
-		if (waitpid(pid, &status, 0) < 0)
+		if (waitpid(pid, &status, 0) < 0) {
 			goto done;
+		}
 		doneflags |= F_WAITPID;
 	}
 
 done:
-	if (doneflags & F_SIGINT)
+	if (doneflags & F_SIGINT) {
 		sigaction(SIGINT, &oldsaint, NULL);
-	if (doneflags & F_SIGQUIT)
+	}
+	if (doneflags & F_SIGQUIT) {
 		sigaction(SIGQUIT, &oldsaquit, NULL);
-	if (doneflags & F_SIGMASK)
+	}
+	if (doneflags & F_SIGMASK) {
 		sigprocmask(SIG_SETMASK, &oldmask, NULL);
+	}
 
 	/* no way to report other errors, use 127 (= shell termination) */
-	if (!(doneflags & F_WAITPID))
+	if (!(doneflags & F_WAITPID)) {
 		return 127;
-	if (WIFEXITED(status))
+	}
+	if (WIFEXITED(status)) {
 		return WEXITSTATUS(status);
-	if (WIFSIGNALED(status))
+	}
+	if (WIFSIGNALED(status)) {
 		return -WTERMSIG(status);
+	}
 	return 127;
 }

contrib/clang-format-ignorelist

@@ -62,6 +62,11 b' contrib/python-zstandard/zstd/compress/z'
 contrib/python-zstandard/zstd/compress/zstd_opt.c
 contrib/python-zstandard/zstd/compress/zstd_opt.h
 contrib/python-zstandard/zstd/decompress/huf_decompress.c
+contrib/python-zstandard/zstd/decompress/zstd_ddict.c
+contrib/python-zstandard/zstd/decompress/zstd_ddict.h
+contrib/python-zstandard/zstd/decompress/zstd_decompress_block.c
+contrib/python-zstandard/zstd/decompress/zstd_decompress_block.h
+contrib/python-zstandard/zstd/decompress/zstd_decompress_internal.h
 contrib/python-zstandard/zstd/decompress/zstd_decompress.c
 contrib/python-zstandard/zstd/deprecated/zbuff_common.c
 contrib/python-zstandard/zstd/deprecated/zbuff_compress.c

contrib/debugshell.py

@@ -7,6 +7,7 b' import mercurial'
 import sys
 from mercurial import (
     demandimport,
+    pycompat,
     registrar,
 )
 
@@ -32,28 +33,30 b' def ipdb(ui, repo, msg, **opts):'
 
     IPython.embed()
 
-@command('debugshell|dbsh', [])
+@command(b'debugshell|dbsh', [])
 def debugshell(ui, repo, **opts):
-    bannermsg = "loaded repo : %s\n" \
-                "using source: %s" % (repo.root,
-                                      mercurial.__path__[0])
+    bannermsg = ("loaded repo : %s\n"
+                 "using source: %s" % (pycompat.sysstr(repo.root),
+                                       mercurial.__path__[0]))
 
     pdbmap = {
         'pdb'  : 'code',
         'ipdb' : 'IPython'
     }
 
-    debugger = ui.config("ui", "debugger")
+    debugger = ui.config(b"ui", b"debugger")
     if not debugger:
         debugger = 'pdb'
+    else:
+        debugger = pycompat.sysstr(debugger)
 
     # if IPython doesn't exist, fallback to code.interact
     try:
         with demandimport.deactivated():
             __import__(pdbmap[debugger])
     except ImportError:
-        ui.warn(("%s debugger specified but %s module was not found\n")
+        ui.warn((b"%s debugger specified but %s module was not found\n")
                 % (debugger, pdbmap[debugger]))
-        debugger = 'pdb'
+        debugger = b'pdb'
 
     getattr(sys.modules[__name__], debugger)(ui, repo, bannermsg, **opts)

contrib/fuzz/manifest.cc

@@ -20,11 +20,19 b' try:'
   lm = lazymanifest(mdata)
   # iterate the whole thing, which causes the code to fully parse
   # every line in the manifest
-  list(lm.iterentries())
+  for e, _, _ in lm.iterentries():
+      # also exercise __getitem__ et al
+      lm[e]
+      e in lm
+      (e + 'nope') in lm
   lm[b'xyzzy'] = (b'\0' * 20, 'x')
   # do an insert, text should change
   assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
+  cloned = lm.filtercopy(lambda x: x != 'xyzzy')
+  assert cloned.text() == mdata, 'cloned text should equal mdata'
+  cloned.diff(lm)
   del lm[b'xyzzy']
+  cloned.diff(lm)
   # should be back to the same
   assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
 except Exception as e:
@@ -39,6 +47,11 b' except Exception as e:'
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
+	// Don't allow fuzzer inputs larger than 100k, since we'll just bog
+	// down and not accomplish much.
+	if (Size > 100000) {
+		return 0;
+	}
 	PyObject *mtext =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();

contrib/fuzz/revlog.cc

@@ -19,6 +19,11 b' from parsers import parse_index2'
 for inline in (True, False):
     try:
         index, cache = parse_index2(data, inline)
+        index.slicechunktodensity(list(range(len(index))), 0.5, 262144)
+        for rev in range(len(index)):
+            node = index[rev][7]
+            partial = index.shortest(node)
+            index.partialmatch(node[:partial])
     except Exception as e:
         pass
         # uncomment this print if you're editing this Python code
@@ -31,6 +36,11 b' for inline in (True, False):'
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
+	// Don't allow fuzzer inputs larger than 60k, since we'll just bog
+	// down and not accomplish much.
+	if (Size > 60000) {
+		return 0;
+	}
 	PyObject *text =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();

contrib/hg-test-mode.el

@@ -53,4 +53,45 b''
   (setq mode-name "hg-test")
   (run-hooks 'hg-test-mode-hook))
 
+(with-eval-after-load "compile"
+  ;; Link to Python sources in tracebacks in .t failures.
+  (add-to-list 'compilation-error-regexp-alist-alist
+               '(hg-test-output-python-tb
+                 "^\\+ +File ['\"]\\([^'\"]+\\)['\"], line \\([0-9]+\\)," 1 2))
+  (add-to-list 'compilation-error-regexp-alist 'hg-test-output-python-tb)
+  ;; Link to source files in test-check-code.t violations.
+  (add-to-list 'compilation-error-regexp-alist-alist
+               '(hg-test-check-code-output
+                 "\\+  \\([^:\n]+\\):\\([0-9]+\\):$" 1 2))
+  (add-to-list 'compilation-error-regexp-alist 'hg-test-check-code-output))
+
+(defun hg-test-mode--test-one-error-line-regexp (test)
+  (erase-buffer)
+  (setq compilation-locs (make-hash-table))
+  (insert (car test))
+  (compilation-parse-errors (point-min) (point-max))
+  (let ((msg (get-text-property 1 'compilation-message)))
+    (should msg)
+    (let ((loc (compilation--message->loc msg))
+          (line (nth 1 test))
+          (file (nth 2 test)))
+      (should (equal (compilation--loc->line loc) line))
+      (should (equal (caar (compilation--loc->file-struct loc)) file)))
+      msg))
+
+(require 'ert)
+(ert-deftest hg-test-mode--compilation-mode-support ()
+  "Test hg-specific compilation-mode regular expressions"
+  (require 'compile)
+  (with-temp-buffer
+    (font-lock-mode -1)
+    (mapc 'hg-test-mode--test-one-error-line-regexp
+          '(
+            ("+  contrib/debugshell.py:37:" 37 "contrib/debugshell.py")
+            ("+    File \"/tmp/hg/mercurial/commands.py\", line 3115, in help_"
+             3115 "/tmp/hg/mercurial/commands.py")
+            ("+    File \"mercurial/dispatch.py\", line 225, in dispatch"
+             225 "mercurial/dispatch.py")))))
+
+
 (provide 'hg-test-mode)

contrib/packaging/hg-docker

@@ -76,7 +76,7 b' def build_docker_image(dockerfile: pathl'
     p.communicate(input=dockerfile)
     if p.returncode:
         raise subprocess.CalledProcessException(
-                p.returncode, 'failed to build docker image: %s %s' \
+                p.returncode, 'failed to build docker image: %s %s'
                 % (p.stdout, p.stderr))
 
 def command_build(args):

contrib/packaging/inno/mercurial.iss

@@ -5,7 +5,7 b''
 #define FileHandle
 #define FileLine
 #define VERSION = "unknown"
-#if FileHandle = FileOpen(SourcePath + "\..\..\mercurial\__version__.py")
+#if FileHandle = FileOpen(SourcePath + "\..\..\..\mercurial\__version__.py")
   #expr FileLine = FileRead(FileHandle)
   #expr FileLine = FileRead(FileHandle)
   #define VERSION = Copy(FileLine, Pos('"', FileLine)+1, Len(FileLine)-Pos('"', FileLine)-1)
@@ -43,7 +43,7 b' AppUpdatesURL=https://mercurial-scm.org/'
 AppID={{4B95A5F1-EF59-4B08-BED8-C891C46121B3}
 AppContact=[e-mail]
 DefaultDirName={pf}\Mercurial
-SourceDir=..\..
+SourceDir=..\..\..
 VersionInfoDescription=Mercurial distributed SCM (version {#VERSION})
 VersionInfoCopyright=Copyright 2005-2019 Matt Mackall and others
 VersionInfoCompany=Matt Mackall and others
@@ -53,6 +53,7 b' SetupIconFile=contrib\\win32\\mercurial.ic'
 AllowNoIcons=true
 DefaultGroupName=Mercurial
 PrivilegesRequired=none
+ChangesEnvironment=true
 
 [Files]
 Source: contrib\mercurial.el; DestDir: {app}/Contrib
@@ -70,17 +71,12 b' Source: contrib\\hgweb.wsgi; DestDir: {ap'
 Source: contrib\win32\ReadMe.html; DestDir: {app}; Flags: isreadme
 Source: contrib\win32\postinstall.txt; DestDir: {app}; DestName: ReleaseNotes.txt
 Source: dist\hg.exe; DestDir: {app}; AfterInstall: Touch('{app}\hg.exe.local')
-#if ARCH == "x64"
 Source: dist\lib\*.dll; Destdir: {app}\lib
 Source: dist\lib\*.pyd; Destdir: {app}\lib
-#else
-Source: dist\w9xpopen.exe; DestDir: {app}
-#endif
 Source: dist\python*.dll; Destdir: {app}; Flags: skipifsourcedoesntexist
 Source: dist\msvc*.dll; DestDir: {app}; Flags: skipifsourcedoesntexist
 Source: dist\Microsoft.VC*.CRT.manifest; DestDir: {app}; Flags: skipifsourcedoesntexist
 Source: dist\lib\library.zip; DestDir: {app}\lib
-Source: dist\add_path.exe; DestDir: {app}
 Source: doc\*.html; DestDir: {app}\Docs
 Source: doc\style.css; DestDir: {app}\Docs
 Source: mercurial\help\*.txt; DestDir: {app}\help
@@ -107,14 +103,22 b' Name: {group}\\Mercurial Configuration Fi'
 Name: {group}\Mercurial Ignore Files; Filename: {app}\Docs\hgignore.5.html
 Name: {group}\Mercurial Web Site; Filename: {app}\Mercurial.url
 
-[Run]
-Filename: "{app}\add_path.exe"; Parameters: "{app}"; Flags: postinstall; Description: "Add the installation path to the search path"
-
-[UninstallRun]
-Filename: "{app}\add_path.exe"; Parameters: "/del {app}"
+[Tasks]
+Name: modifypath; Description: Add the installation path to the search path; Flags: unchecked
 
 [Code]
 procedure Touch(fn: String);
 begin
   SaveStringToFile(ExpandConstant(fn), '', False);
 end;
+
+const
+    ModPathName = 'modifypath';
+    ModPathType = 'user';
+
+function ModPathDir(): TArrayOfString;
+begin
+    setArrayLength(Result, 1)
+    Result[0] := ExpandConstant('{app}');
+end;
+#include "modpath.iss"

contrib/packaging/inno/readme.rst

@@ -1,130 +1,61 b''
-The standalone Windows installer for Mercurial is built in a somewhat
-jury-rigged fashion.
+Requirements
+============
 
-It has the following prerequisites. Ensure to take the packages
-matching the mercurial version you want to build (32-bit or 64-bit).
+Building the Inno installer requires a Windows machine.
 
-  Python 2.6 for Windows
-      http://www.python.org/download/releases/
+The following system dependencies must be installed:
 
-  A compiler:
-    either MinGW
-      http://www.mingw.org/
-    or Microsoft Visual C++ 2008 SP1 Express Edition
-      http://www.microsoft.com/express/Downloads/Download-2008.aspx
-
-  Python for Windows Extensions
-      http://sourceforge.net/projects/pywin32/
-
-  mfc71.dll (just download, don't install; not needed for Python 2.6)
-      http://starship.python.net/crew/mhammond/win32/
-
-  Visual C++ 2008 redistributable package (needed for >= Python 2.6 or if you compile with MSVC)
-    for 32-bit:
-      http://www.microsoft.com/downloads/details.aspx?FamilyID=9b2da534-3e03-4391-8a4d-074b9f2bc1bf
-    for 64-bit:
-      http://www.microsoft.com/downloads/details.aspx?familyid=bd2a6171-e2d6-4230-b809-9a8d7548c1b6
+* Python 2.7 (download from https://www.python.org/downloads/)
+* Microsoft Visual C++ Compiler for Python 2.7
+  (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
+* Inno Setup (http://jrsoftware.org/isdl.php) version 5.4 or newer.
+  Be sure to install the optional Inno Setup Preprocessor feature,
+  which is required.
+* Python 3.5+ (to run the ``build.py`` script)
 
-  The py2exe distutils extension
-      http://sourceforge.net/projects/py2exe/
-
-  GnuWin32 gettext utility (if you want to build translations)
-      http://gnuwin32.sourceforge.net/packages/gettext.htm
-
-  Inno Setup
-      http://www.jrsoftware.org/isdl.php#qsp
-
-      Get and install ispack-5.3.10.exe or later (includes Inno Setup Processor),
-      which is necessary to package Mercurial.
-
-  ISTool - optional
-      http://www.istool.org/default.aspx/
+Building
+========
 
-  add_path (you need only add_path.exe in the zip file)
-      http://www.barisione.org/apps.html#add_path
-
-  Docutils
-      http://docutils.sourceforge.net/
+The ``build.py`` script automates the process of producing an
+Inno installer. It manages fetching and configuring the
+non-system dependencies (such as py2exe, gettext, and various
+Python packages).
 
-  CA Certs file
-      http://curl.haxx.se/ca/cacert.pem
-
-And, of course, Mercurial itself.
-
-Once you have all this installed and built, clone a copy of the
-Mercurial repository you want to package, and name the repo
-C:\hg\hg-release.
-
-In a shell, build a standalone copy of the hg.exe program.
+The script requires an activated ``Visual C++ 2008`` command prompt.
+A shortcut to such a prompt was installed with ``Microsoft Visual C++
+Compiler for Python 2.7``. From your Start Menu, look for
+``Microsoft Visual C++ Compiler Package for Python 2.7`` then launch
+either ``Visual C++ 2008 32-bit Command Prompt`` or
+``Visual C++ 2008 64-bit Command Prompt``.
 
-Building instructions for MinGW:
-  python setup.py build -c mingw32
-  python setup.py py2exe -b 2
-Note: the previously suggested combined command of "python setup.py build -c
-mingw32 py2exe -b 2" doesn't work correctly anymore as it doesn't include the
-extensions in the mercurial subdirectory.
-If you want to create a file named setup.cfg with the contents:
-[build]
-compiler=mingw32
-you can skip the first build step.
+From the prompt, change to the Mercurial source directory. e.g.
+``cd c:\src\hg``.
+
+Next, invoke ``build.py`` to produce an Inno installer. You will
+need to supply the path to the Python interpreter to use.:
 
-Building instructions with MSVC 2008 Express Edition:
-  for 32-bit:
-    "C:\Program Files\Microsoft Visual Studio 9.0\VC\vcvarsall.bat" x86
-    python setup.py py2exe -b 2
-  for 64-bit:
-    "C:\Program Files\Microsoft Visual Studio 9.0\VC\vcvarsall.bat" x86_amd64
-    python setup.py py2exe -b 3
+   $ python3.exe contrib\packaging\inno\build.py \
+       --python c:\python27\python.exe
 
-Copy add_path.exe and cacert.pem files into the dist directory that just got created.
+.. note::
 
-If you are using Python 2.6 or later, or if you are using MSVC 2008 to compile
-mercurial, you must include the C runtime libraries in the installer. To do so,
-install the Visual C++ 2008 redistributable package. Then in your windows\winsxs
-folder, locate the folder containing the dlls version 9.0.21022.8.
-For x86, it should be named like x86_Microsoft.VC90.CRT_(...)_9.0.21022.8(...).
-For x64, it should be named like amd64_Microsoft.VC90.CRT_(...)_9.0.21022.8(...).
-Copy the files named msvcm90.dll, msvcp90.dll and msvcr90.dll into the dist
-directory.
-Then in the windows\winsxs\manifests folder, locate the corresponding manifest
-file (x86_Microsoft.VC90.CRT_(...)_9.0.21022.8(...).manifest for x86,
-amd64_Microsoft.VC90.CRT_(...)_9.0.21022.8(...).manifest for x64), copy it in the
-dist directory and rename it to Microsoft.VC90.CRT.manifest.
+   The script validates that the Visual C++ environment is
+   active and that the architecture of the specified Python
+   interpreter matches the Visual C++ environment and errors
+   if not.
 
-Before building the installer, you have to build Mercurial HTML documentation
-(or fix mercurial.iss to not reference the doc directory):
-
-  cd doc
-  mingw32-make html
-  cd ..
+If everything runs as intended, dependencies will be fetched and
+configured into the ``build`` sub-directory, Mercurial will be built,
+and an installer placed in the ``dist`` sub-directory. The final
+line of output should print the name of the generated installer.
 
-If you use ISTool, you open the C:\hg\hg-release\contrib\win32\mercurial.iss
-file and type Ctrl-F9 to compile the installer file.
-
-Otherwise you run the Inno Setup compiler.  Assuming it's in the path
-you should execute:
-
-  iscc contrib\win32\mercurial.iss /dVERSION=foo
+Additional options may be configured. Run ``build.py --help`` to
+see a list of program flags.
 
-Where 'foo' is the version number you would like to see in the
-'Add/Remove Applications' tool.  The installer will be placed into
-a directory named Output/ at the root of your repository.
-If the /dVERSION=foo parameter is not given in the command line, the
-installer will retrieve the version information from the __version__.py file.
-
-If you want to build an installer for a 64-bit mercurial, add /dARCH=x64 to
-your command line:
-  iscc contrib\win32\mercurial.iss /dARCH=x64
+MinGW
+=====
 
-To automate the steps above you may want to create a batchfile based on the
-following (MinGW build chain):
-
-  echo [build] > setup.cfg
-  echo compiler=mingw32 >> setup.cfg
-  python setup.py py2exe -b 2
-  cd doc
-  mingw32-make html
-  cd ..
-  iscc contrib\win32\mercurial.iss /dVERSION=snapshot
-
-and run it from the root of the hg repository (c:\hg\hg-release).
+It is theoretically possible to generate an installer that uses
+MinGW. This isn't well tested and ``build.py`` and may properly
+support it. See old versions of this file in version control for
+potentially useful hints as to how to achieve this.

contrib/packaging/wix/dist.wxs

@@ -9,28 +9,6 b''
       <Component Id="distOutput" Guid="$(var.dist.guid)" Win64='$(var.IsX64)'>
         <File Name="python27.dll" KeyPath="yes" />
       </Component>
-      <Directory Id="libdir" Name="lib" FileSource="$(var.SourceDir)/lib">
-        <Component Id="libOutput" Guid="$(var.lib.guid)" Win64='$(var.IsX64)'>
-          <File Name="library.zip" KeyPath="yes" />
-          <File Name="mercurial.cext.base85.pyd" />
-          <File Name="mercurial.cext.bdiff.pyd" />
-          <File Name="mercurial.cext.mpatch.pyd" />
-          <File Name="mercurial.cext.osutil.pyd" />
-          <File Name="mercurial.cext.parsers.pyd" />
-          <File Name="mercurial.zstd.pyd" />
-          <File Name="hgext.fsmonitor.pywatchman.bser.pyd" />
-          <File Name="pyexpat.pyd" />
-          <File Name="bz2.pyd" />
-          <File Name="select.pyd" />
-          <File Name="unicodedata.pyd" />
-          <File Name="_ctypes.pyd" />
-          <File Name="_elementtree.pyd" />
-          <File Name="_testcapi.pyd" />
-          <File Name="_hashlib.pyd" />
-          <File Name="_socket.pyd" />
-          <File Name="_ssl.pyd" />
-        </Component>
-      </Directory>
     </DirectoryRef>
   </Fragment>
 

contrib/packaging/wix/mercurial.wxs

@@ -69,7 +69,7 b''
                     KeyPath='yes'/>
           </Component>
           <Component Id='COPYING' Guid='$(var.COPYING.guid)' Win64='$(var.IsX64)'>
-            <File Id='COPYING' Name='COPYING.rtf' Source='contrib\wix\COPYING.rtf'
+            <File Id='COPYING' Name='COPYING.rtf' Source='contrib\packaging\wix\COPYING.rtf'
                   KeyPath='yes'/>
           </Component>
 
@@ -129,6 +129,11 b''
         <MergeRef Id='VCRuntime' />
         <MergeRef Id='VCRuntimePolicy' />
       </Feature>
+      <?ifdef MercurialExtraFeatures?>
+        <?foreach EXTRAFEAT in $(var.MercurialExtraFeatures)?>
+          <FeatureRef Id="$(var.EXTRAFEAT)" />
+        <?endforeach?>
+      <?endif?>
       <Feature Id='Locales' Title='Translations' Description='Translations' Level='1'>
         <ComponentGroupRef Id='localeFolder' />
         <ComponentRef Id='i18nFolder' />
@@ -144,7 +149,7 b''
     <UIRef Id="WixUI_FeatureTree" />
     <UIRef Id="WixUI_ErrorProgressText" />
 
-    <WixVariable Id="WixUILicenseRtf" Value="contrib\wix\COPYING.rtf" />
+    <WixVariable Id="WixUILicenseRtf" Value="contrib\packaging\wix\COPYING.rtf" />
 
     <Icon Id="hgIcon.ico" SourceFile="contrib/win32/mercurial.ico" />
 

contrib/packaging/wix/readme.rst

@@ -1,31 +1,71 b''
-WiX installer source files
+WiX Installer
+=============
+
+The files in this directory are used to produce an MSI installer using
+the WiX Toolset (http://wixtoolset.org/).
+
+The MSI installers require elevated (admin) privileges due to the
+installation of MSVC CRT libraries into the Windows system store. See
+the Inno Setup installers in the ``inno`` sibling directory for installers
+that do not have this requirement.
+
+Requirements
+============
+
+Building the WiX installers requires a Windows machine. The following
+dependencies must be installed:
+
+* Python 2.7 (download from https://www.python.org/downloads/)
+* Microsoft Visual C++ Compiler for Python 2.7
+  (https://www.microsoft.com/en-us/download/details.aspx?id=44266)
+* Python 3.5+ (to run the ``build.py`` script)
+
+Building
+========
+
+The ``build.py`` script automates the process of producing an MSI
+installer. It manages fetching and configuring non-system dependencies
+(such as py2exe, gettext, and various Python packages).
+
+The script requires an activated ``Visual C++ 2008`` command prompt.
+A shortcut to such a prompt was installed with ``Microsoft Visual
+C++ Compiler for Python 2.7``. From your Start Menu, look for
+``Microsoft Visual C++ Compiler Package for Python 2.7`` then
+launch either ``Visual C++ 2008 32-bit Command Prompt`` or
+``Visual C++ 2008 64-bit Command Prompt``.
+
+From the prompt, change to the Mercurial source directory. e.g.
+``cd c:\src\hg``.
+
+Next, invoke ``build.py`` to produce an MSI installer. You will need
+to supply the path to the Python interpreter to use.::
+
+   $ python3 contrib\packaging\wix\build.py \
+      --python c:\python27\python.exe
+
+.. note::
+
+   The script validates that the Visual C++ environment is active and
+   that the architecture of the specified Python interpreter matches the
+   Visual C++ environment. An error is raised otherwise.
+
+If everything runs as intended, dependencies will be fetched and
+configured into the ``build`` sub-directory, Mercurial will be built,
+and an installer placed in the ``dist`` sub-directory. The final line
+of output should print the name of the generated installer.
+
+Additional options may be configured. Run ``build.py --help`` to see
+a list of program flags.
+
+Relationship to TortoiseHG
 ==========================
 
-The files in this folder are used by the thg-winbuild [1] package
-building architecture to create a Mercurial MSI installer.   These files
-are versioned within the Mercurial source tree because the WXS files
-must kept up to date with distribution changes within their branch.  In
-other words, the default branch WXS files are expected to diverge from
-the stable branch WXS files.  Storing them within the same repository is
-the only sane way to keep the source tree and the installer in sync.
-
-The MSI installer builder uses only the mercurial.ini file from the
-contrib/win32 folder, the contents of which have been historically used
-to create an InnoSetup based installer.  The rest of the files there are
-ignored.
+TortoiseHG uses the WiX files in this directory.
 
-The MSI packages built by thg-winbuild require elevated (admin)
-privileges to be installed due to the installation of MSVC CRT libraries
-under the C:\WINDOWS\WinSxS folder.  Thus the InnoSetup installers may
-still be useful to some users.
+The code for building TortoiseHG installers lives at
+https://bitbucket.org/tortoisehg/thg-winbuild and is maintained by
+Steve Borho ([e-mail]).
 
-To build your own MSI packages, clone the thg-winbuild [1] repository
-and follow the README.txt [2] instructions closely.  There are fewer
-prerequisites for a WiX [3] installer than an InnoSetup installer, but
-they are more specific.
-
-Direct questions or comments to Steve Borho <[e-mail]>
-
-[1] http://bitbucket.org/tortoisehg/thg-winbuild
-[2] http://bitbucket.org/tortoisehg/thg-winbuild/src/tip/README.txt
-[3] http://wix.sourceforge.net/
+When changing behavior of the WiX installer, be sure to notify
+the TortoiseHG Project of the changes so they have ample time
+provide feedback and react to those changes.

contrib/perf-utils/discovery-helper.sh

@@ -28,9 +28,13 b''
 
 set -euo pipefail
 
+printusage () {
+     echo "usage: `basename $0` REPO NBHEADS DEPTH [left|right]" >&2
+}
+
 if [ $# -lt 3 ]; then
-     echo "usage: `basename $0` REPO NBHEADS DEPTH"
-     exit 64
+    printusage
+    exit 64
 fi
 
 repo="$1"
@@ -42,8 +46,26 b' shift'
 depth="$1"
 shift
 
-leftrepo="${repo}-left"
-rightrepo="${repo}-right"
+doleft=1
+doright=1
+if [ $# -gt 1 ]; then
+    printusage
+    exit 64
+elif [ $# -eq 1 ]; then
+    if [ "$1" == "left" ]; then
+        doleft=1
+        doright=0
+    elif [ "$1" == "right" ]; then
+        doleft=0
+        doright=1
+    else
+        printusage
+        exit 64
+    fi
+fi
+
+leftrepo="${repo}-${nbheads}h-${depth}d-left"
+rightrepo="${repo}-${nbheads}h-${depth}d-right"
 
 left="first(sort(heads(all()), 'desc'), $nbheads)"
 right="last(sort(heads(all()), 'desc'), $nbheads)"
@@ -51,14 +73,35 b' right="last(sort(heads(all()), \'desc\'), '
 leftsubset="ancestors($left, $depth) and only($left, heads(all() - $left))"
 rightsubset="ancestors($right, $depth) and only($right, heads(all() - $right))"
 
-echo '### building left repository:' $left-repo
-echo '# cloning'
-hg clone --noupdate "${repo}" "${leftrepo}"
-echo '# stripping' '"'${leftsubset}'"'
-hg -R "${leftrepo}" --config extensions.strip= strip --rev "$leftsubset" --no-backup
+echo '### creating left/right repositories with missing changesets:'
+if [ $doleft -eq 1 ]; then
+    echo '# left  revset:' '"'${leftsubset}'"'
+fi
+if [ $doright -eq 1 ]; then
+    echo '# right revset:' '"'${rightsubset}'"'
+fi
 
-echo '### building right repository:' $right-repo
-echo '# cloning'
-hg clone --noupdate "${repo}" "${rightrepo}"
-echo '# stripping:' '"'${rightsubset}'"'
-hg -R "${rightrepo}" --config extensions.strip= strip --rev "$rightsubset" --no-backup
+buildone() {
+    side="$1"
+    dest="$2"
+    revset="$3"
+    echo "### building $side repository: $dest"
+    if [ -e "$dest" ]; then
+        echo "destination repo already exists: $dest" >&2
+        exit 1
+    fi
+    echo '# cloning'
+    if ! cp --recursive --reflink=always ${repo} ${dest}; then
+        hg clone --noupdate "${repo}" "${dest}"
+    fi
+    echo '# stripping' '"'${revset}'"'
+    hg -R "${dest}" --config extensions.strip= strip --rev "$revset" --no-backup
+}
+
+if [ $doleft -eq 1 ]; then
+    buildone left "$leftrepo" "$leftsubset"
+fi
+
+if [ $doright -eq 1 ]; then
+    buildone right "$rightrepo" "$rightsubset"
+fi

contrib/perf.py

@@ -1,5 +1,34 b''
 # perf.py - performance test routines
-'''helper extension to measure performance'''
+'''helper extension to measure performance
+
+Configurations
+==============
+
+``perf``
+--------
+
+``all-timing``
+    When set, additional statistics will be reported for each benchmark: best,
+    worst, median average. If not set only the best timing is reported
+    (default: off).
+
+``presleep``
+  number of second to wait before any group of runs (default: 1)
+
+``run-limits``
+  Control the number of runs each benchmark will perform. The option value
+  should be a list of `<time>-<numberofrun>` pairs. After each run the
+  conditions are considered in order with the following logic:
+
+      If benchmark has been running for <time> seconds, and we have performed
+      <numberofrun> iterations, stop the benchmark,
+
+  The default value is: `3.0-100, 10.0-3`
+
+``stub``
+    When set, benchmarks will only be run once, useful for testing
+    (default: off)
+'''
 
 # "historical portability" policy of perf.py:
 #
@@ -65,6 +94,10 b' try:'
 except ImportError:
     pass
 try:
+    from mercurial.utils import repoviewutil # since 5.0
+except ImportError:
+    repoviewutil = None
+try:
     from mercurial import scmutil # since 1.9 (or 8b252e826c68)
 except ImportError:
     pass
@@ -207,6 +240,9 b' try:'
     configitem(b'perf', b'all-timing',
         default=mercurial.configitems.dynamicdefault,
     )
+    configitem(b'perf', b'run-limits',
+        default=mercurial.configitems.dynamicdefault,
+    )
 except (ImportError, AttributeError):
     pass
 
@@ -279,7 +315,34 b' def gettimer(ui, opts=None):'
 
     # experimental config: perf.all-timing
     displayall = ui.configbool(b"perf", b"all-timing", False)
-    return functools.partial(_timer, fm, displayall=displayall), fm
+
+    # experimental config: perf.run-limits
+    limitspec = ui.configlist(b"perf", b"run-limits", [])
+    limits = []
+    for item in limitspec:
+        parts = item.split(b'-', 1)
+        if len(parts) < 2:
+            ui.warn((b'malformatted run limit entry, missing "-": %s\n'
+                     % item))
+            continue
+        try:
+            time_limit = float(pycompat.sysstr(parts[0]))
+        except ValueError as e:
+            ui.warn((b'malformatted run limit entry, %s: %s\n'
+                     % (pycompat.bytestr(e), item)))
+            continue
+        try:
+            run_limit = int(pycompat.sysstr(parts[1]))
+        except ValueError as e:
+            ui.warn((b'malformatted run limit entry, %s: %s\n'
+                     % (pycompat.bytestr(e), item)))
+            continue
+        limits.append((time_limit, run_limit))
+    if not limits:
+        limits = DEFAULTLIMITS
+
+    t = functools.partial(_timer, fm, displayall=displayall, limits=limits)
+    return t, fm
 
 def stub_timer(fm, func, setup=None, title=None):
     if setup is not None:
@@ -297,12 +360,21 b' def timeone():'
     a, b = ostart, ostop
     r.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
 
-def _timer(fm, func, setup=None, title=None, displayall=False):
+
+# list of stop condition (elapsed time, minimal run count)
+DEFAULTLIMITS = (
+    (3.0, 100),
+    (10.0, 3),
+)
+
+def _timer(fm, func, setup=None, title=None, displayall=False,
+           limits=DEFAULTLIMITS):
     gc.collect()
     results = []
     begin = util.timer()
     count = 0
-    while True:
+    keepgoing = True
+    while keepgoing:
         if setup is not None:
             setup()
         with timeone() as item:
@@ -310,10 +382,12 b' def _timer(fm, func, setup=None, title=N'
         count += 1
         results.append(item[0])
         cstop = util.timer()
-        if cstop - begin > 3 and count >= 100:
-            break
-        if cstop - begin > 10 and count >= 3:
-            break
+        # Look for a stop condition.
+        elapsed = cstop - begin
+        for t, mincount in limits:
+            if elapsed >= t and count >= mincount:
+                keepgoing = False
+                break
 
     formatone(fm, results, title=title, result=r,
               displayall=displayall)
@@ -401,7 +475,8 b' def getbranchmapsubsettable():'
     # subsettable is defined in:
     # - branchmap since 2.9 (or 175c6fd8cacc)
     # - repoview since 2.5 (or 59a9f18d4587)
-    for mod in (branchmap, repoview):
+    # - repoviewutil since 5.0
+    for mod in (branchmap, repoview, repoviewutil):
         subsettable = getattr(mod, 'subsettable', None)
         if subsettable:
             return subsettable
@@ -519,7 +594,11 b' def perfaddremove(ui, repo, **opts):'
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         opts[b'dry_run'] = True
-        timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
+        if b'uipathfn' in getargspec(scmutil.addremove).args:
+            uipathfn = scmutil.getuipathfn(repo)
+            timer(lambda: scmutil.addremove(repo, matcher, b"", uipathfn, opts))
+        else:
+            timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
@@ -535,13 +614,15 b' def clearcaches(cl):'
 
 @command(b'perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
+    """benchmark the computation of a changelog heads"""
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
+    def s():
+        clearcaches(cl)
     def d():
         len(cl.headrevs())
-        clearcaches(cl)
-    timer(d)
+    timer(d, setup=s)
     fm.end()
 
 @command(b'perftags', formatteropts+
@@ -911,9 +992,7 b' def perfphasesremote(ui, repo, dest=None'
         raise error.Abort((b'default repository not configured!'),
                           hint=(b"see 'hg help config.paths'"))
     dest = path.pushloc or path.loc
-    branches = (path.branch, opts.get(b'branch') or [])
     ui.status((b'analysing phase of %s\n') % util.hidepassword(dest))
-    revs, checkout = hg.addbranchrevs(repo, repo, branches, opts.get(b'rev'))
     other = hg.peer(repo, opts, dest)
 
     # easier to perform discovery through the operation
@@ -1014,18 +1093,44 b' def perfignore(ui, repo, **opts):'
     fm.end()
 
 @command(b'perfindex', [
-            (b'', b'rev', b'', b'revision to be looked up (default tip)'),
+            (b'', b'rev', [], b'revision to be looked up (default tip)'),
+            (b'', b'no-lookup', None, b'do not revision lookup post creation'),
          ] + formatteropts)
 def perfindex(ui, repo, **opts):
+    """benchmark index creation time followed by a lookup
+
+    The default is to look `tip` up. Depending on the index implementation,
+    the revision looked up can matters. For example, an implementation
+    scanning the index will have a faster lookup time for `--rev tip` than for
+    `--rev 0`. The number of looked up revisions and their order can also
+    matters.
+
+    Example of useful set to test:
+    * tip
+    * 0
+    * -10:
+    * :10
+    * -10: + :10
+    * :10: + -10:
+    * -10000:
+    * -10000: + 0
+
+    It is not currently possible to check for lookup of a missing node. For
+    deeper lookup benchmarking, checkout the `perfnodemap` command."""
     import mercurial.revlog
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
-    if opts[b'rev'] is None:
-        n = repo[b"tip"].node()
+    if opts[b'no_lookup']:
+        if opts['rev']:
+            raise error.Abort('--no-lookup and --rev are mutually exclusive')
+        nodes = []
+    elif not opts[b'rev']:
+        nodes = [repo[b"tip"].node()]
     else:
-        rev = scmutil.revsingle(repo, opts[b'rev'])
-        n = repo[rev].node()
+        revs = scmutil.revrange(repo, opts[b'rev'])
+        cl = repo.changelog
+        nodes = [cl.node(r) for r in revs]
 
     unfi = repo.unfiltered()
     # find the filecache func directly
@@ -1036,7 +1141,67 b' def perfindex(ui, repo, **opts):'
         clearchangelog(unfi)
     def d():
         cl = makecl(unfi)
-        cl.rev(n)
+        for n in nodes:
+            cl.rev(n)
+    timer(d, setup=setup)
+    fm.end()
+
+@command(b'perfnodemap', [
+          (b'', b'rev', [], b'revision to be looked up (default tip)'),
+          (b'', b'clear-caches', True, b'clear revlog cache between calls'),
+    ] + formatteropts)
+def perfnodemap(ui, repo, **opts):
+    """benchmark the time necessary to look up revision from a cold nodemap
+
+    Depending on the implementation, the amount and order of revision we look
+    up can varies. Example of useful set to test:
+    * tip
+    * 0
+    * -10:
+    * :10
+    * -10: + :10
+    * :10: + -10:
+    * -10000:
+    * -10000: + 0
+
+    The command currently focus on valid binary lookup. Benchmarking for
+    hexlookup, prefix lookup and missing lookup would also be valuable.
+    """
+    import mercurial.revlog
+    opts = _byteskwargs(opts)
+    timer, fm = gettimer(ui, opts)
+    mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
+
+    unfi = repo.unfiltered()
+    clearcaches = opts['clear_caches']
+    # find the filecache func directly
+    # This avoid polluting the benchmark with the filecache logic
+    makecl = unfi.__class__.changelog.func
+    if not opts[b'rev']:
+        raise error.Abort('use --rev to specify revisions to look up')
+    revs = scmutil.revrange(repo, opts[b'rev'])
+    cl = repo.changelog
+    nodes = [cl.node(r) for r in revs]
+
+    # use a list to pass reference to a nodemap from one closure to the next
+    nodeget = [None]
+    def setnodeget():
+        # probably not necessary, but for good measure
+        clearchangelog(unfi)
+        nodeget[0] = makecl(unfi).nodemap.get
+
+    def d():
+        get = nodeget[0]
+        for n in nodes:
+            get(n)
+
+    setup = None
+    if clearcaches:
+        def setup():
+            setnodeget()
+    else:
+        setnodeget()
+        d() # prewarm the data structure
     timer(d, setup=setup)
     fm.end()
 
@@ -1056,6 +1221,13 b' def perfstartup(ui, repo, **opts):'
 
 @command(b'perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
+    """benchmark the time necessary to fetch one changeset's parents.
+
+    The fetch is done using the `node identifier`, traversing all object layers
+    from the repository object. The first N revisions will be used for this
+    benchmark. N is controlled by the ``perf.parentscount`` config option
+    (default: 1000).
+    """
     opts = _byteskwargs(opts)
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
@@ -2290,13 +2462,18 b' def perfbranchmap(ui, repo, *filternames'
             view = repo
         else:
             view = repo.filtered(filtername)
+        if util.safehasattr(view._branchcaches, '_per_filter'):
+            filtered = view._branchcaches._per_filter
+        else:
+            # older versions
+            filtered = view._branchcaches
         def d():
             if clear_revbranch:
                 repo.revbranchcache()._clear()
             if full:
                 view._branchcaches.clear()
             else:
-                view._branchcaches.pop(filtername, None)
+                filtered.pop(filtername, None)
             view.branchmap()
         return d
     # add filter in smaller subset to bigger subset
@@ -2323,10 +2500,15 b' def perfbranchmap(ui, repo, *filternames'
         # add unfiltered
         allfilters.append(None)
 
-    branchcacheread = safeattrsetter(branchmap, b'read')
+    if util.safehasattr(branchmap.branchcache, 'fromfile'):
+        branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
+        branchcacheread.set(classmethod(lambda *args: None))
+    else:
+        # older versions
+        branchcacheread = safeattrsetter(branchmap, b'read')
+        branchcacheread.set(lambda *args: None)
     branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
-    branchcacheread.set(lambda repo: None)
-    branchcachewrite.set(lambda bc, repo: None)
+    branchcachewrite.set(lambda *args: None)
     try:
         for name in allfilters:
             printname = name
@@ -2470,9 +2652,15 b' def perfbranchmapload(ui, repo, filter=b'
 
     repo.branchmap() # make sure we have a relevant, up to date branchmap
 
+    try:
+        fromfile = branchmap.branchcache.fromfile
+    except AttributeError:
+        # older versions
+        fromfile = branchmap.read
+
     currentfilter = filter
     # try once without timer, the filter may not be cached
-    while branchmap.read(repo) is None:
+    while fromfile(repo) is None:
         currentfilter = subsettable.get(currentfilter)
         if currentfilter is None:
             raise error.Abort(b'No branchmap cached for %s repo'
@@ -2483,7 +2671,7 b' def perfbranchmapload(ui, repo, filter=b'
         if clearrevlogs:
             clearchangelog(repo)
     def bench():
-        branchmap.read(repo)
+        fromfile(repo)
     timer(bench, setup=setup)
     fm.end()
 

contrib/python-zstandard/MANIFEST.in

@@ -5,6 +5,5 b' graft tests'
 include make_cffi.py
 include setup_zstd.py
 include zstd.c
-include zstd_cffi.py
 include LICENSE
 include NEWS.rst

contrib/python-zstandard/NEWS.rst

@@ -8,8 +8,18 b' 1.0.0 (not yet released)'
 Actions Blocking Release
 ------------------------
 
-* compression and decompression APIs that support ``io.rawIOBase`` interface
+* compression and decompression APIs that support ``io.RawIOBase`` interface
   (#13).
+* ``stream_writer()`` APIs should support ``io.RawIOBase`` interface.
+* Properly handle non-blocking I/O and partial writes for objects implementing
+  ``io.RawIOBase``.
+* Make ``write_return_read=True`` the default for objects implementing
+  ``io.RawIOBase``.
+* Audit for consistent and proper behavior of ``flush()`` and ``close()`` for
+  all objects implementing ``io.RawIOBase``. Is calling ``close()`` on
+  wrapped stream acceptable, should ``__exit__`` always call ``close()``,
+  should ``close()`` imply ``flush()``, etc.
+* Consider making reads across frames configurable behavior.
 * Refactor module names so C and CFFI extensions live under ``zstandard``
   package.
 * Overall API design review.
@@ -43,6 +53,11 b' Actions Blocking Release'
 * Consider a ``chunker()`` API for decompression.
 * Consider stats for ``chunker()`` API, including finding the last consumed
   offset of input data.
+* Consider exposing ``ZSTD_cParam_getBounds()`` and
+  ``ZSTD_dParam_getBounds()`` APIs.
+* Consider controls over resetting compression contexts (session only, parameters,
+  or session and parameters).
+* Actually use the CFFI backend in fuzzing tests.
 
 Other Actions Not Blocking Release
 ---------------------------------------
@@ -51,6 +66,207 b' Other Actions Not Blocking Release'
 * API for ensuring max memory ceiling isn't exceeded.
 * Move off nose for testing.
 
+0.11.0 (released 2019-02-24)
+============================
+
+Backwards Compatibility Nodes
+-----------------------------
+
+* ``ZstdDecompressor.read()`` now allows reading sizes of ``-1`` or ``0``
+  and defaults to ``-1``, per the documented behavior of
+  ``io.RawIOBase.read()``. Previously, we required an argument that was
+  a positive value.
+* The ``readline()``, ``readlines()``, ``__iter__``, and ``__next__`` methods
+  of ``ZstdDecompressionReader()`` now raise ``io.UnsupportedOperation``
+  instead of ``NotImplementedError``.
+* ``ZstdDecompressor.stream_reader()`` now accepts a ``read_across_frames``
+  argument. The default value will likely be changed in a future release
+  and consumers are advised to pass the argument to avoid unwanted change
+  of behavior in the future.
+* ``setup.py`` now always disables the CFFI backend if the installed
+  CFFI package does not meet the minimum version requirements. Before, it was
+  possible for the CFFI backend to be generated and a run-time error to
+  occur.
+* In the CFFI backend, ``CompressionReader`` and ``DecompressionReader``
+  were renamed to ``ZstdCompressionReader`` and ``ZstdDecompressionReader``,
+  respectively so naming is identical to the C extension. This should have
+  no meaningful end-user impact, as instances aren't meant to be
+  constructed directly.
+* ``ZstdDecompressor.stream_writer()`` now accepts a ``write_return_read``
+  argument to control whether ``write()`` returns the number of bytes
+  read from the source / written to the decompressor. It defaults to off,
+  which preserves the existing behavior of returning the number of bytes
+  emitted from the decompressor. The default will change in a future release
+  so behavior aligns with the specified behavior of ``io.RawIOBase``.
+* ``ZstdDecompressionWriter.__exit__`` now calls ``self.close()``. This
+  will result in that stream plus the underlying stream being closed as
+  well. If this behavior is not desirable, do not use instances as
+  context managers.
+* ``ZstdCompressor.stream_writer()`` now accepts a ``write_return_read``
+  argument to control whether ``write()`` returns the number of bytes read
+  from the source / written to the compressor. It defaults to off, which
+  preserves the existing behavior of returning the number of bytes emitted
+  from the compressor. The default will change in a future release so
+  behavior aligns with the specified behavior of ``io.RawIOBase``.
+* ``ZstdCompressionWriter.__exit__`` now calls ``self.close()``. This will
+  result in that stream plus any underlying stream being closed as well. If
+  this behavior is not desirable, do not use instances as context managers.
+* ``ZstdDecompressionWriter`` no longer requires being used as a context
+  manager (#57).
+* ``ZstdCompressionWriter`` no longer requires being used as a context
+  manager (#57).
+* The ``overlap_size_log`` attribute on ``CompressionParameters`` instances
+  has been deprecated and will be removed in a future release. The
+  ``overlap_log`` attribute should be used instead.
+* The ``overlap_size_log`` argument to ``CompressionParameters`` has been
+  deprecated and will be removed in a future release. The ``overlap_log``
+  argument should be used instead.
+* The ``ldm_hash_every_log`` attribute on ``CompressionParameters`` instances
+  has been deprecated and will be removed in a future release. The
+  ``ldm_hash_rate_log`` attribute should be used instead.
+* The ``ldm_hash_every_log`` argument to ``CompressionParameters`` has been
+  deprecated and will be removed in a future release. The ``ldm_hash_rate_log``
+  argument should be used instead.
+* The ``compression_strategy`` argument to ``CompressionParameters`` has been
+  deprecated and will be removed in a future release. The ``strategy``
+  argument should be used instead.
+* The ``SEARCHLENGTH_MIN`` and ``SEARCHLENGTH_MAX`` constants are deprecated
+  and will be removed in a future release. Use ``MINMATCH_MIN`` and
+  ``MINMATCH_MAX`` instead.
+* The ``zstd_cffi`` module has been renamed to ``zstandard.cffi``. As had
+  been documented in the ``README`` file since the ``0.9.0`` release, the
+  module should not be imported directly at its new location. Instead,
+  ``import zstandard`` to cause an appropriate backend module to be loaded
+  automatically.
+
+Bug Fixes
+---------
+
+* CFFI backend could encounter a failure when sending an empty chunk into
+  ``ZstdDecompressionObj.decompress()``. The issue has been fixed.
+* CFFI backend could encounter an error when calling
+  ``ZstdDecompressionReader.read()`` if there was data remaining in an
+  internal buffer. The issue has been fixed. (#71)
+
+Changes
+-------
+
+* ``ZstDecompressionObj.decompress()`` now properly handles empty inputs in
+  the CFFI backend.
+* ``ZstdCompressionReader`` now implements ``read1()`` and ``readinto1()``.
+  These are part of the ``io.BufferedIOBase`` interface.
+* ``ZstdCompressionReader`` has gained a ``readinto(b)`` method for reading
+  compressed output into an existing buffer.
+* ``ZstdCompressionReader.read()`` now defaults to ``size=-1`` and accepts
+  read sizes of ``-1`` and ``0``. The new behavior aligns with the documented
+  behavior of ``io.RawIOBase``.
+* ``ZstdCompressionReader`` now implements ``readall()``. Previously, this
+  method raised ``NotImplementedError``.
+* ``ZstdDecompressionReader`` now implements ``read1()`` and ``readinto1()``.
+  These are part of the ``io.BufferedIOBase`` interface.
+* ``ZstdDecompressionReader.read()`` now defaults to ``size=-1`` and accepts
+  read sizes of ``-1`` and ``0``. The new behavior aligns with the documented
+  behavior of ``io.RawIOBase``.
+* ``ZstdDecompressionReader()`` now implements ``readall()``. Previously, this
+  method raised ``NotImplementedError``.
+* The ``readline()``, ``readlines()``, ``__iter__``, and ``__next__`` methods
+  of ``ZstdDecompressionReader()`` now raise ``io.UnsupportedOperation``
+  instead of ``NotImplementedError``. This reflects a decision to never
+  implement text-based I/O on (de)compressors and keep the low-level API
+  operating in the binary domain. (#13)
+* ``README.rst`` now documented how to achieve linewise iteration using
+  an ``io.TextIOWrapper`` with a ``ZstdDecompressionReader``.
+* ``ZstdDecompressionReader`` has gained a ``readinto(b)`` method for
+  reading decompressed output into an existing buffer. This allows chaining
+  to an ``io.TextIOWrapper`` on Python 3 without using an ``io.BufferedReader``.
+* ``ZstdDecompressor.stream_reader()`` now accepts a ``read_across_frames``
+  argument to control behavior when the input data has multiple zstd
+  *frames*. When ``False`` (the default for backwards compatibility), a
+  ``read()`` will stop when the end of a zstd *frame* is encountered. When
+  ``True``, ``read()`` can potentially return data spanning multiple zstd
+  *frames*. The default will likely be changed to ``True`` in a future
+  release.
+* ``setup.py`` now performs CFFI version sniffing and disables the CFFI
+  backend if CFFI is too old. Previously, we only used ``install_requires``
+  to enforce the CFFI version and not all build modes would properly enforce
+  the minimum CFFI version. (#69)
+* CFFI's ``ZstdDecompressionReader.read()`` now properly handles data
+  remaining in any internal buffer. Before, repeated ``read()`` could
+  result in *random* errors. (#71)
+* Upgraded various Python packages in CI environment.
+* Upgrade to hypothesis 4.5.11.
+* In the CFFI backend, ``CompressionReader`` and ``DecompressionReader``
+  were renamed to ``ZstdCompressionReader`` and ``ZstdDecompressionReader``,
+  respectively.
+* ``ZstdDecompressor.stream_writer()`` now accepts a ``write_return_read``
+  argument to control whether ``write()`` returns the number of bytes read
+  from the source. It defaults to ``False`` to preserve backwards
+  compatibility.
+* ``ZstdDecompressor.stream_writer()`` now implements the ``io.RawIOBase``
+  interface and behaves as a proper stream object.
+* ``ZstdCompressor.stream_writer()`` now accepts a ``write_return_read``
+  argument to control whether ``write()`` returns the number of bytes read
+  from the source. It defaults to ``False`` to preserve backwards
+  compatibility.
+* ``ZstdCompressionWriter`` now implements the ``io.RawIOBase`` interface and
+  behaves as a proper stream object. ``close()`` will now close the stream
+  and the underlying stream (if possible). ``__exit__`` will now call
+  ``close()``. Methods like ``writable()`` and ``fileno()`` are implemented.
+* ``ZstdDecompressionWriter`` no longer must be used as a context manager.
+* ``ZstdCompressionWriter`` no longer must be used as a context manager.
+  When not using as a context manager, it is important to call
+  ``flush(FRAME_FRAME)`` or the compression stream won't be properly
+  terminated and decoders may complain about malformed input.
+* ``ZstdCompressionWriter.flush()`` (what is returned from
+  ``ZstdCompressor.stream_writer()``) now accepts an argument controlling the
+  flush behavior. Its value can be one of the new constants
+  ``FLUSH_BLOCK`` or ``FLUSH_FRAME``.
+* ``ZstdDecompressionObj`` instances now have a ``flush([length=None])`` method.
+  This provides parity with standard library equivalent types. (#65)
+* ``CompressionParameters`` no longer redundantly store individual compression
+  parameters on each instance. Instead, compression parameters are stored inside
+  the underlying ``ZSTD_CCtx_params`` instance. Attributes for obtaining
+  parameters are now properties rather than instance variables.
+* Exposed the ``STRATEGY_BTULTRA2`` constant.
+* ``CompressionParameters`` instances now expose an ``overlap_log`` attribute.
+  This behaves identically to the ``overlap_size_log`` attribute.
+* ``CompressionParameters()`` now accepts an ``overlap_log`` argument that
+  behaves identically to the ``overlap_size_log`` argument. An error will be
+  raised if both arguments are specified.
+* ``CompressionParameters`` instances now expose an ``ldm_hash_rate_log``
+  attribute. This behaves identically to the ``ldm_hash_every_log`` attribute.
+* ``CompressionParameters()`` now accepts a ``ldm_hash_rate_log`` argument that
+  behaves identically to the ``ldm_hash_every_log`` argument. An error will be
+  raised if both arguments are specified.
+* ``CompressionParameters()`` now accepts a ``strategy`` argument that behaves
+  identically to the ``compression_strategy`` argument. An error will be raised
+  if both arguments are specified.
+* The ``MINMATCH_MIN`` and ``MINMATCH_MAX`` constants were added. They are
+  semantically equivalent to the old ``SEARCHLENGTH_MIN`` and
+  ``SEARCHLENGTH_MAX`` constants.
+* Bundled zstandard library upgraded from 1.3.7 to 1.3.8.
+* ``setup.py`` denotes support for Python 3.7 (Python 3.7 was supported and
+  tested in the 0.10 release).
+* ``zstd_cffi`` module has been renamed to ``zstandard.cffi``.
+* ``ZstdCompressor.stream_writer()`` now reuses a buffer in order to avoid
+  allocating a new buffer for every operation. This should result in faster
+  performance in cases where ``write()`` or ``flush()`` are being called
+  frequently. (#62)
+* Bundled zstandard library upgraded from 1.3.6 to 1.3.7.
+
+0.10.2 (released 2018-11-03)
+============================
+
+Bug Fixes
+---------
+
+* ``zstd_cffi.py`` added to ``setup.py`` (#60).
+
+Changes
+-------
+
+* Change some integer casts to avoid ``ssize_t`` (#61).
+
 0.10.1 (released 2018-10-08)
 ============================
 

contrib/python-zstandard/README.rst

@@ -20,9 +20,9 b' https://github.com/indygreg/python-zstan'
 Requirements
 ============
 
-This extension is designed to run with Python 2.7, 3.4, 3.5, and 3.6
-on common platforms (Linux, Windows, and OS X). x86 and x86_64 are well-tested
-on Windows. Only x86_64 is well-tested on Linux and macOS.
+This extension is designed to run with Python 2.7, 3.4, 3.5, 3.6, and 3.7
+on common platforms (Linux, Windows, and OS X). On PyPy (both PyPy2 and PyPy3) we support version 6.0.0 and above. 
+x86 and x86_64 are well-tested on Windows. Only x86_64 is well-tested on Linux and macOS.
 
 Installing
 ==========
@@ -215,7 +215,7 b' Instances can also be used as context ma'
 
                # Do something with compressed chunk.
 
-When the context manager exists or ``close()`` is called, the stream is closed,
+When the context manager exits or ``close()`` is called, the stream is closed,
 underlying resources are released, and future operations against the compression
 stream will fail.
 
@@ -251,8 +251,54 b' emitted so far.'
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
-``stream_writer(fh)`` (which behaves as a context manager) allows you to *stream*
-data into a compressor.::
+``stream_writer(fh)`` allows you to *stream* data into a compressor.
+
+Returned instances implement the ``io.RawIOBase`` interface. Only methods
+that involve writing will do useful things.
+
+The argument to ``stream_writer()`` must have a ``write(data)`` method. As
+compressed data is available, ``write()`` will be called with the compressed
+data as its argument. Many common Python types implement ``write()``, including
+open file handles and ``io.BytesIO``.
+
+The ``write(data)`` method is used to feed data into the compressor.
+
+The ``flush([flush_mode=FLUSH_BLOCK])`` method can be called to evict whatever
+data remains within the compressor's internal state into the output object. This
+may result in 0 or more ``write()`` calls to the output object. This method
+accepts an optional ``flush_mode`` argument to control the flushing behavior.
+Its value can be any of the ``FLUSH_*`` constants.
+
+Both ``write()`` and ``flush()`` return the number of bytes written to the
+object's ``write()``. In many cases, small inputs do not accumulate enough
+data to cause a write and ``write()`` will return ``0``.
+
+Calling ``close()`` will mark the stream as closed and subsequent I/O
+operations will raise ``ValueError`` (per the documented behavior of
+``io.RawIOBase``). ``close()`` will also call ``close()`` on the underlying
+stream if such a method exists.
+
+Typically usage is as follows::
+
+   cctx = zstd.ZstdCompressor(level=10)
+   compressor = cctx.stream_writer(fh)
+
+   compressor.write(b'chunk 0\n')
+   compressor.write(b'chunk 1\n')
+   compressor.flush()
+   # Receiver will be able to decode ``chunk 0\nchunk 1\n`` at this point.
+   # Receiver is also expecting more data in the zstd *frame*.
+
+   compressor.write(b'chunk 2\n')
+   compressor.flush(zstd.FLUSH_FRAME)
+   # Receiver will be able to decode ``chunk 0\nchunk 1\nchunk 2``.
+   # Receiver is expecting no more data, as the zstd frame is closed.
+   # Any future calls to ``write()`` at this point will construct a new
+   # zstd frame.
+
+Instances can be used as context managers. Exiting the context manager is
+the equivalent of calling ``close()``, which is equivalent to calling
+``flush(zstd.FLUSH_FRAME)``::
 
    cctx = zstd.ZstdCompressor(level=10)
    with cctx.stream_writer(fh) as compressor:
@@ -260,22 +306,12 b' data into a compressor.::'
        compressor.write(b'chunk 1')
        ...
 
-The argument to ``stream_writer()`` must have a ``write(data)`` method. As
-compressed data is available, ``write()`` will be called with the compressed
-data as its argument. Many common Python types implement ``write()``, including
-open file handles and ``io.BytesIO``.
+.. important::
 
-``stream_writer()`` returns an object representing a streaming compressor
-instance. It **must** be used as a context manager. That object's
-``write(data)`` method is used to feed data into the compressor.
-
-A ``flush()`` method can be called to evict whatever data remains within the
-compressor's internal state into the output object. This may result in 0 or
-more ``write()`` calls to the output object.
-
-Both ``write()`` and ``flush()`` return the number of bytes written to the
-object's ``write()``. In many cases, small inputs do not accumulate enough
-data to cause a write and ``write()`` will return ``0``.
+   If ``flush(FLUSH_FRAME)`` is not called, emitted data doesn't constitute
+   a full zstd *frame* and consumers of this data may complain about malformed
+   input. It is recommended to use instances as a context manager to ensure
+   *frames* are properly finished.
 
 If the size of the data being fed to this streaming compressor is known,
 you can declare it before compression begins::
@@ -310,6 +346,14 b' Thte total number of bytes written so fa'
         ...
         total_written = compressor.tell()
 
+``stream_writer()`` accepts a ``write_return_read`` boolean argument to control
+the return value of ``write()``. When ``False`` (the default), ``write()`` returns
+the number of bytes that were ``write()``en to the underlying object. When
+``True``, ``write()`` returns the number of bytes read from the input that
+were subsequently written to the compressor. ``True`` is the *proper* behavior
+for ``write()`` as specified by the ``io.RawIOBase`` interface and will become
+the default value in a future release.
+
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
@@ -654,27 +698,63 b' will raise ``ValueError`` if attempted.'
 ``tell()`` returns the number of decompressed bytes read so far.
 
 Not all I/O methods are implemented. Notably missing is support for
-``readline()``, ``readlines()``, and linewise iteration support. Support for
-these is planned for a future release.
+``readline()``, ``readlines()``, and linewise iteration support. This is
+because streams operate on binary data - not text data. If you want to
+convert decompressed output to text, you can chain an ``io.TextIOWrapper``
+to the stream::
+
+   with open(path, 'rb') as fh:
+       dctx = zstd.ZstdDecompressor()
+       stream_reader = dctx.stream_reader(fh)
+       text_stream = io.TextIOWrapper(stream_reader, encoding='utf-8')
+
+       for line in text_stream:
+           ...
+
+The ``read_across_frames`` argument to ``stream_reader()`` controls the
+behavior of read operations when the end of a zstd *frame* is encountered.
+When ``False`` (the default), a read will complete when the end of a
+zstd *frame* is encountered. When ``True``, a read can potentially
+return data spanning multiple zstd *frames*.
 
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
-``stream_writer(fh)`` can be used to incrementally send compressed data to a
-decompressor.::
+``stream_writer(fh)`` allows you to *stream* data into a decompressor.
+
+Returned instances implement the ``io.RawIOBase`` interface. Only methods
+that involve writing will do useful things.
+
+The argument to ``stream_writer()`` is typically an object that also implements
+``io.RawIOBase``. But any object with a ``write(data)`` method will work. Many
+common Python types conform to this interface, including open file handles
+and ``io.BytesIO``.
+
+Behavior is similar to ``ZstdCompressor.stream_writer()``: compressed data
+is sent to the decompressor by calling ``write(data)`` and decompressed
+output is written to the underlying stream by calling its ``write(data)``
+method.::
 
     dctx = zstd.ZstdDecompressor()
-    with dctx.stream_writer(fh) as decompressor:
-        decompressor.write(compressed_data)
+    decompressor = dctx.stream_writer(fh)
 
-This behaves similarly to ``zstd.ZstdCompressor``: compressed data is written to
-the decompressor by calling ``write(data)`` and decompressed output is written
-to the output object by calling its ``write(data)`` method.
+    decompressor.write(compressed_data)
+    ...
+
 
 Calls to ``write()`` will return the number of bytes written to the output
 object. Not all inputs will result in bytes being written, so return values
 of ``0`` are possible.
 
+Like the ``stream_writer()`` compressor, instances can be used as context
+managers. However, context managers add no extra special behavior and offer
+little to no benefit to being used.
+
+Calling ``close()`` will mark the stream as closed and subsequent I/O operations
+will raise ``ValueError`` (per the documented behavior of ``io.RawIOBase``).
+``close()`` will also call ``close()`` on the underlying stream if such a
+method exists.
+
 The size of chunks being ``write()`` to the destination can be specified::
 
     dctx = zstd.ZstdDecompressor()
@@ -687,6 +767,13 b' You can see how much memory is being use'
     with dctx.stream_writer(fh) as decompressor:
         byte_size = decompressor.memory_size()
 
+``stream_writer()`` accepts a ``write_return_read`` boolean argument to control
+the return value of ``write()``. When ``False`` (the default)``, ``write()``
+returns the number of bytes that were ``write()``en to the underlying stream.
+When ``True``, ``write()`` returns the number of bytes read from the input.
+``True`` is the *proper* behavior for ``write()`` as specified by the
+``io.RawIOBase`` interface and will become the default in a future release.
+
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
@@ -791,6 +878,10 b' these temporary chunks by passing ``writ'
    memory (re)allocations, this streaming decompression API isn't as
    efficient as other APIs.
 
+For compatibility with the standard library APIs, instances expose a
+``flush([length=None])`` method. This method no-ops and has no meaningful
+side-effects, making it safe to call any time.
+
 Batch Decompression API
 ^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -1147,18 +1238,21 b' follows:'
 * search_log
 * min_match
 * target_length
-* compression_strategy
+* strategy
+* compression_strategy (deprecated: same as ``strategy``)
 * write_content_size
 * write_checksum
 * write_dict_id
 * job_size
-* overlap_size_log
+* overlap_log
+* overlap_size_log (deprecated: same as ``overlap_log``)
 * force_max_window
 * enable_ldm
 * ldm_hash_log
 * ldm_min_match
 * ldm_bucket_size_log
-* ldm_hash_every_log
+* ldm_hash_rate_log
+* ldm_hash_every_log (deprecated: same as ``ldm_hash_rate_log``)
 * threads
 
 Some of these are very low-level settings. It may help to consult the official
@@ -1240,6 +1334,13 b' FRAME_HEADER'
 MAGIC_NUMBER
     Frame header as an integer
 
+FLUSH_BLOCK
+    Flushing behavior that denotes to flush a zstd block. A decompressor will
+    be able to decode all data fed into the compressor so far.
+FLUSH_FRAME
+    Flushing behavior that denotes to end a zstd frame. Any new data fed
+    to the compressor will start a new frame.
+
 CONTENTSIZE_UNKNOWN
     Value for content size when the content size is unknown.
 CONTENTSIZE_ERROR
@@ -1261,10 +1362,18 b' SEARCHLOG_MIN'
     Minimum value for compression parameter
 SEARCHLOG_MAX
     Maximum value for compression parameter
+MINMATCH_MIN
+    Minimum value for compression parameter
+MINMATCH_MAX
+    Maximum value for compression parameter
 SEARCHLENGTH_MIN
     Minimum value for compression parameter
+
+    Deprecated: use ``MINMATCH_MIN``
 SEARCHLENGTH_MAX
     Maximum value for compression parameter
+
+    Deprecated: use ``MINMATCH_MAX``
 TARGETLENGTH_MIN
     Minimum value for compression parameter
 STRATEGY_FAST
@@ -1283,6 +1392,8 b' STRATEGY_BTOPT'
     Compression strategy
 STRATEGY_BTULTRA
     Compression strategy
+STRATEGY_BTULTRA2
+    Compression strategy
 
 FORMAT_ZSTD1
     Zstandard frame format

contrib/python-zstandard/c-ext/compressionchunker.c

@@ -43,7 +43,7 b' static PyObject* ZstdCompressionChunkerI'
 	/* If we have data left in the input, consume it. */
 	while (chunker->input.pos < chunker->input.size) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(chunker->compressor->cctx, &chunker->output,
+		zresult = ZSTD_compressStream2(chunker->compressor->cctx, &chunker->output,
 			&chunker->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
@@ -104,7 +104,7 b' static PyObject* ZstdCompressionChunkerI'
 	}
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_compress_generic(chunker->compressor->cctx, &chunker->output,
+	zresult = ZSTD_compressStream2(chunker->compressor->cctx, &chunker->output,
 		&chunker->input, zFlushMode);
 	Py_END_ALLOW_THREADS
 

contrib/python-zstandard/c-ext/compressiondict.c

@@ -298,13 +298,9 b' static PyObject* ZstdCompressionDict_pre'
 		cParams = ZSTD_getCParams(level, 0, self->dictSize);
 	}
 	else {
-		cParams.chainLog = compressionParams->chainLog;
-		cParams.hashLog = compressionParams->hashLog;
-		cParams.searchLength = compressionParams->minMatch;
-		cParams.searchLog = compressionParams->searchLog;
-		cParams.strategy = compressionParams->compressionStrategy;
-		cParams.targetLength = compressionParams->targetLength;
-		cParams.windowLog = compressionParams->windowLog;
+		if (to_cparams(compressionParams, &cParams)) {
+			return NULL;
+		}
 	}
 
 	assert(!self->cdict);

contrib/python-zstandard/c-ext/compressionparams.c

@@ -10,7 +10,7 b''
 
 extern PyObject* ZstdError;
 
-int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value) {
+int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value) {
 	size_t zresult = ZSTD_CCtxParam_setParameter(params, param, value);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "unable to set compression context parameter: %s",
@@ -23,28 +23,41 b' int set_parameter(ZSTD_CCtx_params* para'
 
 #define TRY_SET_PARAMETER(params, param, value) if (set_parameter(params, param, value)) return -1;
 
+#define TRY_COPY_PARAMETER(source, dest, param) { \
+	int result; \
+	size_t zresult = ZSTD_CCtxParam_getParameter(source, param, &result); \
+	if (ZSTD_isError(zresult)) { \
+		return 1; \
+	} \
+	zresult = ZSTD_CCtxParam_setParameter(dest, param, result); \
+	if (ZSTD_isError(zresult)) { \
+		return 1; \
+	} \
+}
+
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj) {
-	TRY_SET_PARAMETER(params, ZSTD_p_format, obj->format);
-	TRY_SET_PARAMETER(params, ZSTD_p_compressionLevel, (unsigned)obj->compressionLevel);
-	TRY_SET_PARAMETER(params, ZSTD_p_windowLog, obj->windowLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_hashLog, obj->hashLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_chainLog, obj->chainLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_searchLog, obj->searchLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_minMatch, obj->minMatch);
-	TRY_SET_PARAMETER(params, ZSTD_p_targetLength, obj->targetLength);
-	TRY_SET_PARAMETER(params, ZSTD_p_compressionStrategy, obj->compressionStrategy);
-	TRY_SET_PARAMETER(params, ZSTD_p_contentSizeFlag, obj->contentSizeFlag);
-	TRY_SET_PARAMETER(params, ZSTD_p_checksumFlag, obj->checksumFlag);
-	TRY_SET_PARAMETER(params, ZSTD_p_dictIDFlag, obj->dictIDFlag);
-	TRY_SET_PARAMETER(params, ZSTD_p_nbWorkers, obj->threads);
-	TRY_SET_PARAMETER(params, ZSTD_p_jobSize, obj->jobSize);
-	TRY_SET_PARAMETER(params, ZSTD_p_overlapSizeLog, obj->overlapSizeLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_forceMaxWindow, obj->forceMaxWindow);
-	TRY_SET_PARAMETER(params, ZSTD_p_enableLongDistanceMatching, obj->enableLongDistanceMatching);
-	TRY_SET_PARAMETER(params, ZSTD_p_ldmHashLog, obj->ldmHashLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_ldmMinMatch, obj->ldmMinMatch);
-	TRY_SET_PARAMETER(params, ZSTD_p_ldmBucketSizeLog, obj->ldmBucketSizeLog);
-	TRY_SET_PARAMETER(params, ZSTD_p_ldmHashEveryLog, obj->ldmHashEveryLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_nbWorkers);
+
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_format);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_compressionLevel);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_windowLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_hashLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_chainLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_searchLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_minMatch);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_targetLength);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_strategy);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_contentSizeFlag);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_checksumFlag);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_dictIDFlag);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_jobSize);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_overlapLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_forceMaxWindow);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_enableLongDistanceMatching);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmMinMatch);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmBucketSizeLog);
+	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashRateLog);
 
 	return 0;
 }
@@ -64,6 +77,41 b' int reset_params(ZstdCompressionParamete'
 	return set_parameters(params->params, params);
 }
 
+#define TRY_GET_PARAMETER(params, param, value) { \
+    size_t zresult = ZSTD_CCtxParam_getParameter(params, param, value); \
+    if (ZSTD_isError(zresult)) { \
+        PyErr_Format(ZstdError, "unable to retrieve parameter: %s", ZSTD_getErrorName(zresult)); \
+        return 1; \
+    } \
+}
+
+int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams) {
+	int value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_windowLog, &value);
+	cparams->windowLog = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_chainLog, &value);
+	cparams->chainLog = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_hashLog, &value);
+	cparams->hashLog = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_searchLog, &value);
+	cparams->searchLog = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_minMatch, &value);
+	cparams->minMatch = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_targetLength, &value);
+	cparams->targetLength = value;
+
+	TRY_GET_PARAMETER(params->params, ZSTD_c_strategy, &value);
+	cparams->strategy = value;
+
+	return 0;
+}
+
 static int ZstdCompressionParameters_init(ZstdCompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"format",
@@ -75,50 +123,60 b' static int ZstdCompressionParameters_ini'
 		"min_match",
 		"target_length",
 		"compression_strategy",
+		"strategy",
 		"write_content_size",
 		"write_checksum",
 		"write_dict_id",
 		"job_size",
+		"overlap_log",
 		"overlap_size_log",
 		"force_max_window",
 		"enable_ldm",
 		"ldm_hash_log",
 		"ldm_min_match",
 		"ldm_bucket_size_log",
+		"ldm_hash_rate_log",
 		"ldm_hash_every_log",
 		"threads",
 		NULL
 	};
 
-	unsigned format = 0;
+	int format = 0;
 	int compressionLevel = 0;
-	unsigned windowLog = 0;
-	unsigned hashLog = 0;
-	unsigned chainLog = 0;
-	unsigned searchLog = 0;
-	unsigned minMatch = 0;
-	unsigned targetLength = 0;
-	unsigned compressionStrategy = 0;
-	unsigned contentSizeFlag = 1;
-	unsigned checksumFlag = 0;
-	unsigned dictIDFlag = 0;
-	unsigned jobSize = 0;
-	unsigned overlapSizeLog = 0;
-	unsigned forceMaxWindow = 0;
-	unsigned enableLDM = 0;
-	unsigned ldmHashLog = 0;
-	unsigned ldmMinMatch = 0;
-	unsigned ldmBucketSizeLog = 0;
-	unsigned ldmHashEveryLog = 0;
+	int windowLog = 0;
+	int hashLog = 0;
+	int chainLog = 0;
+	int searchLog = 0;
+	int minMatch = 0;
+	int targetLength = 0;
+	int compressionStrategy = -1;
+	int strategy = -1;
+	int contentSizeFlag = 1;
+	int checksumFlag = 0;
+	int dictIDFlag = 0;
+	int jobSize = 0;
+	int overlapLog = -1;
+	int overlapSizeLog = -1;
+	int forceMaxWindow = 0;
+	int enableLDM = 0;
+	int ldmHashLog = 0;
+	int ldmMinMatch = 0;
+	int ldmBucketSizeLog = 0;
+	int ldmHashRateLog = -1;
+	int ldmHashEveryLog = -1;
 	int threads = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-		"|IiIIIIIIIIIIIIIIIIIIi:CompressionParameters",
+		"|iiiiiiiiiiiiiiiiiiiiiiii:CompressionParameters",
 		kwlist, &format, &compressionLevel, &windowLog, &hashLog, &chainLog,
-		&searchLog, &minMatch, &targetLength, &compressionStrategy,
-		&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapSizeLog,
-		&forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch, &ldmBucketSizeLog,
-		&ldmHashEveryLog, &threads)) {
+		&searchLog, &minMatch, &targetLength, &compressionStrategy, &strategy,
+		&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapLog,
+		&overlapSizeLog, &forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch,
+		&ldmBucketSizeLog, &ldmHashRateLog, &ldmHashEveryLog, &threads)) {
+		return -1;
+	}
+
+	if (reset_params(self)) {
 		return -1;
 	}
 
@@ -126,32 +184,70 b' static int ZstdCompressionParameters_ini'
 		threads = cpu_count();
 	}
 
-	self->format = format;
-	self->compressionLevel = compressionLevel;
-	self->windowLog = windowLog;
-	self->hashLog = hashLog;
-	self->chainLog = chainLog;
-	self->searchLog = searchLog;
-	self->minMatch = minMatch;
-	self->targetLength = targetLength;
-	self->compressionStrategy = compressionStrategy;
-	self->contentSizeFlag = contentSizeFlag;
-	self->checksumFlag = checksumFlag;
-	self->dictIDFlag = dictIDFlag;
-	self->threads = threads;
-	self->jobSize = jobSize;
-	self->overlapSizeLog = overlapSizeLog;
-	self->forceMaxWindow = forceMaxWindow;
-	self->enableLongDistanceMatching = enableLDM;
-	self->ldmHashLog = ldmHashLog;
-	self->ldmMinMatch = ldmMinMatch;
-	self->ldmBucketSizeLog = ldmBucketSizeLog;
-	self->ldmHashEveryLog = ldmHashEveryLog;
+	/* We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
+	 * because setting ZSTD_c_nbWorkers resets the other parameters. */
+	TRY_SET_PARAMETER(self->params, ZSTD_c_nbWorkers, threads);
+
+	TRY_SET_PARAMETER(self->params, ZSTD_c_format, format);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_compressionLevel, compressionLevel);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_windowLog, windowLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_hashLog, hashLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_chainLog, chainLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_searchLog, searchLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_minMatch, minMatch);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_targetLength, targetLength);
 
-	if (reset_params(self)) {
+	if (compressionStrategy != -1 && strategy != -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot specify both compression_strategy and strategy");
+		return -1;
+    }
+
+	if (compressionStrategy != -1) {
+		strategy = compressionStrategy;
+	}
+	else if (strategy == -1) {
+		strategy = 0;
+	}
+
+	TRY_SET_PARAMETER(self->params, ZSTD_c_strategy, strategy);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_contentSizeFlag, contentSizeFlag);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_checksumFlag, checksumFlag);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_dictIDFlag, dictIDFlag);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_jobSize, jobSize);
+
+	if (overlapLog != -1 && overlapSizeLog != -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot specify both overlap_log and overlap_size_log");
 		return -1;
 	}
 
+	if (overlapSizeLog != -1) {
+		overlapLog = overlapSizeLog;
+	}
+	else if (overlapLog == -1) {
+		overlapLog = 0;
+	}
+
+	TRY_SET_PARAMETER(self->params, ZSTD_c_overlapLog, overlapLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_forceMaxWindow, forceMaxWindow);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_enableLongDistanceMatching, enableLDM);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashLog, ldmHashLog);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmMinMatch, ldmMinMatch);
+	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmBucketSizeLog, ldmBucketSizeLog);
+
+	if (ldmHashRateLog != -1 && ldmHashEveryLog != -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot specify both ldm_hash_rate_log and ldm_hash_everyLog");
+		return -1;
+	}
+
+	if (ldmHashEveryLog != -1) {
+		ldmHashRateLog = ldmHashEveryLog;
+	}
+	else if (ldmHashRateLog == -1) {
+		ldmHashRateLog = 0;
+	}
+
+	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashRateLog, ldmHashRateLog);
+
 	return 0;
 }
 
@@ -259,7 +355,7 b' ZstdCompressionParametersObject* Compres'
 
 	val = PyDict_GetItemString(kwargs, "min_match");
 	if (!val) {
-		val = PyLong_FromUnsignedLong(params.searchLength);
+		val = PyLong_FromUnsignedLong(params.minMatch);
 		if (!val) {
 			goto cleanup;
 		}
@@ -336,6 +432,41 b' static void ZstdCompressionParameters_de'
 	PyObject_Del(self);
 }
 
+#define PARAM_GETTER(name, param) PyObject* ZstdCompressionParameters_get_##name(PyObject* self, void* unused) { \
+    int result; \
+    size_t zresult; \
+    ZstdCompressionParametersObject* p = (ZstdCompressionParametersObject*)(self); \
+    zresult = ZSTD_CCtxParam_getParameter(p->params, param, &result); \
+    if (ZSTD_isError(zresult)) { \
+        PyErr_Format(ZstdError, "unable to get compression parameter: %s", \
+            ZSTD_getErrorName(zresult)); \
+        return NULL; \
+    } \
+    return PyLong_FromLong(result); \
+}
+
+PARAM_GETTER(format, ZSTD_c_format)
+PARAM_GETTER(compression_level, ZSTD_c_compressionLevel)
+PARAM_GETTER(window_log, ZSTD_c_windowLog)
+PARAM_GETTER(hash_log, ZSTD_c_hashLog)
+PARAM_GETTER(chain_log, ZSTD_c_chainLog)
+PARAM_GETTER(search_log, ZSTD_c_searchLog)
+PARAM_GETTER(min_match, ZSTD_c_minMatch)
+PARAM_GETTER(target_length, ZSTD_c_targetLength)
+PARAM_GETTER(compression_strategy, ZSTD_c_strategy)
+PARAM_GETTER(write_content_size, ZSTD_c_contentSizeFlag)
+PARAM_GETTER(write_checksum, ZSTD_c_checksumFlag)
+PARAM_GETTER(write_dict_id, ZSTD_c_dictIDFlag)
+PARAM_GETTER(job_size, ZSTD_c_jobSize)
+PARAM_GETTER(overlap_log, ZSTD_c_overlapLog)
+PARAM_GETTER(force_max_window, ZSTD_c_forceMaxWindow)
+PARAM_GETTER(enable_ldm, ZSTD_c_enableLongDistanceMatching)
+PARAM_GETTER(ldm_hash_log, ZSTD_c_ldmHashLog)
+PARAM_GETTER(ldm_min_match, ZSTD_c_ldmMinMatch)
+PARAM_GETTER(ldm_bucket_size_log, ZSTD_c_ldmBucketSizeLog)
+PARAM_GETTER(ldm_hash_rate_log, ZSTD_c_ldmHashRateLog)
+PARAM_GETTER(threads, ZSTD_c_nbWorkers)
+
 static PyMethodDef ZstdCompressionParameters_methods[] = {
 	{
 		"from_level",
@@ -352,70 +483,34 b' static PyMethodDef ZstdCompressionParame'
 	{ NULL, NULL }
 };
 
-static PyMemberDef ZstdCompressionParameters_members[] = {
-	{ "format", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, format), READONLY,
-	  "compression format" },
-	{ "compression_level", T_INT,
-	  offsetof(ZstdCompressionParametersObject, compressionLevel), READONLY,
-	  "compression level" },
-	{ "window_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, windowLog), READONLY,
-	  "window log" },
-	{ "hash_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, hashLog), READONLY,
-	  "hash log" },
-	{ "chain_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, chainLog), READONLY,
-	  "chain log" },
-	{ "search_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, searchLog), READONLY,
-	  "search log" },
-	{ "min_match", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, minMatch), READONLY,
-	  "search length" },
-	{ "target_length", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, targetLength), READONLY,
-	  "target length" },
-	{ "compression_strategy", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, compressionStrategy), READONLY,
-	  "compression strategy" },
-	{ "write_content_size", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, contentSizeFlag), READONLY,
-	  "whether to write content size in frames" },
-	{ "write_checksum", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, checksumFlag), READONLY,
-	  "whether to write checksum in frames" },
-	{ "write_dict_id", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, dictIDFlag), READONLY,
-	  "whether to write dictionary ID in frames" },
-	{ "threads", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, threads), READONLY,
-	  "number of threads to use" },
-	{ "job_size", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, jobSize), READONLY,
-	  "size of compression job when using multiple threads" },
-	{ "overlap_size_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, overlapSizeLog), READONLY,
-	  "Size of previous input reloaded at the beginning of each job" },
-	{ "force_max_window", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, forceMaxWindow), READONLY,
-	  "force back references to remain smaller than window size" },
-	{ "enable_ldm", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, enableLongDistanceMatching), READONLY,
-	  "whether to enable long distance matching" },
-	{ "ldm_hash_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, ldmHashLog), READONLY,
-	  "Size of the table for long distance matching, as a power of 2" },
-	{ "ldm_min_match", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, ldmMinMatch), READONLY,
-	  "minimum size of searched matches for long distance matcher" },
-	{ "ldm_bucket_size_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, ldmBucketSizeLog), READONLY,
-	  "log size of each bucket in the LDM hash table for collision resolution" },
-	{ "ldm_hash_every_log", T_UINT,
-	  offsetof(ZstdCompressionParametersObject, ldmHashEveryLog), READONLY,
-	  "frequency of inserting/looking up entries in the LDM hash table" },
+#define GET_SET_ENTRY(name) { #name, ZstdCompressionParameters_get_##name, NULL, NULL, NULL }
+
+static PyGetSetDef ZstdCompressionParameters_getset[] = {
+	GET_SET_ENTRY(format),
+	GET_SET_ENTRY(compression_level),
+	GET_SET_ENTRY(window_log),
+	GET_SET_ENTRY(hash_log),
+	GET_SET_ENTRY(chain_log),
+	GET_SET_ENTRY(search_log),
+	GET_SET_ENTRY(min_match),
+	GET_SET_ENTRY(target_length),
+	GET_SET_ENTRY(compression_strategy),
+	GET_SET_ENTRY(write_content_size),
+	GET_SET_ENTRY(write_checksum),
+	GET_SET_ENTRY(write_dict_id),
+	GET_SET_ENTRY(threads),
+	GET_SET_ENTRY(job_size),
+	GET_SET_ENTRY(overlap_log),
+	/* TODO remove this deprecated attribute */
+	{ "overlap_size_log", ZstdCompressionParameters_get_overlap_log, NULL, NULL, NULL },
+	GET_SET_ENTRY(force_max_window),
+	GET_SET_ENTRY(enable_ldm),
+	GET_SET_ENTRY(ldm_hash_log),
+	GET_SET_ENTRY(ldm_min_match),
+	GET_SET_ENTRY(ldm_bucket_size_log),
+	GET_SET_ENTRY(ldm_hash_rate_log),
+	/* TODO remove this deprecated attribute */
+	{ "ldm_hash_every_log", ZstdCompressionParameters_get_ldm_hash_rate_log, NULL, NULL, NULL },
 	{ NULL }
 };
 
@@ -448,8 +543,8 b' PyTypeObject ZstdCompressionParametersTy'
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	ZstdCompressionParameters_methods, /* tp_methods */
-	ZstdCompressionParameters_members, /* tp_members */
-	0,                         /* tp_getset */
+	0,                          /* tp_members */
+	ZstdCompressionParameters_getset,  /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */

contrib/python-zstandard/c-ext/compressionreader.c

@@ -128,6 +128,96 b' static PyObject* reader_tell(ZstdCompres'
 	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
 }
 
+int read_compressor_input(ZstdCompressionReader* self) {
+	if (self->finishedInput) {
+		return 0;
+	}
+
+	if (self->input.pos != self->input.size) {
+		return 0;
+	}
+
+	if (self->reader) {
+		Py_buffer buffer;
+
+		assert(self->readResult == NULL);
+
+		self->readResult = PyObject_CallMethod(self->reader, "read",
+		    "k", self->readSize);
+
+		if (NULL == self->readResult) {
+			return -1;
+		}
+
+		memset(&buffer, 0, sizeof(buffer));
+
+		if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
+			return -1;
+		}
+
+		/* EOF */
+		if (0 == buffer.len) {
+			self->finishedInput = 1;
+			Py_CLEAR(self->readResult);
+		}
+		else {
+			self->input.src = buffer.buf;
+			self->input.size = buffer.len;
+			self->input.pos = 0;
+		}
+
+		PyBuffer_Release(&buffer);
+	}
+	else {
+		assert(self->buffer.buf);
+
+		self->input.src = self->buffer.buf;
+		self->input.size = self->buffer.len;
+		self->input.pos = 0;
+	}
+
+	return 1;
+}
+
+int compress_input(ZstdCompressionReader* self, ZSTD_outBuffer* output) {
+	size_t oldPos;
+	size_t zresult;
+
+	/* If we have data left over, consume it. */
+	if (self->input.pos < self->input.size) {
+		oldPos = output->pos;
+
+		Py_BEGIN_ALLOW_THREADS
+		zresult = ZSTD_compressStream2(self->compressor->cctx,
+		    output, &self->input, ZSTD_e_continue);
+		Py_END_ALLOW_THREADS
+
+		self->bytesCompressed += output->pos - oldPos;
+
+		/* Input exhausted. Clear out state tracking. */
+		if (self->input.pos == self->input.size) {
+			memset(&self->input, 0, sizeof(self->input));
+			Py_CLEAR(self->readResult);
+
+			if (self->buffer.buf) {
+				self->finishedInput = 1;
+			}
+		}
+
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
+			return -1;
+		}
+	}
+
+    if (output->pos && output->pos == output->size) {
+        return 1;
+    }
+    else {
+        return 0;
+    }
+}
+
 static PyObject* reader_read(ZstdCompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
@@ -140,25 +230,30 b' static PyObject* reader_read(ZstdCompres'
 	Py_ssize_t resultSize;
 	size_t zresult;
 	size_t oldPos;
+	int readResult, compressResult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
-	if (self->finishedOutput) {
-		return PyBytes_FromStringAndSize("", 0);
-	}
-
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "n", kwlist, &size)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
-	if (size < 1) {
-		PyErr_SetString(PyExc_ValueError, "cannot read negative or size 0 amounts");
+	if (size < -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
+	if (size == -1) {
+		return PyObject_CallMethod((PyObject*)self, "readall", NULL);
+	}
+
+	if (self->finishedOutput || size == 0) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
@@ -172,86 +267,34 b' static PyObject* reader_read(ZstdCompres'
 
 readinput:
 
-	/* If we have data left over, consume it. */
-	if (self->input.pos < self->input.size) {
-		oldPos = self->output.pos;
-
-		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->compressor->cctx,
-			&self->output, &self->input, ZSTD_e_continue);
-
-		Py_END_ALLOW_THREADS
-
-		self->bytesCompressed += self->output.pos - oldPos;
-
-		/* Input exhausted. Clear out state tracking. */
-		if (self->input.pos == self->input.size) {
-			memset(&self->input, 0, sizeof(self->input));
-			Py_CLEAR(self->readResult);
+    compressResult = compress_input(self, &self->output);
 
-			if (self->buffer.buf) {
-				self->finishedInput = 1;
-			}
-		}
-
-		if (ZSTD_isError(zresult)) {
-			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
-			return NULL;
-		}
-
-		if (self->output.pos) {
-			/* If no more room in output, emit it. */
-			if (self->output.pos == self->output.size) {
-				memset(&self->output, 0, sizeof(self->output));
-				return result;
-			}
-
-			/*
-			 * There is room in the output. We fall through to below, which will either
-			 * get more input for us or will attempt to end the stream.
-			 */
-		}
-
-		/* Fall through to gather more input. */
+	if (-1 == compressResult) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+	else if (0 == compressResult) {
+		/* There is room in the output. We fall through to below, which will
+		 * either get more input for us or will attempt to end the stream.
+		 */
+	}
+	else if (1 == compressResult) {
+		memset(&self->output, 0, sizeof(self->output));
+		return result;
+	}
+	else {
+		assert(0);
 	}
 
-	if (!self->finishedInput) {
-		if (self->reader) {
-			Py_buffer buffer;
-
-			assert(self->readResult == NULL);
-			self->readResult = PyObject_CallMethod(self->reader, "read",
-				"k", self->readSize);
-			if (self->readResult == NULL) {
-				return NULL;
-			}
-
-			memset(&buffer, 0, sizeof(buffer));
-
-			if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
-				return NULL;
-			}
+	readResult = read_compressor_input(self);
 
-			/* EOF */
-			if (0 == buffer.len) {
-				self->finishedInput = 1;
-				Py_CLEAR(self->readResult);
-			}
-			else {
-				self->input.src = buffer.buf;
-				self->input.size = buffer.len;
-				self->input.pos = 0;
-			}
-
-			PyBuffer_Release(&buffer);
-		}
-		else {
-			assert(self->buffer.buf);
-
-			self->input.src = self->buffer.buf;
-			self->input.size = self->buffer.len;
-			self->input.pos = 0;
-		}
+	if (-1 == readResult) {
+		return NULL;
+	}
+	else if (0 == readResult) { }
+	else if (1 == readResult) { }
+	else {
+		assert(0);
 	}
 
 	if (self->input.size) {
@@ -261,7 +304,7 b' readinput:'
 	/* Else EOF */
 	oldPos = self->output.pos;
 
-	zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+	zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 		&self->input, ZSTD_e_end);
 
 	self->bytesCompressed += self->output.pos - oldPos;
@@ -269,6 +312,7 b' readinput:'
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error ending compression stream: %s",
 			ZSTD_getErrorName(zresult));
+		Py_XDECREF(result);
 		return NULL;
 	}
 
@@ -288,9 +332,394 b' readinput:'
 	return result;
 }
 
+static PyObject* reader_read1(ZstdCompressionReader* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"size",
+		NULL
+	};
+
+	Py_ssize_t size = -1;
+	PyObject* result = NULL;
+	char* resultBuffer;
+	Py_ssize_t resultSize;
+	ZSTD_outBuffer output;
+	int compressResult;
+	size_t oldPos;
+	size_t zresult;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n:read1", kwlist, &size)) {
+		return NULL;
+	}
+
+	if (size < -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
+		return NULL;
+	}
+
+	if (self->finishedOutput || size == 0) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
+	if (size == -1) {
+		size = ZSTD_CStreamOutSize();
+	}
+
+	result = PyBytes_FromStringAndSize(NULL, size);
+	if (NULL == result) {
+		return NULL;
+	}
+
+	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
+
+	output.dst = resultBuffer;
+	output.size = resultSize;
+	output.pos = 0;
+
+	/* read1() is supposed to use at most 1 read() from the underlying stream.
+	   However, we can't satisfy this requirement with compression because
+	   not every input will generate output. We /could/ flush the compressor,
+	   but this may not be desirable. We allow multiple read() from the
+	   underlying stream. But unlike read(), we return as soon as output data
+	   is available.
+	*/
+
+	compressResult = compress_input(self, &output);
+
+	if (-1 == compressResult) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+	else if (0 == compressResult || 1 == compressResult) { }
+	else {
+		assert(0);
+	}
+
+	if (output.pos) {
+		goto finally;
+	}
+
+	while (!self->finishedInput) {
+		int readResult = read_compressor_input(self);
+
+		if (-1 == readResult) {
+			Py_XDECREF(result);
+			return NULL;
+		}
+		else if (0 == readResult || 1 == readResult) { }
+		else {
+			assert(0);
+		}
+
+		compressResult = compress_input(self, &output);
+
+		if (-1 == compressResult) {
+			Py_XDECREF(result);
+			return NULL;
+		}
+		else if (0 == compressResult || 1 == compressResult) { }
+		else {
+			assert(0);
+		}
+
+		if (output.pos) {
+			goto finally;
+		}
+	}
+
+	/* EOF */
+	oldPos = output.pos;
+
+	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
+        ZSTD_e_end);
+
+	self->bytesCompressed += output.pos - oldPos;
+
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error ending compression stream: %s",
+		    ZSTD_getErrorName(zresult));
+		Py_XDECREF(result);
+		return NULL;
+	}
+
+	if (zresult == 0) {
+		self->finishedOutput = 1;
+	}
+
+finally:
+	if (result) {
+		if (safe_pybytes_resize(&result, output.pos)) {
+			Py_XDECREF(result);
+			return NULL;
+		}
+	}
+
+	return result;
+}
+
 static PyObject* reader_readall(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
-	return NULL;
+	PyObject* chunks = NULL;
+	PyObject* empty = NULL;
+	PyObject* result = NULL;
+
+	/* Our strategy is to collect chunks into a list then join all the
+	 * chunks at the end. We could potentially use e.g. an io.BytesIO. But
+	 * this feels simple enough to implement and avoids potentially expensive
+	 * reallocations of large buffers.
+	 */
+	chunks = PyList_New(0);
+	if (NULL == chunks) {
+		return NULL;
+	}
+
+	while (1) {
+		PyObject* chunk = PyObject_CallMethod(self, "read", "i", 1048576);
+		if (NULL == chunk) {
+			Py_DECREF(chunks);
+			return NULL;
+		}
+
+		if (!PyBytes_Size(chunk)) {
+			Py_DECREF(chunk);
+			break;
+		}
+
+		if (PyList_Append(chunks, chunk)) {
+			Py_DECREF(chunk);
+			Py_DECREF(chunks);
+			return NULL;
+		}
+
+		Py_DECREF(chunk);
+	}
+
+	empty = PyBytes_FromStringAndSize("", 0);
+	if (NULL == empty) {
+		Py_DECREF(chunks);
+		return NULL;
+	}
+
+	result = PyObject_CallMethod(empty, "join", "O", chunks);
+
+	Py_DECREF(empty);
+	Py_DECREF(chunks);
+
+	return result;
+}
+
+static PyObject* reader_readinto(ZstdCompressionReader* self, PyObject* args) {
+	Py_buffer dest;
+	ZSTD_outBuffer output;
+	int readResult, compressResult;
+	PyObject* result = NULL;
+	size_t zresult;
+	size_t oldPos;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyLong_FromLong(0);
+	}
+
+	if (!PyArg_ParseTuple(args, "w*:readinto", &dest)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+		    "destination buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	output.dst = dest.buf;
+	output.size = dest.len;
+	output.pos = 0;
+
+	compressResult = compress_input(self, &output);
+
+	if (-1 == compressResult) {
+		goto finally;
+	}
+	else if (0 == compressResult) {	}
+	else if (1 == compressResult) {
+		result = PyLong_FromSize_t(output.pos);
+		goto finally;
+	}
+	else {
+		assert(0);
+	}
+
+	while (!self->finishedInput) {
+		readResult = read_compressor_input(self);
+
+		if (-1 == readResult) {
+			goto finally;
+		}
+		else if (0 == readResult || 1 == readResult) {}
+		else {
+			assert(0);
+		}
+
+		compressResult = compress_input(self, &output);
+
+		if (-1 == compressResult) {
+			goto finally;
+		}
+		else if (0 == compressResult) { }
+		else if (1 == compressResult) {
+			result = PyLong_FromSize_t(output.pos);
+			goto finally;
+		}
+		else {
+			assert(0);
+		}
+	}
+
+	/* EOF */
+	oldPos = output.pos;
+
+	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
+	    ZSTD_e_end);
+
+	self->bytesCompressed += self->output.pos - oldPos;
+
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error ending compression stream: %s",
+		    ZSTD_getErrorName(zresult));
+		goto finally;
+	}
+
+	assert(output.pos);
+
+	if (0 == zresult) {
+		self->finishedOutput = 1;
+	}
+
+	result = PyLong_FromSize_t(output.pos);
+
+finally:
+	PyBuffer_Release(&dest);
+
+	return result;
+}
+
+static PyObject* reader_readinto1(ZstdCompressionReader* self, PyObject* args) {
+	Py_buffer dest;
+	PyObject* result = NULL;
+	ZSTD_outBuffer output;
+	int compressResult;
+	size_t oldPos;
+	size_t zresult;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyLong_FromLong(0);
+	}
+
+	if (!PyArg_ParseTuple(args, "w*:readinto1", &dest)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+		    "destination buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	output.dst = dest.buf;
+	output.size = dest.len;
+	output.pos = 0;
+
+	compressResult = compress_input(self, &output);
+
+	if (-1 == compressResult) {
+		goto finally;
+	}
+	else if (0 == compressResult || 1 == compressResult) { }
+	else {
+		assert(0);
+	}
+
+	if (output.pos) {
+		result = PyLong_FromSize_t(output.pos);
+		goto finally;
+	}
+
+	while (!self->finishedInput) {
+		int readResult = read_compressor_input(self);
+
+		if (-1 == readResult) {
+			goto finally;
+		}
+		else if (0 == readResult || 1 == readResult) { }
+		else {
+			assert(0);
+		}
+
+		compressResult = compress_input(self, &output);
+
+		if (-1 == compressResult) {
+			goto finally;
+		}
+		else if (0 == compressResult) { }
+		else if (1 == compressResult) {
+			result = PyLong_FromSize_t(output.pos);
+			goto finally;
+		}
+		else {
+			assert(0);
+		}
+
+		/* If we produced output and we're not done with input, emit
+		 * that output now, as we've hit restrictions of read1().
+		 */
+		if (output.pos && !self->finishedInput) {
+			result = PyLong_FromSize_t(output.pos);
+			goto finally;
+		}
+
+		/* Otherwise we either have no output or we've exhausted the
+		 * input. Either we try to get more input or we fall through
+		 * to EOF below */
+	}
+
+	/* EOF */
+	oldPos = output.pos;
+
+	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
+	    ZSTD_e_end);
+
+	self->bytesCompressed += self->output.pos - oldPos;
+
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error ending compression stream: %s",
+		    ZSTD_getErrorName(zresult));
+		goto finally;
+	}
+
+	assert(output.pos);
+
+	if (0 == zresult) {
+		self->finishedOutput = 1;
+	}
+
+	result = PyLong_FromSize_t(output.pos);
+
+finally:
+	PyBuffer_Release(&dest);
+
+	return result;
 }
 
 static PyObject* reader_iter(PyObject* self) {
@@ -315,7 +744,10 b' static PyMethodDef reader_methods[] = {'
 	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read compressed data") },
+	{ "read1", (PyCFunction)reader_read1, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
+	{ "readinto", (PyCFunction)reader_readinto, METH_VARARGS, NULL },
+	{ "readinto1", (PyCFunction)reader_readinto1, METH_VARARGS, NULL },
 	{ "readline", (PyCFunction)reader_readline, METH_VARARGS, PyDoc_STR("Not implemented") },
 	{ "readlines", (PyCFunction)reader_readlines, METH_VARARGS, PyDoc_STR("Not implemented") },
 	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,

contrib/python-zstandard/c-ext/compressionwriter.c

@@ -18,24 +18,23 b' static void ZstdCompressionWriter_deallo'
 	Py_XDECREF(self->compressor);
 	Py_XDECREF(self->writer);
 
+	PyMem_Free(self->output.dst);
+	self->output.dst = NULL;
+
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) {
-	size_t zresult;
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
 
 	if (self->entered) {
 		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
-	zresult = ZSTD_CCtx_setPledgedSrcSize(self->compressor->cctx, self->sourceSize);
-	if (ZSTD_isError(zresult)) {
-		PyErr_Format(ZstdError, "error setting source size: %s",
-			ZSTD_getErrorName(zresult));
-		return NULL;
-	}
-
 	self->entered = 1;
 
 	Py_INCREF(self);
@@ -46,10 +45,6 b' static PyObject* ZstdCompressionWriter_e'
 	PyObject* exc_type;
 	PyObject* exc_value;
 	PyObject* exc_tb;
-	size_t zresult;
-
-	ZSTD_outBuffer output;
-	PyObject* res;
 
 	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
 		return NULL;
@@ -58,46 +53,11 b' static PyObject* ZstdCompressionWriter_e'
 	self->entered = 0;
 
 	if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) {
-		ZSTD_inBuffer inBuffer;
-
-		inBuffer.src = NULL;
-		inBuffer.size = 0;
-		inBuffer.pos = 0;
-
-		output.dst = PyMem_Malloc(self->outSize);
-		if (!output.dst) {
-			return PyErr_NoMemory();
-		}
-		output.size = self->outSize;
-		output.pos = 0;
+		PyObject* result = PyObject_CallMethod((PyObject*)self, "close", NULL);
 
-		while (1) {
-			zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &inBuffer, ZSTD_e_end);
-			if (ZSTD_isError(zresult)) {
-				PyErr_Format(ZstdError, "error ending compression stream: %s",
-					ZSTD_getErrorName(zresult));
-				PyMem_Free(output.dst);
-				return NULL;
-			}
-
-			if (output.pos) {
-#if PY_MAJOR_VERSION >= 3
-				res = PyObject_CallMethod(self->writer, "write", "y#",
-#else
-				res = PyObject_CallMethod(self->writer, "write", "s#",
-#endif
-					output.dst, output.pos);
-				Py_XDECREF(res);
-			}
-
-			if (!zresult) {
-				break;
-			}
-
-			output.pos = 0;
+		if (NULL == result) {
+			return NULL;
 		}
-
-		PyMem_Free(output.dst);
 	}
 
 	Py_RETURN_FALSE;
@@ -117,7 +77,6 b' static PyObject* ZstdCompressionWriter_w'
 	Py_buffer source;
 	size_t zresult;
 	ZSTD_inBuffer input;
-	ZSTD_outBuffer output;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
 
@@ -130,143 +89,240 b' static PyObject* ZstdCompressionWriter_w'
 		return NULL;
 	}
 
-	if (!self->entered) {
-		PyErr_SetString(ZstdError, "compress must be called from an active context manager");
-		goto finally;
-	}
-
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
-	output.dst = PyMem_Malloc(self->outSize);
-	if (!output.dst) {
-		PyErr_NoMemory();
-		goto finally;
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
 	}
-	output.size = self->outSize;
-	output.pos = 0;
+
+	self->output.pos = 0;
 
 	input.src = source.buf;
 	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < source.len) {
+	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &input, ZSTD_e_continue);
+		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output, &input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
-			PyMem_Free(output.dst);
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			goto finally;
 		}
 
 		/* Copy data from output buffer to writer. */
-		if (output.pos) {
+		if (self->output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			res = PyObject_CallMethod(self->writer, "write", "y#",
 #else
 			res = PyObject_CallMethod(self->writer, "write", "s#",
 #endif
-				output.dst, output.pos);
+				self->output.dst, self->output.pos);
 			Py_XDECREF(res);
-			totalWrite += output.pos;
-			self->bytesCompressed += output.pos;
+			totalWrite += self->output.pos;
+			self->bytesCompressed += self->output.pos;
 		}
-		output.pos = 0;
+		self->output.pos = 0;
 	}
 
-	PyMem_Free(output.dst);
-
-	result = PyLong_FromSsize_t(totalWrite);
+	if (self->writeReturnRead) {
+		result = PyLong_FromSize_t(input.pos);
+	}
+	else {
+		result = PyLong_FromSsize_t(totalWrite);
+	}
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
-static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args) {
+static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"flush_mode",
+		NULL
+	};
+
 	size_t zresult;
-	ZSTD_outBuffer output;
 	ZSTD_inBuffer input;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
+	unsigned flush_mode = 0;
+	ZSTD_EndDirective flush;
 
-	if (!self->entered) {
-		PyErr_SetString(ZstdError, "flush must be called from an active context manager");
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|I:flush",
+		kwlist, &flush_mode)) {
 		return NULL;
 	}
 
+	switch (flush_mode) {
+		case 0:
+			flush = ZSTD_e_flush;
+			break;
+		case 1:
+			flush = ZSTD_e_end;
+			break;
+		default:
+			PyErr_Format(PyExc_ValueError, "unknown flush_mode: %d", flush_mode);
+			return NULL;
+	}
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	self->output.pos = 0;
+
 	input.src = NULL;
 	input.size = 0;
 	input.pos = 0;
 
-	output.dst = PyMem_Malloc(self->outSize);
-	if (!output.dst) {
-		return PyErr_NoMemory();
-	}
-	output.size = self->outSize;
-	output.pos = 0;
-
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &input, ZSTD_e_flush);
+		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output, &input, flush);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
-			PyMem_Free(output.dst);
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		/* Copy data from output buffer to writer. */
-		if (output.pos) {
+		if (self->output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			res = PyObject_CallMethod(self->writer, "write", "y#",
 #else
 			res = PyObject_CallMethod(self->writer, "write", "s#",
 #endif
-				output.dst, output.pos);
+				self->output.dst, self->output.pos);
 			Py_XDECREF(res);
-			totalWrite += output.pos;
-			self->bytesCompressed += output.pos;
+			totalWrite += self->output.pos;
+			self->bytesCompressed += self->output.pos;
 		}
 
-		output.pos = 0;
+		self->output.pos = 0;
 
 		if (!zresult) {
 			break;
 		}
 	}
 
-	PyMem_Free(output.dst);
+	return PyLong_FromSsize_t(totalWrite);
+}
+
+static PyObject* ZstdCompressionWriter_close(ZstdCompressionWriter* self) {
+	PyObject* result;
+
+	if (self->closed) {
+		Py_RETURN_NONE;
+	}
+
+	result = PyObject_CallMethod((PyObject*)self, "flush", "I", 1);
+	self->closed = 1;
+
+	if (NULL == result) {
+	    return NULL;
+	}
 
-	return PyLong_FromSsize_t(totalWrite);
+    /* Call close on underlying stream as well. */
+	if (PyObject_HasAttrString(self->writer, "close")) {
+		return PyObject_CallMethod(self->writer, "close", NULL);
+	}
+
+	Py_RETURN_NONE;
+}
+
+static PyObject* ZstdCompressionWriter_fileno(ZstdCompressionWriter* self) {
+	if (PyObject_HasAttrString(self->writer, "fileno")) {
+		return PyObject_CallMethod(self->writer, "fileno", NULL);
+	}
+	else {
+		PyErr_SetString(PyExc_OSError, "fileno not available on underlying writer");
+		return NULL;
+	}
 }
 
 static PyObject* ZstdCompressionWriter_tell(ZstdCompressionWriter* self) {
 	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
 }
 
+static PyObject* ZstdCompressionWriter_writelines(PyObject* self, PyObject* args) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* ZstdCompressionWriter_false(PyObject* self, PyObject* args) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* ZstdCompressionWriter_true(PyObject* self, PyObject* args) {
+	Py_RETURN_TRUE;
+}
+
+static PyObject* ZstdCompressionWriter_unsupported(PyObject* self, PyObject* args, PyObject* kwargs) {
+	PyObject* iomod;
+	PyObject* exc;
+
+	iomod = PyImport_ImportModule("io");
+	if (NULL == iomod) {
+		return NULL;
+	}
+
+	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
+	if (NULL == exc) {
+		Py_DECREF(iomod);
+		return NULL;
+	}
+
+	PyErr_SetNone(exc);
+	Py_DECREF(exc);
+	Py_DECREF(iomod);
+
+	return NULL;
+}
+
 static PyMethodDef ZstdCompressionWriter_methods[] = {
 	{ "__enter__", (PyCFunction)ZstdCompressionWriter_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context.") },
 	{ "__exit__", (PyCFunction)ZstdCompressionWriter_exit, METH_VARARGS,
 	PyDoc_STR("Exit a compression context.") },
+	{ "close", (PyCFunction)ZstdCompressionWriter_close, METH_NOARGS, NULL },
+	{ "fileno", (PyCFunction)ZstdCompressionWriter_fileno, METH_NOARGS, NULL },
+	{ "isatty", (PyCFunction)ZstdCompressionWriter_false, METH_NOARGS, NULL },
+	{ "readable", (PyCFunction)ZstdCompressionWriter_false, METH_NOARGS, NULL },
+	{ "readline", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readlines", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "seek", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "seekable", ZstdCompressionWriter_false, METH_NOARGS, NULL },
+	{ "truncate", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "writable", ZstdCompressionWriter_true, METH_NOARGS, NULL },
+	{ "writelines", ZstdCompressionWriter_writelines, METH_VARARGS, NULL },
+	{ "read", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readall", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readinto", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "memory_size", (PyCFunction)ZstdCompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size of the underlying compressor") },
 	{ "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
-	{ "flush", (PyCFunction)ZstdCompressionWriter_flush, METH_NOARGS,
+	{ "flush", (PyCFunction)ZstdCompressionWriter_flush, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Flush data and finish a zstd frame") },
 	{ "tell", (PyCFunction)ZstdCompressionWriter_tell, METH_NOARGS,
 	PyDoc_STR("Returns current number of bytes compressed") },
 	{ NULL, NULL }
 };
 
+static PyMemberDef ZstdCompressionWriter_members[] = {
+	 { "closed", T_BOOL, offsetof(ZstdCompressionWriter, closed), READONLY, NULL },
+	 { NULL }
+};
+
 PyTypeObject ZstdCompressionWriterType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionWriter",  /* tp_name */
@@ -296,7 +352,7 b' PyTypeObject ZstdCompressionWriterType ='
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdCompressionWriter_methods,  /* tp_methods */
-	0,                              /* tp_members */
+	ZstdCompressionWriter_members,  /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */

contrib/python-zstandard/c-ext/compressobj.c

@@ -59,9 +59,9 b' static PyObject* ZstdCompressionObj_comp'
 	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < source.len) {
+	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+			zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 				&input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
@@ -154,7 +154,7 b' static PyObject* ZstdCompressionObj_flus'
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&input, zFlushMode);
 		Py_END_ALLOW_THREADS
 

contrib/python-zstandard/c-ext/compressor.c

@@ -204,27 +204,27 b' static int ZstdCompressor_init(ZstdCompr'
 		}
 	}
 	else {
-		if (set_parameter(self->params, ZSTD_p_compressionLevel, level)) {
+		if (set_parameter(self->params, ZSTD_c_compressionLevel, level)) {
 			return -1;
 		}
 
-		if (set_parameter(self->params, ZSTD_p_contentSizeFlag,
+		if (set_parameter(self->params, ZSTD_c_contentSizeFlag,
 			writeContentSize ? PyObject_IsTrue(writeContentSize) : 1)) {
 			return -1;
 		}
 
-		if (set_parameter(self->params, ZSTD_p_checksumFlag,
+		if (set_parameter(self->params, ZSTD_c_checksumFlag,
 			writeChecksum ? PyObject_IsTrue(writeChecksum) : 0)) {
 			return -1;
 		}
 
-		if (set_parameter(self->params, ZSTD_p_dictIDFlag,
+		if (set_parameter(self->params, ZSTD_c_dictIDFlag,
 			writeDictID ? PyObject_IsTrue(writeDictID) : 1)) {
 			return -1;
 		}
 
 		if (threads) {
-			if (set_parameter(self->params, ZSTD_p_nbWorkers, threads)) {
+			if (set_parameter(self->params, ZSTD_c_nbWorkers, threads)) {
 				return -1;
 			}
 		}
@@ -344,7 +344,7 b' static PyObject* ZstdCompressor_copy_str'
 		return NULL;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
@@ -391,7 +391,7 b' static PyObject* ZstdCompressor_copy_str'
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_compress_generic(self->cctx, &output, &input, ZSTD_e_continue);
+			zresult = ZSTD_compressStream2(self->cctx, &output, &input, ZSTD_e_continue);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
@@ -421,7 +421,7 b' static PyObject* ZstdCompressor_copy_str'
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->cctx, &output, &input, ZSTD_e_end);
+		zresult = ZSTD_compressStream2(self->cctx, &output, &input, ZSTD_e_end);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
@@ -517,7 +517,7 b' static ZstdCompressionReader* ZstdCompre'
 		goto except;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
@@ -577,7 +577,7 b' static PyObject* ZstdCompressor_compress'
 		goto finally;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	destSize = ZSTD_compressBound(source.len);
 	output = PyBytes_FromStringAndSize(NULL, destSize);
@@ -605,7 +605,7 b' static PyObject* ZstdCompressor_compress'
 	/* By avoiding ZSTD_compress(), we don't necessarily write out content
 		size. This means the argument to ZstdCompressor to control frame
 		parameters is honored. */
-	zresult = ZSTD_compress_generic(self->cctx, &outBuffer, &inBuffer, ZSTD_e_end);
+	zresult = ZSTD_compressStream2(self->cctx, &outBuffer, &inBuffer, ZSTD_e_end);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
@@ -651,7 +651,7 b' static ZstdCompressionObj* ZstdCompresso'
 		return NULL;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, inSize);
 	if (ZSTD_isError(zresult)) {
@@ -740,7 +740,7 b' static ZstdCompressorIterator* ZstdCompr'
 		goto except;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
@@ -794,16 +794,19 b' static ZstdCompressionWriter* ZstdCompre'
 		"writer",
 		"size",
 		"write_size",
+		"write_return_read",
 		NULL
 	};
 
 	PyObject* writer;
 	ZstdCompressionWriter* result;
+	size_t zresult;
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t outSize = ZSTD_CStreamOutSize();
+	PyObject* writeReturnRead = NULL;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kk:stream_writer", kwlist,
-		&writer, &sourceSize, &outSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|KkO:stream_writer", kwlist,
+		&writer, &sourceSize, &outSize, &writeReturnRead)) {
 		return NULL;
 	}
 
@@ -812,22 +815,38 b' static ZstdCompressionWriter* ZstdCompre'
 		return NULL;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
+
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
+	}
 
 	result = (ZstdCompressionWriter*)PyObject_CallObject((PyObject*)&ZstdCompressionWriterType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
+	result->output.dst = PyMem_Malloc(outSize);
+	if (!result->output.dst) {
+		Py_DECREF(result);
+		return (ZstdCompressionWriter*)PyErr_NoMemory();
+	}
+
+	result->output.pos = 0;
+	result->output.size = outSize;
+
 	result->compressor = self;
 	Py_INCREF(result->compressor);
 
 	result->writer = writer;
 	Py_INCREF(result->writer);
 
-	result->sourceSize = sourceSize;
 	result->outSize = outSize;
 	result->bytesCompressed = 0;
+	result->writeReturnRead = writeReturnRead ? PyObject_IsTrue(writeReturnRead) : 0;
 
 	return result;
 }
@@ -853,7 +872,7 b' static ZstdCompressionChunker* ZstdCompr'
 		return NULL;
 	}
 
-	ZSTD_CCtx_reset(self->cctx);
+	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
@@ -1115,7 +1134,7 b' static void compress_worker(WorkerState*'
 			break;
 		}
 
-		zresult = ZSTD_compress_generic(state->cctx, &opOutBuffer, &opInBuffer, ZSTD_e_end);
+		zresult = ZSTD_compressStream2(state->cctx, &opOutBuffer, &opInBuffer, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;

contrib/python-zstandard/c-ext/compressoriterator.c

@@ -57,7 +57,7 b' feedcompressor:'
 	/* If we have data left in the input, consume it. */
 	if (self->input.pos < self->input.size) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&self->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
@@ -127,7 +127,7 b' feedcompressor:'
 		self->input.size = 0;
 		self->input.pos = 0;
 
-		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&self->input, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
@@ -152,7 +152,7 b' feedcompressor:'
 	self->input.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+	zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 		&self->input, ZSTD_e_continue);
 	Py_END_ALLOW_THREADS
 

contrib/python-zstandard/c-ext/constants.c

@@ -32,6 +32,9 b' void constants_module_init(PyObject* mod'
 	ZstdError = PyErr_NewException("zstd.ZstdError", NULL, NULL);
 	PyModule_AddObject(mod, "ZstdError", ZstdError);
 
+	PyModule_AddIntConstant(mod, "FLUSH_BLOCK", 0);
+	PyModule_AddIntConstant(mod, "FLUSH_FRAME", 1);
+
 	PyModule_AddIntConstant(mod, "COMPRESSOBJ_FLUSH_FINISH", compressorobj_flush_finish);
 	PyModule_AddIntConstant(mod, "COMPRESSOBJ_FLUSH_BLOCK", compressorobj_flush_block);
 
@@ -77,8 +80,11 b' void constants_module_init(PyObject* mod'
 	PyModule_AddIntConstant(mod, "HASHLOG3_MAX", ZSTD_HASHLOG3_MAX);
 	PyModule_AddIntConstant(mod, "SEARCHLOG_MIN", ZSTD_SEARCHLOG_MIN);
 	PyModule_AddIntConstant(mod, "SEARCHLOG_MAX", ZSTD_SEARCHLOG_MAX);
-	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MIN", ZSTD_SEARCHLENGTH_MIN);
-	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MAX", ZSTD_SEARCHLENGTH_MAX);
+	PyModule_AddIntConstant(mod, "MINMATCH_MIN", ZSTD_MINMATCH_MIN);
+	PyModule_AddIntConstant(mod, "MINMATCH_MAX", ZSTD_MINMATCH_MAX);
+	/* TODO SEARCHLENGTH_* is deprecated. */
+	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MIN", ZSTD_MINMATCH_MIN);
+	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MAX", ZSTD_MINMATCH_MAX);
 	PyModule_AddIntConstant(mod, "TARGETLENGTH_MIN", ZSTD_TARGETLENGTH_MIN);
 	PyModule_AddIntConstant(mod, "TARGETLENGTH_MAX", ZSTD_TARGETLENGTH_MAX);
 	PyModule_AddIntConstant(mod, "LDM_MINMATCH_MIN", ZSTD_LDM_MINMATCH_MIN);
@@ -93,6 +99,7 b' void constants_module_init(PyObject* mod'
 	PyModule_AddIntConstant(mod, "STRATEGY_BTLAZY2", ZSTD_btlazy2);
 	PyModule_AddIntConstant(mod, "STRATEGY_BTOPT", ZSTD_btopt);
 	PyModule_AddIntConstant(mod, "STRATEGY_BTULTRA", ZSTD_btultra);
+	PyModule_AddIntConstant(mod, "STRATEGY_BTULTRA2", ZSTD_btultra2);
 
 	PyModule_AddIntConstant(mod, "DICT_TYPE_AUTO", ZSTD_dct_auto);
 	PyModule_AddIntConstant(mod, "DICT_TYPE_RAWCONTENT", ZSTD_dct_rawContent);

contrib/python-zstandard/c-ext/decompressionreader.c

@@ -102,6 +102,114 b' static PyObject* reader_isatty(PyObject*'
 	Py_RETURN_FALSE;
 }
 
+/**
+ * Read available input.
+ *
+ * Returns 0 if no data was added to input.
+ * Returns 1 if new input data is available.
+ * Returns -1 on error and sets a Python exception as a side-effect.
+ */
+int read_decompressor_input(ZstdDecompressionReader* self) {
+	if (self->finishedInput) {
+		return 0;
+	}
+
+	if (self->input.pos != self->input.size) {
+		return 0;
+	}
+
+	if (self->reader) {
+        Py_buffer buffer;
+
+        assert(self->readResult == NULL);
+        self->readResult = PyObject_CallMethod(self->reader, "read",
+            "k", self->readSize);
+        if (NULL == self->readResult) {
+            return -1;
+        }
+
+        memset(&buffer, 0, sizeof(buffer));
+
+        if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
+            return -1;
+        }
+
+        /* EOF */
+        if (0 == buffer.len) {
+            self->finishedInput = 1;
+            Py_CLEAR(self->readResult);
+        }
+        else {
+            self->input.src = buffer.buf;
+            self->input.size = buffer.len;
+            self->input.pos = 0;
+        }
+
+        PyBuffer_Release(&buffer);
+	}
+	else {
+		assert(self->buffer.buf);
+        /*
+         * We should only get here once since expectation is we always
+         * exhaust input buffer before reading again.
+         */
+        assert(self->input.src == NULL);
+
+		self->input.src = self->buffer.buf;
+        self->input.size = self->buffer.len;
+        self->input.pos = 0;
+	}
+
+	return 1;
+}
+
+/**
+ * Decompresses available input into an output buffer.
+ *
+ * Returns 0 if we need more input.
+ * Returns 1 if output buffer should be emitted.
+ * Returns -1 on error and sets a Python exception.
+ */
+int decompress_input(ZstdDecompressionReader* self, ZSTD_outBuffer* output) {
+	size_t zresult;
+
+	if (self->input.pos >= self->input.size) {
+		return 0;
+	}
+
+	Py_BEGIN_ALLOW_THREADS
+	zresult = ZSTD_decompressStream(self->decompressor->dctx, output, &self->input);
+	Py_END_ALLOW_THREADS
+
+	/* Input exhausted. Clear our state tracking. */
+	if (self->input.pos == self->input.size) {
+		memset(&self->input, 0, sizeof(self->input));
+		Py_CLEAR(self->readResult);
+
+		if (self->buffer.buf) {
+			self->finishedInput = 1;
+		}
+	}
+
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "zstd decompress error: %s", ZSTD_getErrorName(zresult));
+		return -1;
+	}
+
+	/* We fulfilled the full read request. Signal to emit. */
+	if (output->pos && output->pos == output->size) {
+		return 1;
+	}
+	/* We're at the end of a frame and we aren't allowed to return data
+	   spanning frames. */
+	else if (output->pos && zresult == 0 && !self->readAcrossFrames) {
+		return 1;
+	}
+
+	/* There is more room in the output. Signal to collect more data. */
+	return 0;
+}
+
 static PyObject* reader_read(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
@@ -113,26 +221,30 b' static PyObject* reader_read(ZstdDecompr'
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
-	size_t zresult;
+	int decompressResult, readResult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
-	if (self->finishedOutput) {
-		return PyBytes_FromStringAndSize("", 0);
-	}
-
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "n", kwlist, &size)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
-	if (size < 1) {
-		PyErr_SetString(PyExc_ValueError, "cannot read negative or size 0 amounts");
+	if (size < -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
+	if (size == -1) {
+		return PyObject_CallMethod((PyObject*)self, "readall", NULL);
+	}
+
+	if (self->finishedOutput || size == 0) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
@@ -146,85 +258,38 b' static PyObject* reader_read(ZstdDecompr'
 
 readinput:
 
-	/* Consume input data left over from last time. */
-	if (self->input.pos < self->input.size) {
-		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_decompress_generic(self->decompressor->dctx,
-			&output, &self->input);
-		Py_END_ALLOW_THREADS
+	decompressResult = decompress_input(self, &output);
 
-		/* Input exhausted. Clear our state tracking. */
-		if (self->input.pos == self->input.size) {
-			memset(&self->input, 0, sizeof(self->input));
-			Py_CLEAR(self->readResult);
+	if (-1 == decompressResult) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+	else if (0 == decompressResult) { }
+	else if (1 == decompressResult) {
+		self->bytesDecompressed += output.pos;
 
-			if (self->buffer.buf) {
-				self->finishedInput = 1;
+		if (output.pos != output.size) {
+			if (safe_pybytes_resize(&result, output.pos)) {
+				Py_XDECREF(result);
+				return NULL;
 			}
 		}
-
-		if (ZSTD_isError(zresult)) {
-			PyErr_Format(ZstdError, "zstd decompress error: %s", ZSTD_getErrorName(zresult));
-			return NULL;
-		}
-		else if (0 == zresult) {
-			self->finishedOutput = 1;
-		}
-
-		/* We fulfilled the full read request. Emit it. */
-		if (output.pos && output.pos == output.size) {
-			self->bytesDecompressed += output.size;
-			return result;
-		}
-
-		/*
-		 * There is more room in the output. Fall through to try to collect
-		 * more data so we can try to fill the output.
-		 */
+		return result;
+	}
+	else {
+		assert(0);
 	}
 
-	if (!self->finishedInput) {
-		if (self->reader) {
-			Py_buffer buffer;
-
-			assert(self->readResult == NULL);
-			self->readResult = PyObject_CallMethod(self->reader, "read",
-				"k", self->readSize);
-			if (NULL == self->readResult) {
-				return NULL;
-			}
-
-			memset(&buffer, 0, sizeof(buffer));
-
-			if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
-				return NULL;
-			}
+	readResult = read_decompressor_input(self);
 
-			/* EOF */
-			if (0 == buffer.len) {
-				self->finishedInput = 1;
-				Py_CLEAR(self->readResult);
-			}
-			else {
-				self->input.src = buffer.buf;
-				self->input.size = buffer.len;
-				self->input.pos = 0;
-			}
-
-			PyBuffer_Release(&buffer);
-		}
-		else {
-			assert(self->buffer.buf);
-			/*
-			 * We should only get here once since above block will exhaust
-			 * source buffer until finishedInput is set.
-			 */
-			assert(self->input.src == NULL);
-
-			self->input.src = self->buffer.buf;
-			self->input.size = self->buffer.len;
-			self->input.pos = 0;
-		}
+	if (-1 == readResult) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+	else if (0 == readResult) {}
+	else if (1 == readResult) {}
+	else {
+		assert(0);
 	}
 
 	if (self->input.size) {
@@ -242,18 +307,288 b' readinput:'
 	return result;
 }
 
+static PyObject* reader_read1(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"size",
+		NULL
+	};
+
+	Py_ssize_t size = -1;
+	PyObject* result = NULL;
+	char* resultBuffer;
+	Py_ssize_t resultSize;
+	ZSTD_outBuffer output;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
+		return NULL;
+	}
+
+	if (size < -1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
+		return NULL;
+	}
+
+	if (self->finishedOutput || size == 0) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
+	if (size == -1) {
+		size = ZSTD_DStreamOutSize();
+	}
+
+	result = PyBytes_FromStringAndSize(NULL, size);
+	if (NULL == result) {
+		return NULL;
+	}
+
+	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
+
+	output.dst = resultBuffer;
+	output.size = resultSize;
+	output.pos = 0;
+
+	/* read1() is supposed to use at most 1 read() from the underlying stream.
+	 * However, we can't satisfy this requirement with decompression due to the
+	 * nature of how decompression works. Our strategy is to read + decompress
+	 * until we get any output, at which point we return. This satisfies the
+	 * intent of the read1() API to limit read operations.
+	 */
+	while (!self->finishedInput) {
+		int readResult, decompressResult;
+
+		readResult = read_decompressor_input(self);
+		if (-1 == readResult) {
+			Py_XDECREF(result);
+			return NULL;
+		}
+		else if (0 == readResult || 1 == readResult) { }
+		else {
+			assert(0);
+		}
+
+		decompressResult = decompress_input(self, &output);
+
+		if (-1 == decompressResult) {
+			Py_XDECREF(result);
+			return NULL;
+		}
+		else if (0 == decompressResult || 1 == decompressResult) { }
+		else {
+			assert(0);
+		}
+
+		if (output.pos) {
+		    break;
+		}
+	}
+
+	self->bytesDecompressed += output.pos;
+	if (safe_pybytes_resize(&result, output.pos)) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+
+	return result;
+}
+
+static PyObject* reader_readinto(ZstdDecompressionReader* self, PyObject* args) {
+	Py_buffer dest;
+	ZSTD_outBuffer output;
+	int decompressResult, readResult;
+	PyObject* result = NULL;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyLong_FromLong(0);
+	}
+
+	if (!PyArg_ParseTuple(args, "w*:readinto", &dest)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"destination buffer should be contiguous and have at most one dimension");
+	    goto finally;
+	}
+
+	output.dst = dest.buf;
+	output.size = dest.len;
+	output.pos = 0;
+
+readinput:
+
+	decompressResult = decompress_input(self, &output);
+
+	if (-1 == decompressResult) {
+		goto finally;
+	}
+	else if (0 == decompressResult) { }
+	else if (1 == decompressResult) {
+		self->bytesDecompressed += output.pos;
+		result = PyLong_FromSize_t(output.pos);
+		goto finally;
+	}
+	else {
+		assert(0);
+	}
+
+	readResult = read_decompressor_input(self);
+
+	if (-1 == readResult) {
+		goto finally;
+	}
+	else if (0 == readResult) {}
+	else if (1 == readResult) {}
+	else {
+		assert(0);
+	}
+
+	if (self->input.size) {
+		goto readinput;
+	}
+
+	/* EOF */
+	self->bytesDecompressed += output.pos;
+	result = PyLong_FromSize_t(output.pos);
+
+finally:
+	PyBuffer_Release(&dest);
+
+	return result;
+}
+
+static PyObject* reader_readinto1(ZstdDecompressionReader* self, PyObject* args) {
+	Py_buffer dest;
+	ZSTD_outBuffer output;
+	PyObject* result = NULL;
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyLong_FromLong(0);
+	}
+
+	if (!PyArg_ParseTuple(args, "w*:readinto1", &dest)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"destination buffer should be contiguous and have at most one dimension");
+	    goto finally;
+	}
+
+	output.dst = dest.buf;
+	output.size = dest.len;
+	output.pos = 0;
+
+	while (!self->finishedInput && !self->finishedOutput) {
+		int decompressResult, readResult;
+
+		readResult = read_decompressor_input(self);
+
+		if (-1 == readResult) {
+			goto finally;
+		}
+		else if (0 == readResult || 1 == readResult) {}
+		else {
+			assert(0);
+		}
+
+		decompressResult = decompress_input(self, &output);
+
+		if (-1 == decompressResult) {
+			goto finally;
+		}
+		else if (0 == decompressResult || 1 == decompressResult) {}
+		else {
+			assert(0);
+		}
+
+		if (output.pos) {
+			break;
+		}
+	}
+
+	self->bytesDecompressed += output.pos;
+	result = PyLong_FromSize_t(output.pos);
+
+finally:
+	PyBuffer_Release(&dest);
+
+	return result;
+}
+
 static PyObject* reader_readall(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
-	return NULL;
+	PyObject* chunks = NULL;
+	PyObject* empty = NULL;
+	PyObject* result = NULL;
+
+	/* Our strategy is to collect chunks into a list then join all the
+	 * chunks at the end. We could potentially use e.g. an io.BytesIO. But
+	 * this feels simple enough to implement and avoids potentially expensive
+	 * reallocations of large buffers.
+	 */
+	chunks = PyList_New(0);
+	if (NULL == chunks) {
+		return NULL;
+	}
+
+	while (1) {
+		PyObject* chunk = PyObject_CallMethod(self, "read", "i", 1048576);
+		if (NULL == chunk) {
+			Py_DECREF(chunks);
+			return NULL;
+		}
+
+		if (!PyBytes_Size(chunk)) {
+			Py_DECREF(chunk);
+			break;
+		}
+
+		if (PyList_Append(chunks, chunk)) {
+			Py_DECREF(chunk);
+			Py_DECREF(chunks);
+			return NULL;
+		}
+
+		Py_DECREF(chunk);
+	}
+
+	empty = PyBytes_FromStringAndSize("", 0);
+	if (NULL == empty) {
+		Py_DECREF(chunks);
+		return NULL;
+	}
+
+	result = PyObject_CallMethod(empty, "join", "O", chunks);
+
+	Py_DECREF(empty);
+	Py_DECREF(chunks);
+
+	return result;
 }
 
 static PyObject* reader_readline(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
+	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_readlines(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
+	set_unsupported_operation();
 	return NULL;
 }
 
@@ -345,12 +680,12 b' static PyObject* reader_writelines(PyObj'
 }
 
 static PyObject* reader_iter(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
+	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iternext(PyObject* self) {
-	PyErr_SetNone(PyExc_NotImplementedError);
+	set_unsupported_operation();
 	return NULL;
 }
 
@@ -367,6 +702,10 b' static PyMethodDef reader_methods[] = {'
 	PyDoc_STR("Returns True") },
 	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("read compressed data") },
+	{ "read1", (PyCFunction)reader_read1, METH_VARARGS | METH_KEYWORDS,
+	PyDoc_STR("read compressed data") },
+	{ "readinto", (PyCFunction)reader_readinto, METH_VARARGS, NULL },
+	{ "readinto1", (PyCFunction)reader_readinto1, METH_VARARGS, NULL },
 	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readline", (PyCFunction)reader_readline, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readlines", (PyCFunction)reader_readlines, METH_NOARGS, PyDoc_STR("Not implemented") },

contrib/python-zstandard/c-ext/decompressionwriter.c

@@ -22,12 +22,13 b' static void ZstdDecompressionWriter_deal'
 }
 
 static PyObject* ZstdDecompressionWriter_enter(ZstdDecompressionWriter* self) {
-	if (self->entered) {
-		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
-	if (ensure_dctx(self->decompressor, 1)) {
+	if (self->entered) {
+		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
@@ -40,6 +41,10 b' static PyObject* ZstdDecompressionWriter'
 static PyObject* ZstdDecompressionWriter_exit(ZstdDecompressionWriter* self, PyObject* args) {
 	self->entered = 0;
 
+	if (NULL == PyObject_CallMethod((PyObject*)self, "close", NULL)) {
+		return NULL;
+	}
+
 	Py_RETURN_FALSE;
 }
 
@@ -76,9 +81,9 b' static PyObject* ZstdDecompressionWriter'
 		goto finally;
 	}
 
-	if (!self->entered) {
-		PyErr_SetString(ZstdError, "write must be called from an active context manager");
-		goto finally;
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
 	}
 
 	output.dst = PyMem_Malloc(self->outSize);
@@ -93,9 +98,9 b' static PyObject* ZstdDecompressionWriter'
 	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < source.len) {
+	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_decompress_generic(self->decompressor->dctx, &output, &input);
+		zresult = ZSTD_decompressStream(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
@@ -120,13 +125,94 b' static PyObject* ZstdDecompressionWriter'
 
 	PyMem_Free(output.dst);
 
-	result = PyLong_FromSsize_t(totalWrite);
+	if (self->writeReturnRead) {
+		result = PyLong_FromSize_t(input.pos);
+	}
+	else {
+		result = PyLong_FromSsize_t(totalWrite);
+	}
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
+static PyObject* ZstdDecompressionWriter_close(ZstdDecompressionWriter* self) {
+	PyObject* result;
+
+	if (self->closed) {
+		Py_RETURN_NONE;
+	}
+
+	result = PyObject_CallMethod((PyObject*)self, "flush", NULL);
+	self->closed = 1;
+
+	if (NULL == result) {
+		return NULL;
+	}
+
+	/* Call close on underlying stream as well. */
+	if (PyObject_HasAttrString(self->writer, "close")) {
+		return PyObject_CallMethod(self->writer, "close", NULL);
+	}
+
+	Py_RETURN_NONE;
+}
+
+static PyObject* ZstdDecompressionWriter_fileno(ZstdDecompressionWriter* self) {
+	if (PyObject_HasAttrString(self->writer, "fileno")) {
+		return PyObject_CallMethod(self->writer, "fileno", NULL);
+	}
+	else {
+		PyErr_SetString(PyExc_OSError, "fileno not available on underlying writer");
+		return NULL;
+	}
+}
+
+static PyObject* ZstdDecompressionWriter_flush(ZstdDecompressionWriter* self) {
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (PyObject_HasAttrString(self->writer, "flush")) {
+		return PyObject_CallMethod(self->writer, "flush", NULL);
+	}
+	else {
+		Py_RETURN_NONE;
+	}
+}
+
+static PyObject* ZstdDecompressionWriter_false(PyObject* self, PyObject* args) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* ZstdDecompressionWriter_true(PyObject* self, PyObject* args) {
+	Py_RETURN_TRUE;
+}
+
+static PyObject* ZstdDecompressionWriter_unsupported(PyObject* self, PyObject* args, PyObject* kwargs) {
+	PyObject* iomod;
+	PyObject* exc;
+
+	iomod = PyImport_ImportModule("io");
+	if (NULL == iomod) {
+		return NULL;
+	}
+
+	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
+	if (NULL == exc) {
+		Py_DECREF(iomod);
+		return NULL;
+	}
+
+	PyErr_SetNone(exc);
+	Py_DECREF(exc);
+	Py_DECREF(iomod);
+
+	return NULL;
+}
+
 static PyMethodDef ZstdDecompressionWriter_methods[] = {
 	{ "__enter__", (PyCFunction)ZstdDecompressionWriter_enter, METH_NOARGS,
 	PyDoc_STR("Enter a decompression context.") },
@@ -134,11 +220,32 b' static PyMethodDef ZstdDecompressionWrit'
 	PyDoc_STR("Exit a decompression context.") },
 	{ "memory_size", (PyCFunction)ZstdDecompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size in bytes of the underlying decompressor.") },
+	{ "close", (PyCFunction)ZstdDecompressionWriter_close, METH_NOARGS, NULL },
+	{ "fileno", (PyCFunction)ZstdDecompressionWriter_fileno, METH_NOARGS, NULL },
+	{ "flush", (PyCFunction)ZstdDecompressionWriter_flush, METH_NOARGS, NULL },
+	{ "isatty", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
+	{ "readable", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
+	{ "readline", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readlines", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "seek", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "seekable", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
+	{ "tell", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "truncate", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "writable", ZstdDecompressionWriter_true, METH_NOARGS, NULL },
+	{ "writelines" , (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "read", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readall", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
+	{ "readinto", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
 	{ NULL, NULL }
 };
 
+static PyMemberDef ZstdDecompressionWriter_members[] = {
+	{ "closed", T_BOOL, offsetof(ZstdDecompressionWriter, closed), READONLY, NULL },
+	{ NULL }
+};
+
 PyTypeObject ZstdDecompressionWriterType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressionWriter", /* tp_name */
@@ -168,7 +275,7 b' PyTypeObject ZstdDecompressionWriterType'
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdDecompressionWriter_methods,/* tp_methods */
-	0,                              /* tp_members */
+	ZstdDecompressionWriter_members,/* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */

contrib/python-zstandard/c-ext/decompressobj.c

@@ -75,7 +75,7 b' static PyObject* DecompressionObj_decomp'
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_decompress_generic(self->decompressor->dctx, &output, &input);
+		zresult = ZSTD_decompressStream(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
@@ -130,9 +130,26 b' finally:'
 	return result;
 }
 
+static PyObject* DecompressionObj_flush(ZstdDecompressionObj* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"length",
+		NULL
+	};
+
+	PyObject* length = NULL;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O:flush", kwlist, &length)) {
+	return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
 static PyMethodDef DecompressionObj_methods[] = {
 	{ "decompress", (PyCFunction)DecompressionObj_decompress,
 	  METH_VARARGS | METH_KEYWORDS, PyDoc_STR("decompress data") },
+	{ "flush", (PyCFunction)DecompressionObj_flush,
+	  METH_VARARGS | METH_KEYWORDS, PyDoc_STR("no-op") },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/decompressor.c

@@ -17,7 +17,7 b' extern PyObject* ZstdError;'
 int ensure_dctx(ZstdDecompressor* decompressor, int loadDict) {
 	size_t zresult;
 
-	ZSTD_DCtx_reset(decompressor->dctx);
+	ZSTD_DCtx_reset(decompressor->dctx, ZSTD_reset_session_only);
 
 	if (decompressor->maxWindowSize) {
 		zresult = ZSTD_DCtx_setMaxWindowSize(decompressor->dctx, decompressor->maxWindowSize);
@@ -229,7 +229,7 b' static PyObject* Decompressor_copy_strea'
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompress_generic(self->dctx, &output, &input);
+			zresult = ZSTD_decompressStream(self->dctx, &output, &input);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
@@ -379,7 +379,7 b' PyObject* Decompressor_decompress(ZstdDe'
 	inBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
+	zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
@@ -550,28 +550,35 b' finally:'
 }
 
 PyDoc_STRVAR(Decompressor_stream_reader__doc__,
-"stream_reader(source, [read_size=default])\n"
+"stream_reader(source, [read_size=default, [read_across_frames=False]])\n"
 "\n"
 "Obtain an object that behaves like an I/O stream that can be used for\n"
 "reading decompressed output from an object.\n"
 "\n"
 "The source object can be any object with a ``read(size)`` method or that\n"
 "conforms to the buffer protocol.\n"
+"\n"
+"``read_across_frames`` controls the behavior of ``read()`` when the end\n"
+"of a zstd frame is reached. When ``True``, ``read()`` can potentially\n"
+"return data belonging to multiple zstd frames. When ``False``, ``read()``\n"
+"will return when the end of a frame is reached.\n"
 );
 
 static ZstdDecompressionReader* Decompressor_stream_reader(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"source",
 		"read_size",
+		"read_across_frames",
 		NULL
 	};
 
 	PyObject* source;
 	size_t readSize = ZSTD_DStreamInSize();
+	PyObject* readAcrossFrames = NULL;
 	ZstdDecompressionReader* result;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:stream_reader", kwlist,
-		&source, &readSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kO:stream_reader", kwlist,
+		&source, &readSize, &readAcrossFrames)) {
 		return NULL;
 	}
 
@@ -604,6 +611,7 b' static ZstdDecompressionReader* Decompre'
 
 	result->decompressor = self;
 	Py_INCREF(self);
+	result->readAcrossFrames = readAcrossFrames ? PyObject_IsTrue(readAcrossFrames) : 0;
 
 	return result;
 }
@@ -625,15 +633,17 b' static ZstdDecompressionWriter* Decompre'
 	static char* kwlist[] = {
 		"writer",
 		"write_size",
+		"write_return_read",
 		NULL
 	};
 
 	PyObject* writer;
 	size_t outSize = ZSTD_DStreamOutSize();
+	PyObject* writeReturnRead = NULL;
 	ZstdDecompressionWriter* result;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:stream_writer", kwlist,
-		&writer, &outSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kO:stream_writer", kwlist,
+		&writer, &outSize, &writeReturnRead)) {
 		return NULL;
 	}
 
@@ -642,6 +652,10 b' static ZstdDecompressionWriter* Decompre'
 		return NULL;
 	}
 
+	if (ensure_dctx(self, 1)) {
+		return NULL;
+	}
+
 	result = (ZstdDecompressionWriter*)PyObject_CallObject((PyObject*)&ZstdDecompressionWriterType, NULL);
 	if (!result) {
 		return NULL;
@@ -654,6 +668,7 b' static ZstdDecompressionWriter* Decompre'
 	Py_INCREF(result->writer);
 
 	result->outSize = outSize;
+	result->writeReturnRead = writeReturnRead ? PyObject_IsTrue(writeReturnRead) : 0;
 
 	return result;
 }
@@ -756,7 +771,7 b' static PyObject* Decompressor_decompress'
 	inBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
+	zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "could not decompress chunk 0: %s", ZSTD_getErrorName(zresult));
@@ -852,7 +867,7 b' static PyObject* Decompressor_decompress'
 			outBuffer.pos = 0;
 
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
+			zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
@@ -892,7 +907,7 b' static PyObject* Decompressor_decompress'
 			outBuffer.pos = 0;
 
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
+			zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
@@ -1176,7 +1191,7 b' static void decompress_worker(WorkerStat'
 		inBuffer.size = sourceSize;
 		inBuffer.pos = 0;
 
-		zresult = ZSTD_decompress_generic(state->dctx, &outBuffer, &inBuffer);
+		zresult = ZSTD_decompressStream(state->dctx, &outBuffer, &inBuffer);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;

contrib/python-zstandard/c-ext/decompressoriterator.c

@@ -57,7 +57,7 b' static DecompressorIteratorResult read_d'
 	self->output.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_decompress_generic(self->decompressor->dctx, &self->output, &self->input);
+	zresult = ZSTD_decompressStream(self->decompressor->dctx, &self->output, &self->input);
 	Py_END_ALLOW_THREADS
 
 	/* We're done with the pointer. Nullify to prevent anyone from getting a

contrib/python-zstandard/c-ext/python-zstandard.h

@@ -16,7 +16,7 b''
 #include <zdict.h>
 
 /* Remember to change the string in zstandard/__init__ as well */
-#define PYTHON_ZSTANDARD_VERSION "0.10.1"
+#define PYTHON_ZSTANDARD_VERSION "0.11.0"
 
 typedef enum {
 	compressorobj_flush_finish,
@@ -31,27 +31,6 b' typedef enum {'
 typedef struct {
 	PyObject_HEAD
 	ZSTD_CCtx_params* params;
-	unsigned format;
-	int compressionLevel;
-	unsigned windowLog;
-	unsigned hashLog;
-	unsigned chainLog;
-	unsigned searchLog;
-	unsigned minMatch;
-	unsigned targetLength;
-	unsigned compressionStrategy;
-	unsigned contentSizeFlag;
-	unsigned checksumFlag;
-	unsigned dictIDFlag;
-	unsigned threads;
-	unsigned jobSize;
-	unsigned overlapSizeLog;
-	unsigned forceMaxWindow;
-	unsigned enableLongDistanceMatching;
-	unsigned ldmHashLog;
-	unsigned ldmMinMatch;
-	unsigned ldmBucketSizeLog;
-	unsigned ldmHashEveryLog;
 } ZstdCompressionParametersObject;
 
 extern PyTypeObject ZstdCompressionParametersType;
@@ -129,9 +108,11 b' typedef struct {'
 
 	ZstdCompressor* compressor;
 	PyObject* writer;
-	unsigned long long sourceSize;
+	ZSTD_outBuffer output;
 	size_t outSize;
 	int entered;
+	int closed;
+	int writeReturnRead;
 	unsigned long long bytesCompressed;
 } ZstdCompressionWriter;
 
@@ -235,6 +216,8 b' typedef struct {'
 	PyObject* reader;
 	/* Size for read() operations on reader. */
 	size_t readSize;
+	/* Whether a read() can return data spanning multiple zstd frames. */
+	int readAcrossFrames;
 	/* Buffer to read from (if reading from a buffer). */
 	Py_buffer buffer;
 
@@ -267,6 +250,8 b' typedef struct {'
 	PyObject* writer;
 	size_t outSize;
 	int entered;
+	int closed;
+	int writeReturnRead;
 } ZstdDecompressionWriter;
 
 extern PyTypeObject ZstdDecompressionWriterType;
@@ -360,8 +345,9 b' typedef struct {'
 
 extern PyTypeObject ZstdBufferWithSegmentsCollectionType;
 
-int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
+int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj);
+int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams);
 FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs);
 int ensure_ddict(ZstdCompressionDict* dict);
 int ensure_dctx(ZstdDecompressor* decompressor, int loadDict);

contrib/python-zstandard/make_cffi.py

@@ -36,7 +36,9 b" SOURCES = ['zstd/%s' % p for p in ("
     'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
+    'decompress/zstd_ddict.c',
     'decompress/zstd_decompress.c',
+    'decompress/zstd_decompress_block.c',
     'dictBuilder/cover.c',
     'dictBuilder/fastcover.c',
     'dictBuilder/divsufsort.c',

contrib/python-zstandard/setup.py

@@ -5,12 +5,32 b''
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
+from __future__ import print_function
+
+from distutils.version import LooseVersion
 import os
 import sys
 from setuptools import setup
 
+# Need change in 1.10 for ffi.from_buffer() to handle all buffer types
+# (like memoryview).
+# Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
+# garbage collection pitfalls.
+MINIMUM_CFFI_VERSION = '1.11'
+
 try:
     import cffi
+
+    # PyPy (and possibly other distros) have CFFI distributed as part of
+    # them. The install_requires for CFFI below won't work. We need to sniff
+    # out the CFFI version here and reject CFFI if it is too old.
+    cffi_version = LooseVersion(cffi.__version__)
+    if cffi_version < LooseVersion(MINIMUM_CFFI_VERSION):
+        print('CFFI 1.11 or newer required (%s found); '
+              'not building CFFI backend' % cffi_version,
+              file=sys.stderr)
+        cffi = None
+
 except ImportError:
     cffi = None
 
@@ -49,12 +69,7 b' install_requires = []'
 if cffi:
     import make_cffi
     extensions.append(make_cffi.ffi.distutils_extension())
-
-    # Need change in 1.10 for ffi.from_buffer() to handle all buffer types
-    # (like memoryview).
-    # Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
-    # garbage collection pitfalls.
-    install_requires.append('cffi>=1.11')
+    install_requires.append('cffi>=%s' % MINIMUM_CFFI_VERSION)
 
 version = None
 
@@ -88,6 +103,7 b' setup('
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
     ],
     keywords='zstandard zstd compression',
     packages=['zstandard'],

contrib/python-zstandard/setup_zstd.py

@@ -30,7 +30,9 b" zstd_sources = ['zstd/%s' % p for p in ("
     'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
+    'decompress/zstd_ddict.c',
     'decompress/zstd_decompress.c',
+    'decompress/zstd_decompress_block.c',
     'dictBuilder/cover.c',
     'dictBuilder/divsufsort.c',
     'dictBuilder/fastcover.c',

contrib/python-zstandard/tests/common.py

@@ -79,12 +79,37 b' def make_cffi(cls):'
     return cls
 
 
-class OpCountingBytesIO(io.BytesIO):
+class NonClosingBytesIO(io.BytesIO):
+    """BytesIO that saves the underlying buffer on close().
+
+    This allows us to access written data after close().
+    """
     def __init__(self, *args, **kwargs):
+        super(NonClosingBytesIO, self).__init__(*args, **kwargs)
+        self._saved_buffer = None
+
+    def close(self):
+        self._saved_buffer = self.getvalue()
+        return super(NonClosingBytesIO, self).close()
+
+    def getvalue(self):
+        if self.closed:
+            return self._saved_buffer
+        else:
+            return super(NonClosingBytesIO, self).getvalue()
+
+
+class OpCountingBytesIO(NonClosingBytesIO):
+    def __init__(self, *args, **kwargs):
+        self._flush_count = 0
         self._read_count = 0
         self._write_count = 0
         return super(OpCountingBytesIO, self).__init__(*args, **kwargs)
 
+    def flush(self):
+        self._flush_count += 1
+        return super(OpCountingBytesIO, self).flush()
+
     def read(self, *args):
         self._read_count += 1
         return super(OpCountingBytesIO, self).read(*args)
@@ -117,6 +142,13 b' def random_input_data():'
             except OSError:
                 pass
 
+    # Also add some actual random data.
+    _source_files.append(os.urandom(100))
+    _source_files.append(os.urandom(1000))
+    _source_files.append(os.urandom(10000))
+    _source_files.append(os.urandom(100000))
+    _source_files.append(os.urandom(1000000))
+
     return _source_files
 
 
@@ -140,12 +172,14 b' def generate_samples():'
 
 
 if hypothesis:
-    default_settings = hypothesis.settings()
+    default_settings = hypothesis.settings(deadline=10000)
     hypothesis.settings.register_profile('default', default_settings)
 
-    ci_settings = hypothesis.settings(max_examples=2500,
-                                      max_iterations=2500)
+    ci_settings = hypothesis.settings(deadline=20000, max_examples=1000)
     hypothesis.settings.register_profile('ci', ci_settings)
 
+    expensive_settings = hypothesis.settings(deadline=None, max_examples=10000)
+    hypothesis.settings.register_profile('expensive', expensive_settings)
+
     hypothesis.settings.load_profile(
         os.environ.get('HYPOTHESIS_PROFILE', 'default'))

contrib/python-zstandard/tests/test_buffer_util.py

@@ -8,6 +8,9 b" ss = struct.Struct('=QQ')"
 
 class TestBufferWithSegments(unittest.TestCase):
     def test_arguments(self):
+        if not hasattr(zstd, 'BufferWithSegments'):
+            self.skipTest('BufferWithSegments not available')
+
         with self.assertRaises(TypeError):
             zstd.BufferWithSegments()
 
@@ -19,10 +22,16 b' class TestBufferWithSegments(unittest.Te'
             zstd.BufferWithSegments(b'foo', b'\x00\x00')
 
     def test_invalid_offset(self):
+        if not hasattr(zstd, 'BufferWithSegments'):
+            self.skipTest('BufferWithSegments not available')
+
         with self.assertRaisesRegexp(ValueError, 'offset within segments array references memory'):
             zstd.BufferWithSegments(b'foo', ss.pack(0, 4))
 
     def test_invalid_getitem(self):
+        if not hasattr(zstd, 'BufferWithSegments'):
+            self.skipTest('BufferWithSegments not available')
+
         b = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
 
         with self.assertRaisesRegexp(IndexError, 'offset must be non-negative'):
@@ -35,6 +44,9 b' class TestBufferWithSegments(unittest.Te'
             test = b[2]
 
     def test_single(self):
+        if not hasattr(zstd, 'BufferWithSegments'):
+            self.skipTest('BufferWithSegments not available')
+
         b = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
         self.assertEqual(len(b), 1)
         self.assertEqual(b.size, 3)
@@ -45,6 +57,9 b' class TestBufferWithSegments(unittest.Te'
         self.assertEqual(b[0].tobytes(), b'foo')
 
     def test_multiple(self):
+        if not hasattr(zstd, 'BufferWithSegments'):
+            self.skipTest('BufferWithSegments not available')
+
         b = zstd.BufferWithSegments(b'foofooxfooxy', b''.join([ss.pack(0, 3),
                                                                ss.pack(3, 4),
                                                                ss.pack(7, 5)]))
@@ -59,10 +74,16 b' class TestBufferWithSegments(unittest.Te'
 
 class TestBufferWithSegmentsCollection(unittest.TestCase):
     def test_empty_constructor(self):
+        if not hasattr(zstd, 'BufferWithSegmentsCollection'):
+            self.skipTest('BufferWithSegmentsCollection not available')
+
         with self.assertRaisesRegexp(ValueError, 'must pass at least 1 argument'):
             zstd.BufferWithSegmentsCollection()
 
     def test_argument_validation(self):
+        if not hasattr(zstd, 'BufferWithSegmentsCollection'):
+            self.skipTest('BufferWithSegmentsCollection not available')
+
         with self.assertRaisesRegexp(TypeError, 'arguments must be BufferWithSegments'):
             zstd.BufferWithSegmentsCollection(None)
 
@@ -74,6 +95,9 b' class TestBufferWithSegmentsCollection(u'
             zstd.BufferWithSegmentsCollection(zstd.BufferWithSegments(b'', b''))
 
     def test_length(self):
+        if not hasattr(zstd, 'BufferWithSegmentsCollection'):
+            self.skipTest('BufferWithSegmentsCollection not available')
+
         b1 = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
         b2 = zstd.BufferWithSegments(b'barbaz', b''.join([ss.pack(0, 3),
                                                           ss.pack(3, 3)]))
@@ -91,6 +115,9 b' class TestBufferWithSegmentsCollection(u'
         self.assertEqual(c.size(), 9)
 
     def test_getitem(self):
+        if not hasattr(zstd, 'BufferWithSegmentsCollection'):
+            self.skipTest('BufferWithSegmentsCollection not available')
+
         b1 = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
         b2 = zstd.BufferWithSegments(b'barbaz', b''.join([ss.pack(0, 3),
                                                           ss.pack(3, 3)]))

contrib/python-zstandard/tests/test_compressor.py

@@ -1,14 +1,17 b''
 import hashlib
 import io
+import os
 import struct
 import sys
 import tarfile
+import tempfile
 import unittest
 
 import zstandard as zstd
 
 from .common import (
     make_cffi,
+    NonClosingBytesIO,
     OpCountingBytesIO,
 )
 
@@ -272,7 +275,7 b' class TestCompressor_compressobj(unittes'
 
         params = zstd.get_frame_parameters(result)
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
-        self.assertEqual(params.window_size, 1048576)
+        self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
@@ -321,7 +324,7 b' class TestCompressor_compressobj(unittes'
         cobj.compress(b'foo')
         cobj.flush()
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'cannot call compress\(\) after compressor'):
+        with self.assertRaisesRegexp(zstd.ZstdError, r'cannot call compress\(\) after compressor'):
             cobj.compress(b'foo')
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'compressor object already finished'):
@@ -453,7 +456,7 b' class TestCompressor_copy_stream(unittes'
 
         params = zstd.get_frame_parameters(dest.getvalue())
         self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
-        self.assertEqual(params.window_size, 1048576)
+        self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
@@ -605,10 +608,6 b' class TestCompressor_stream_reader(unitt'
             with self.assertRaises(io.UnsupportedOperation):
                 reader.readlines()
 
-            # This could probably be implemented someday.
-            with self.assertRaises(NotImplementedError):
-                reader.readall()
-
             with self.assertRaises(io.UnsupportedOperation):
                 iter(reader)
 
@@ -644,15 +643,16 b' class TestCompressor_stream_reader(unitt'
             with self.assertRaisesRegexp(ValueError, 'stream is closed'):
                 reader.read(10)
 
-    def test_read_bad_size(self):
+    def test_read_sizes(self):
         cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
 
         with cctx.stream_reader(b'foo') as reader:
-            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
-                reader.read(-1)
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative amounts less than -1'):
+                reader.read(-2)
 
-            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
-                reader.read(0)
+            self.assertEqual(reader.read(0), b'')
+            self.assertEqual(reader.read(), foo)
 
     def test_read_buffer(self):
         cctx = zstd.ZstdCompressor()
@@ -746,11 +746,202 b' class TestCompressor_stream_reader(unitt'
         with cctx.stream_reader(source, size=42):
             pass
 
+    def test_readall(self):
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(b'foo' * 1024)
+
+        reader = cctx.stream_reader(b'foo' * 1024)
+        self.assertEqual(reader.readall(), frame)
+
+    def test_readinto(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
+        reader = cctx.stream_reader(b'foo')
+        with self.assertRaises(Exception):
+            reader.readinto(b'foobar')
+
+        # readinto() with sufficiently large destination.
+        b = bytearray(1024)
+        reader = cctx.stream_reader(b'foo')
+        self.assertEqual(reader.readinto(b), len(foo))
+        self.assertEqual(b[0:len(foo)], foo)
+        self.assertEqual(reader.readinto(b), 0)
+        self.assertEqual(b[0:len(foo)], foo)
+
+        # readinto() with small reads.
+        b = bytearray(1024)
+        reader = cctx.stream_reader(b'foo', read_size=1)
+        self.assertEqual(reader.readinto(b), len(foo))
+        self.assertEqual(b[0:len(foo)], foo)
+
+        # Too small destination buffer.
+        b = bytearray(2)
+        reader = cctx.stream_reader(b'foo')
+        self.assertEqual(reader.readinto(b), 2)
+        self.assertEqual(b[:], foo[0:2])
+        self.assertEqual(reader.readinto(b), 2)
+        self.assertEqual(b[:], foo[2:4])
+        self.assertEqual(reader.readinto(b), 2)
+        self.assertEqual(b[:], foo[4:6])
+
+    def test_readinto1(self):
+        cctx = zstd.ZstdCompressor()
+        foo = b''.join(cctx.read_to_iter(io.BytesIO(b'foo')))
+
+        reader = cctx.stream_reader(b'foo')
+        with self.assertRaises(Exception):
+            reader.readinto1(b'foobar')
+
+        b = bytearray(1024)
+        source = OpCountingBytesIO(b'foo')
+        reader = cctx.stream_reader(source)
+        self.assertEqual(reader.readinto1(b), len(foo))
+        self.assertEqual(b[0:len(foo)], foo)
+        self.assertEqual(source._read_count, 2)
+
+        # readinto1() with small reads.
+        b = bytearray(1024)
+        source = OpCountingBytesIO(b'foo')
+        reader = cctx.stream_reader(source, read_size=1)
+        self.assertEqual(reader.readinto1(b), len(foo))
+        self.assertEqual(b[0:len(foo)], foo)
+        self.assertEqual(source._read_count, 4)
+
+    def test_read1(self):
+        cctx = zstd.ZstdCompressor()
+        foo = b''.join(cctx.read_to_iter(io.BytesIO(b'foo')))
+
+        b = OpCountingBytesIO(b'foo')
+        reader = cctx.stream_reader(b)
+
+        self.assertEqual(reader.read1(), foo)
+        self.assertEqual(b._read_count, 2)
+
+        b = OpCountingBytesIO(b'foo')
+        reader = cctx.stream_reader(b)
+
+        self.assertEqual(reader.read1(0), b'')
+        self.assertEqual(reader.read1(2), foo[0:2])
+        self.assertEqual(b._read_count, 2)
+        self.assertEqual(reader.read1(2), foo[2:4])
+        self.assertEqual(reader.read1(1024), foo[4:])
+
 
 @make_cffi
 class TestCompressor_stream_writer(unittest.TestCase):
+    def test_io_api(self):
+        buffer = io.BytesIO()
+        cctx = zstd.ZstdCompressor()
+        writer = cctx.stream_writer(buffer)
+
+        self.assertFalse(writer.isatty())
+        self.assertFalse(writer.readable())
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline(size=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines(hint=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.seek(0)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.seek(10, os.SEEK_SET)
+
+        self.assertFalse(writer.seekable())
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate(size=42)
+
+        self.assertTrue(writer.writable())
+
+        with self.assertRaises(NotImplementedError):
+            writer.writelines([])
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read(size=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readall()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readinto(None)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.fileno()
+
+        self.assertFalse(writer.closed)
+
+    def test_fileno_file(self):
+        with tempfile.TemporaryFile('wb') as tf:
+            cctx = zstd.ZstdCompressor()
+            writer = cctx.stream_writer(tf)
+
+            self.assertEqual(writer.fileno(), tf.fileno())
+
+    def test_close(self):
+        buffer = NonClosingBytesIO()
+        cctx = zstd.ZstdCompressor(level=1)
+        writer = cctx.stream_writer(buffer)
+
+        writer.write(b'foo' * 1024)
+        self.assertFalse(writer.closed)
+        self.assertFalse(buffer.closed)
+        writer.close()
+        self.assertTrue(writer.closed)
+        self.assertTrue(buffer.closed)
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            writer.write(b'foo')
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            writer.flush()
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            with writer:
+                pass
+
+        self.assertEqual(buffer.getvalue(),
+                         b'\x28\xb5\x2f\xfd\x00\x48\x55\x00\x00\x18\x66\x6f'
+                         b'\x6f\x01\x00\xfa\xd3\x77\x43')
+
+        # Context manager exit should close stream.
+        buffer = io.BytesIO()
+        writer = cctx.stream_writer(buffer)
+
+        with writer:
+            writer.write(b'foo')
+
+        self.assertTrue(writer.closed)
+
     def test_empty(self):
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(buffer) as compressor:
             compressor.write(b'')
@@ -764,6 +955,25 b' class TestCompressor_stream_writer(unitt'
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
+        # Test without context manager.
+        buffer = io.BytesIO()
+        compressor = cctx.stream_writer(buffer)
+        self.assertEqual(compressor.write(b''), 0)
+        self.assertEqual(buffer.getvalue(), b'')
+        self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 9)
+        result = buffer.getvalue()
+        self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
+
+        params = zstd.get_frame_parameters(result)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(params.window_size, 524288)
+        self.assertEqual(params.dict_id, 0)
+        self.assertFalse(params.has_checksum)
+
+        # Test write_return_read=True
+        compressor = cctx.stream_writer(buffer, write_return_read=True)
+        self.assertEqual(compressor.write(b''), 0)
+
     def test_input_types(self):
         expected = b'\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f'
         cctx = zstd.ZstdCompressor(level=1)
@@ -778,14 +988,17 b' class TestCompressor_stream_writer(unitt'
         ]
 
         for source in sources:
-            buffer = io.BytesIO()
+            buffer = NonClosingBytesIO()
             with cctx.stream_writer(buffer) as compressor:
                 compressor.write(source)
 
             self.assertEqual(buffer.getvalue(), expected)
 
+            compressor = cctx.stream_writer(buffer, write_return_read=True)
+            self.assertEqual(compressor.write(source), len(source))
+
     def test_multiple_compress(self):
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=5)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
@@ -794,9 +1007,27 b' class TestCompressor_stream_writer(unitt'
 
         result = buffer.getvalue()
         self.assertEqual(result,
-                         b'\x28\xb5\x2f\xfd\x00\x50\x75\x00\x00\x38\x66\x6f'
+                         b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f'
                          b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23')
 
+        # Test without context manager.
+        buffer = io.BytesIO()
+        compressor = cctx.stream_writer(buffer)
+        self.assertEqual(compressor.write(b'foo'), 0)
+        self.assertEqual(compressor.write(b'bar'), 0)
+        self.assertEqual(compressor.write(b'x' * 8192), 0)
+        self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
+        result = buffer.getvalue()
+        self.assertEqual(result,
+                         b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x38\x66\x6f'
+                         b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23')
+
+        # Test with write_return_read=True.
+        compressor = cctx.stream_writer(buffer, write_return_read=True)
+        self.assertEqual(compressor.write(b'foo'), 3)
+        self.assertEqual(compressor.write(b'barbiz'), 6)
+        self.assertEqual(compressor.write(b'x' * 8192), 8192)
+
     def test_dictionary(self):
         samples = []
         for i in range(128):
@@ -807,9 +1038,9 b' class TestCompressor_stream_writer(unitt'
         d = zstd.train_dictionary(8192, samples)
 
         h = hashlib.sha1(d.as_bytes()).hexdigest()
-        self.assertEqual(h, '2b3b6428da5bf2c9cc9d4bb58ba0bc5990dd0e79')
+        self.assertEqual(h, '88ca0d38332aff379d4ced166a51c280a7679aad')
 
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=9, dict_data=d)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
@@ -825,7 +1056,7 b' class TestCompressor_stream_writer(unitt'
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
-        self.assertEqual(h, '23f88344263678478f5f82298e0a5d1833125786')
+        self.assertEqual(h, '8703b4316f274d26697ea5dd480f29c08e85d940')
 
         source = b'foo' + b'bar' + (b'foo' * 16384)
 
@@ -842,9 +1073,9 b' class TestCompressor_stream_writer(unitt'
             min_match=5,
             search_log=4,
             target_length=10,
-            compression_strategy=zstd.STRATEGY_FAST)
+            strategy=zstd.STRATEGY_FAST)
 
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(compression_params=params)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
@@ -863,12 +1094,12 b' class TestCompressor_stream_writer(unitt'
         self.assertEqual(h, '2a8111d72eb5004cdcecbdac37da9f26720d30ef')
 
     def test_write_checksum(self):
-        no_checksum = io.BytesIO()
+        no_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(no_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
 
-        with_checksum = io.BytesIO()
+        with_checksum = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
         with cctx.stream_writer(with_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
@@ -886,12 +1117,12 b' class TestCompressor_stream_writer(unitt'
                          len(no_checksum.getvalue()) + 4)
 
     def test_write_content_size(self):
-        no_size = io.BytesIO()
+        no_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         with cctx.stream_writer(no_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
-        with_size = io.BytesIO()
+        with_size = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(with_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
@@ -902,7 +1133,7 b' class TestCompressor_stream_writer(unitt'
                          len(no_size.getvalue()))
 
         # Declaring size will write the header.
-        with_size = io.BytesIO()
+        with_size = NonClosingBytesIO()
         with cctx.stream_writer(with_size, size=len(b'foobar' * 256)) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
@@ -927,7 +1158,7 b' class TestCompressor_stream_writer(unitt'
 
         d = zstd.train_dictionary(1024, samples)
 
-        with_dict_id = io.BytesIO()
+        with_dict_id = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         with cctx.stream_writer(with_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
@@ -935,7 +1166,7 b' class TestCompressor_stream_writer(unitt'
         self.assertEqual(with_dict_id.getvalue()[4:5], b'\x03')
 
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
-        no_dict_id = io.BytesIO()
+        no_dict_id = NonClosingBytesIO()
         with cctx.stream_writer(no_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
 
@@ -1009,8 +1240,32 b' class TestCompressor_stream_writer(unitt'
         header = trailing[0:3]
         self.assertEqual(header, b'\x01\x00\x00')
 
+    def test_flush_frame(self):
+        cctx = zstd.ZstdCompressor(level=3)
+        dest = OpCountingBytesIO()
+
+        with cctx.stream_writer(dest) as compressor:
+            self.assertEqual(compressor.write(b'foobar' * 8192), 0)
+            self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 23)
+            compressor.write(b'biz' * 16384)
+
+        self.assertEqual(dest.getvalue(),
+                         # Frame 1.
+                         b'\x28\xb5\x2f\xfd\x00\x58\x75\x00\x00\x30\x66\x6f\x6f'
+                         b'\x62\x61\x72\x01\x00\xf7\xbf\xe8\xa5\x08'
+                         # Frame 2.
+                         b'\x28\xb5\x2f\xfd\x00\x58\x5d\x00\x00\x18\x62\x69\x7a'
+                         b'\x01\x00\xfa\x3f\x75\x37\x04')
+
+    def test_bad_flush_mode(self):
+        cctx = zstd.ZstdCompressor()
+        dest = io.BytesIO()
+        with cctx.stream_writer(dest) as compressor:
+            with self.assertRaisesRegexp(ValueError, 'unknown flush_mode: 42'):
+                compressor.flush(flush_mode=42)
+
     def test_multithreaded(self):
-        dest = io.BytesIO()
+        dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(threads=2)
         with cctx.stream_writer(dest) as compressor:
             compressor.write(b'a' * 1048576)
@@ -1043,22 +1298,21 b' class TestCompressor_stream_writer(unitt'
             pass
 
     def test_tarfile_compat(self):
-        raise unittest.SkipTest('not yet fully working')
-
-        dest = io.BytesIO()
+        dest = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor()
         with cctx.stream_writer(dest) as compressor:
-            with tarfile.open('tf', mode='w', fileobj=compressor) as tf:
+            with tarfile.open('tf', mode='w|', fileobj=compressor) as tf:
                 tf.add(__file__, 'test_compressor.py')
 
-        dest.seek(0)
+        dest = io.BytesIO(dest.getvalue())
 
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_reader(dest) as reader:
-            with tarfile.open(mode='r:', fileobj=reader) as tf:
+            with tarfile.open(mode='r|', fileobj=reader) as tf:
                 for member in tf:
                     self.assertEqual(member.name, 'test_compressor.py')
 
+
 @make_cffi
 class TestCompressor_read_to_iter(unittest.TestCase):
     def test_type_validation(self):
@@ -1192,7 +1446,7 b' class TestCompressor_chunker(unittest.Te'
 
         it = chunker.finish()
 
-        self.assertEqual(next(it), b'\x28\xb5\x2f\xfd\x00\x50\x01\x00\x00')
+        self.assertEqual(next(it), b'\x28\xb5\x2f\xfd\x00\x58\x01\x00\x00')
 
         with self.assertRaises(StopIteration):
             next(it)
@@ -1214,7 +1468,7 b' class TestCompressor_chunker(unittest.Te'
         it = chunker.finish()
 
         self.assertEqual(next(it),
-                         b'\x28\xb5\x2f\xfd\x00\x50\x7d\x00\x00\x48\x66\x6f'
+                         b'\x28\xb5\x2f\xfd\x00\x58\x7d\x00\x00\x48\x66\x6f'
                          b'\x6f\x62\x61\x72\x62\x61\x7a\x01\x00\xe4\xe4\x8e')
 
         with self.assertRaises(StopIteration):
@@ -1258,7 +1512,7 b' class TestCompressor_chunker(unittest.Te'
 
         self.assertEqual(
             b''.join(chunks),
-            b'\x28\xb5\x2f\xfd\x00\x50\x55\x00\x00\x18\x66\x6f\x6f\x01\x00'
+            b'\x28\xb5\x2f\xfd\x00\x58\x55\x00\x00\x18\x66\x6f\x6f\x01\x00'
             b'\xfa\xd3\x77\x43')
 
         dctx = zstd.ZstdDecompressor()
@@ -1283,7 +1537,7 b' class TestCompressor_chunker(unittest.Te'
 
             self.assertEqual(list(chunker.compress(source)), [])
             self.assertEqual(list(chunker.finish()), [
-                b'\x28\xb5\x2f\xfd\x00\x50\x19\x00\x00\x66\x6f\x6f'
+                b'\x28\xb5\x2f\xfd\x00\x58\x19\x00\x00\x66\x6f\x6f'
             ])
 
     def test_flush(self):
@@ -1296,7 +1550,7 b' class TestCompressor_chunker(unittest.Te'
         chunks1 = list(chunker.flush())
 
         self.assertEqual(chunks1, [
-            b'\x28\xb5\x2f\xfd\x00\x50\x8c\x00\x00\x30\x66\x6f\x6f\x62\x61\x72'
+            b'\x28\xb5\x2f\xfd\x00\x58\x8c\x00\x00\x30\x66\x6f\x6f\x62\x61\x72'
             b'\x02\x00\xfa\x03\xfe\xd0\x9f\xbe\x1b\x02'
         ])
 
@@ -1326,7 +1580,7 b' class TestCompressor_chunker(unittest.Te'
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
-                'cannot call compress\(\) after compression finished'):
+                r'cannot call compress\(\) after compression finished'):
             list(chunker.compress(b'foo'))
 
     def test_flush_after_finish(self):
@@ -1338,7 +1592,7 b' class TestCompressor_chunker(unittest.Te'
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
-                'cannot call flush\(\) after compression finished'):
+                r'cannot call flush\(\) after compression finished'):
             list(chunker.flush())
 
     def test_finish_after_finish(self):
@@ -1350,7 +1604,7 b' class TestCompressor_chunker(unittest.Te'
 
         with self.assertRaisesRegexp(
                 zstd.ZstdError,
-                'cannot call finish\(\) after compression finished'):
+                r'cannot call finish\(\) after compression finished'):
             list(chunker.finish())
 
 
@@ -1358,6 +1612,9 b' class TestCompressor_multi_compress_to_b'
     def test_invalid_inputs(self):
         cctx = zstd.ZstdCompressor()
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         with self.assertRaises(TypeError):
             cctx.multi_compress_to_buffer(True)
 
@@ -1370,6 +1627,9 b' class TestCompressor_multi_compress_to_b'
     def test_empty_input(self):
         cctx = zstd.ZstdCompressor()
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         with self.assertRaisesRegexp(ValueError, 'no source elements found'):
             cctx.multi_compress_to_buffer([])
 
@@ -1379,6 +1639,9 b' class TestCompressor_multi_compress_to_b'
     def test_list_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         original = [b'foo' * 12, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
         b = cctx.multi_compress_to_buffer(original)
@@ -1394,6 +1657,9 b' class TestCompressor_multi_compress_to_b'
     def test_buffer_with_segments_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         original = [b'foo' * 4, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
 
@@ -1412,6 +1678,9 b' class TestCompressor_multi_compress_to_b'
     def test_buffer_with_segments_collection_input(self):
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         original = [
             b'foo1',
             b'foo2' * 2,
@@ -1449,6 +1718,9 b' class TestCompressor_multi_compress_to_b'
 
         cctx = zstd.ZstdCompressor(write_checksum=True)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         frames = []
         frames.extend(b'x' * 64 for i in range(256))
         frames.extend(b'y' * 64 for i in range(256))

contrib/python-zstandard/tests/test_compressor_fuzzing.py

@@ -12,6 +12,7 b' import zstandard as zstd'
 
 from . common import (
     make_cffi,
+    NonClosingBytesIO,
     random_input_data,
 )
 
@@ -19,6 +20,62 b' from . common import ('
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
 class TestCompressor_stream_reader_fuzzing(unittest.TestCase):
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_stream_source_read(self, original, level, source_read_size,
+                                read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                chunk = reader.read(read_size)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_buffer_source_read(self, original, level, source_read_size,
+                                read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                chunk = reader.read(read_size)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
                       source_read_size=strategies.integers(1, 16384),
@@ -33,15 +90,17 b' class TestCompressor_stream_reader_fuzzi'
                                 read_size=source_read_size) as reader:
             chunks = []
             while True:
-                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                read_size = read_sizes.draw(strategies.integers(-1, 16384))
                 chunk = reader.read(read_size)
+                if not chunk and read_size:
+                    break
 
-                if not chunk:
-                    break
                 chunks.append(chunk)
 
         self.assertEqual(b''.join(chunks), ref_frame)
 
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
                       source_read_size=strategies.integers(1, 16384),
@@ -57,14 +116,343 b' class TestCompressor_stream_reader_fuzzi'
                                 read_size=source_read_size) as reader:
             chunks = []
             while True:
+                read_size = read_sizes.draw(strategies.integers(-1, 16384))
+                chunk = reader.read(read_size)
+                if not chunk and read_size:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_stream_source_readinto(self, original, level,
+                                    source_read_size, read_size):
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                b = bytearray(read_size)
+                count = reader.readinto(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_buffer_source_readinto(self, original, level,
+                                    source_read_size, read_size):
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                b = bytearray(read_size)
+                count = reader.readinto(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_stream_source_readinto_variance(self, original, level,
+                                             source_read_size, read_sizes):
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
                 read_size = read_sizes.draw(strategies.integers(1, 16384))
-                chunk = reader.read(read_size)
+                b = bytearray(read_size)
+                count = reader.readinto(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_buffer_source_readinto_variance(self, original, level,
+                                             source_read_size, read_sizes):
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                b = bytearray(read_size)
+                count = reader.readinto(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_stream_source_read1(self, original, level, source_read_size,
+                                 read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                chunk = reader.read1(read_size)
                 if not chunk:
                     break
+
                 chunks.append(chunk)
 
         self.assertEqual(b''.join(chunks), ref_frame)
 
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(-1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_buffer_source_read1(self, original, level, source_read_size,
+                                 read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                chunk = reader.read1(read_size)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_stream_source_read1_variance(self, original, level, source_read_size,
+                                          read_sizes):
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(-1, 16384))
+                chunk = reader.read1(read_size)
+                if not chunk and read_size:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_buffer_source_read1_variance(self, original, level, source_read_size,
+                                          read_sizes):
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(-1, 16384))
+                chunk = reader.read1(read_size)
+                if not chunk and read_size:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_stream_source_readinto1(self, original, level, source_read_size,
+                                     read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                b = bytearray(read_size)
+                count = reader.readinto1(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_size=strategies.integers(1, zstd.COMPRESSION_RECOMMENDED_OUTPUT_SIZE))
+    def test_buffer_source_readinto1(self, original, level, source_read_size,
+                                     read_size):
+        if read_size == 0:
+            read_size = -1
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                b = bytearray(read_size)
+                count = reader.readinto1(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_stream_source_readinto1_variance(self, original, level, source_read_size,
+                                              read_sizes):
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                b = bytearray(read_size)
+                count = reader.readinto1(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_buffer_source_readinto1_variance(self, original, level, source_read_size,
+                                              read_sizes):
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                b = bytearray(read_size)
+                count = reader.readinto1(b)
+
+                if not count:
+                    break
+
+                chunks.append(bytes(b[0:count]))
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
@@ -77,7 +465,7 b' class TestCompressor_stream_writer_fuzzi'
         ref_frame = refctx.compress(original)
 
         cctx = zstd.ZstdCompressor(level=level)
-        b = io.BytesIO()
+        b = NonClosingBytesIO()
         with cctx.stream_writer(b, size=len(original), write_size=write_size) as compressor:
             compressor.write(original)
 
@@ -219,6 +607,9 b' class TestCompressor_multi_compress_to_b'
                                    write_checksum=True,
                                    **kwargs)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         result = cctx.multi_compress_to_buffer(original, threads=-1)
 
         self.assertEqual(len(result), len(original))

contrib/python-zstandard/tests/test_data_structures.py

@@ -15,17 +15,17 b' class TestCompressionParameters(unittest'
                                        chain_log=zstd.CHAINLOG_MIN,
                                        hash_log=zstd.HASHLOG_MIN,
                                        search_log=zstd.SEARCHLOG_MIN,
-                                       min_match=zstd.SEARCHLENGTH_MIN + 1,
+                                       min_match=zstd.MINMATCH_MIN + 1,
                                        target_length=zstd.TARGETLENGTH_MIN,
-                                       compression_strategy=zstd.STRATEGY_FAST)
+                                       strategy=zstd.STRATEGY_FAST)
 
         zstd.ZstdCompressionParameters(window_log=zstd.WINDOWLOG_MAX,
                                        chain_log=zstd.CHAINLOG_MAX,
                                        hash_log=zstd.HASHLOG_MAX,
                                        search_log=zstd.SEARCHLOG_MAX,
-                                       min_match=zstd.SEARCHLENGTH_MAX - 1,
+                                       min_match=zstd.MINMATCH_MAX - 1,
                                        target_length=zstd.TARGETLENGTH_MAX,
-                                       compression_strategy=zstd.STRATEGY_BTULTRA)
+                                       strategy=zstd.STRATEGY_BTULTRA2)
 
     def test_from_level(self):
         p = zstd.ZstdCompressionParameters.from_level(1)
@@ -43,7 +43,7 b' class TestCompressionParameters(unittest'
                                            search_log=4,
                                            min_match=5,
                                            target_length=8,
-                                           compression_strategy=1)
+                                           strategy=1)
         self.assertEqual(p.window_log, 10)
         self.assertEqual(p.chain_log, 6)
         self.assertEqual(p.hash_log, 7)
@@ -59,9 +59,10 b' class TestCompressionParameters(unittest'
         self.assertEqual(p.threads, 4)
 
         p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576,
-                                       overlap_size_log=6)
+                                           overlap_log=6)
         self.assertEqual(p.threads, 2)
         self.assertEqual(p.job_size, 1048576)
+        self.assertEqual(p.overlap_log, 6)
         self.assertEqual(p.overlap_size_log, 6)
 
         p = zstd.ZstdCompressionParameters(compression_level=-1)
@@ -85,8 +86,9 b' class TestCompressionParameters(unittest'
         p = zstd.ZstdCompressionParameters(ldm_bucket_size_log=7)
         self.assertEqual(p.ldm_bucket_size_log, 7)
 
-        p = zstd.ZstdCompressionParameters(ldm_hash_every_log=8)
+        p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
         self.assertEqual(p.ldm_hash_every_log, 8)
+        self.assertEqual(p.ldm_hash_rate_log, 8)
 
     def test_estimated_compression_context_size(self):
         p = zstd.ZstdCompressionParameters(window_log=20,
@@ -95,12 +97,44 b' class TestCompressionParameters(unittest'
                                            search_log=1,
                                            min_match=5,
                                            target_length=16,
-                                           compression_strategy=zstd.STRATEGY_DFAST)
+                                           strategy=zstd.STRATEGY_DFAST)
 
         # 32-bit has slightly different values from 64-bit.
         self.assertAlmostEqual(p.estimated_compression_context_size(), 1294072,
                                delta=250)
 
+    def test_strategy(self):
+        with self.assertRaisesRegexp(ValueError, 'cannot specify both compression_strategy'):
+            zstd.ZstdCompressionParameters(strategy=0, compression_strategy=0)
+
+        p = zstd.ZstdCompressionParameters(strategy=2)
+        self.assertEqual(p.compression_strategy, 2)
+
+        p = zstd.ZstdCompressionParameters(strategy=3)
+        self.assertEqual(p.compression_strategy, 3)
+
+    def test_ldm_hash_rate_log(self):
+        with self.assertRaisesRegexp(ValueError, 'cannot specify both ldm_hash_rate_log'):
+            zstd.ZstdCompressionParameters(ldm_hash_rate_log=8, ldm_hash_every_log=4)
+
+        p = zstd.ZstdCompressionParameters(ldm_hash_rate_log=8)
+        self.assertEqual(p.ldm_hash_every_log, 8)
+
+        p = zstd.ZstdCompressionParameters(ldm_hash_every_log=16)
+        self.assertEqual(p.ldm_hash_every_log, 16)
+
+    def test_overlap_log(self):
+        with self.assertRaisesRegexp(ValueError, 'cannot specify both overlap_log'):
+            zstd.ZstdCompressionParameters(overlap_log=1, overlap_size_log=9)
+
+        p = zstd.ZstdCompressionParameters(overlap_log=2)
+        self.assertEqual(p.overlap_log, 2)
+        self.assertEqual(p.overlap_size_log, 2)
+
+        p = zstd.ZstdCompressionParameters(overlap_size_log=4)
+        self.assertEqual(p.overlap_log, 4)
+        self.assertEqual(p.overlap_size_log, 4)
+
 
 @make_cffi
 class TestFrameParameters(unittest.TestCase):

contrib/python-zstandard/tests/test_data_structures_fuzzing.py

@@ -24,8 +24,8 b' s_hashlog = strategies.integers(min_valu'
                                 max_value=zstd.HASHLOG_MAX)
 s_searchlog = strategies.integers(min_value=zstd.SEARCHLOG_MIN,
                                     max_value=zstd.SEARCHLOG_MAX)
-s_searchlength = strategies.integers(min_value=zstd.SEARCHLENGTH_MIN,
-                                     max_value=zstd.SEARCHLENGTH_MAX)
+s_minmatch = strategies.integers(min_value=zstd.MINMATCH_MIN,
+                                 max_value=zstd.MINMATCH_MAX)
 s_targetlength = strategies.integers(min_value=zstd.TARGETLENGTH_MIN,
                                      max_value=zstd.TARGETLENGTH_MAX)
 s_strategy = strategies.sampled_from((zstd.STRATEGY_FAST,
@@ -35,41 +35,42 b' s_strategy = strategies.sampled_from((zs'
                                         zstd.STRATEGY_LAZY2,
                                         zstd.STRATEGY_BTLAZY2,
                                         zstd.STRATEGY_BTOPT,
-                                        zstd.STRATEGY_BTULTRA))
+                                        zstd.STRATEGY_BTULTRA,
+                                        zstd.STRATEGY_BTULTRA2))
 
 
 @make_cffi
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 class TestCompressionParametersHypothesis(unittest.TestCase):
     @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog,
-                        s_searchlength, s_targetlength, s_strategy)
+                        s_minmatch, s_targetlength, s_strategy)
     def test_valid_init(self, windowlog, chainlog, hashlog, searchlog,
-                        searchlength, targetlength, strategy):
+                        minmatch, targetlength, strategy):
         zstd.ZstdCompressionParameters(window_log=windowlog,
                                        chain_log=chainlog,
                                        hash_log=hashlog,
                                        search_log=searchlog,
-                                       min_match=searchlength,
+                                       min_match=minmatch,
                                        target_length=targetlength,
-                                       compression_strategy=strategy)
+                                       strategy=strategy)
 
     @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog,
-                        s_searchlength, s_targetlength, s_strategy)
+                      s_minmatch, s_targetlength, s_strategy)
     def test_estimated_compression_context_size(self, windowlog, chainlog,
                                                 hashlog, searchlog,
-                                                searchlength, targetlength,
+                                                minmatch, targetlength,
                                                 strategy):
-        if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
-            searchlength += 1
-        elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
-            searchlength -= 1
+        if minmatch == zstd.MINMATCH_MIN and strategy in (zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
+            minmatch += 1
+        elif minmatch == zstd.MINMATCH_MAX and strategy != zstd.STRATEGY_FAST:
+            minmatch -= 1
 
         p = zstd.ZstdCompressionParameters(window_log=windowlog,
                                            chain_log=chainlog,
                                            hash_log=hashlog,
                                            search_log=searchlog,
-                                           min_match=searchlength,
+                                           min_match=minmatch,
                                            target_length=targetlength,
-                                           compression_strategy=strategy)
+                                           strategy=strategy)
         size = p.estimated_compression_context_size()
 

contrib/python-zstandard/tests/test_decompressor.py

@@ -3,6 +3,7 b' import os'
 import random
 import struct
 import sys
+import tempfile
 import unittest
 
 import zstandard as zstd
@@ -10,6 +11,7 b' import zstandard as zstd'
 from .common import (
     generate_samples,
     make_cffi,
+    NonClosingBytesIO,
     OpCountingBytesIO,
 )
 
@@ -219,7 +221,7 b' class TestDecompressor_decompress(unitte'
         cctx = zstd.ZstdCompressor(write_content_size=False)
         frame = cctx.compress(source)
 
-        dctx = zstd.ZstdDecompressor(max_window_size=1)
+        dctx = zstd.ZstdDecompressor(max_window_size=2**zstd.WINDOWLOG_MIN)
 
         with self.assertRaisesRegexp(
             zstd.ZstdError, 'decompression error: Frame requires too much memory'):
@@ -302,19 +304,16 b' class TestDecompressor_stream_reader(uni'
         dctx = zstd.ZstdDecompressor()
 
         with dctx.stream_reader(b'foo') as reader:
-            with self.assertRaises(NotImplementedError):
+            with self.assertRaises(io.UnsupportedOperation):
                 reader.readline()
 
-            with self.assertRaises(NotImplementedError):
+            with self.assertRaises(io.UnsupportedOperation):
                 reader.readlines()
 
-            with self.assertRaises(NotImplementedError):
-                reader.readall()
-
-            with self.assertRaises(NotImplementedError):
+            with self.assertRaises(io.UnsupportedOperation):
                 iter(reader)
 
-            with self.assertRaises(NotImplementedError):
+            with self.assertRaises(io.UnsupportedOperation):
                 next(reader)
 
             with self.assertRaises(io.UnsupportedOperation):
@@ -347,15 +346,18 b' class TestDecompressor_stream_reader(uni'
             with self.assertRaisesRegexp(ValueError, 'stream is closed'):
                 reader.read(1)
 
-    def test_bad_read_size(self):
+    def test_read_sizes(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
         dctx = zstd.ZstdDecompressor()
 
-        with dctx.stream_reader(b'foo') as reader:
-            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
-                reader.read(-1)
+        with dctx.stream_reader(foo) as reader:
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative amounts less than -1'):
+                reader.read(-2)
 
-            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
-                reader.read(0)
+            self.assertEqual(reader.read(0), b'')
+            self.assertEqual(reader.read(), b'foo')
 
     def test_read_buffer(self):
         cctx = zstd.ZstdCompressor()
@@ -524,13 +526,243 b' class TestDecompressor_stream_reader(uni'
         reader = dctx.stream_reader(source)
 
         with reader:
-            with self.assertRaises(TypeError):
-                reader.read()
+            reader.read(0)
 
         with reader:
             with self.assertRaisesRegexp(ValueError, 'stream is closed'):
                 reader.read(100)
 
+    def test_partial_read(self):
+        # Inspired by https://github.com/indygreg/python-zstandard/issues/71.
+        buffer = io.BytesIO()
+        cctx = zstd.ZstdCompressor()
+        writer = cctx.stream_writer(buffer)
+        writer.write(bytearray(os.urandom(1000000)))
+        writer.flush(zstd.FLUSH_FRAME)
+        buffer.seek(0)
+
+        dctx = zstd.ZstdDecompressor()
+        reader = dctx.stream_reader(buffer)
+
+        while True:
+            chunk = reader.read(8192)
+            if not chunk:
+                break
+
+    def test_read_multiple_frames(self):
+        cctx = zstd.ZstdCompressor()
+        source = io.BytesIO()
+        writer = cctx.stream_writer(source)
+        writer.write(b'foo')
+        writer.flush(zstd.FLUSH_FRAME)
+        writer.write(b'bar')
+        writer.flush(zstd.FLUSH_FRAME)
+
+        dctx = zstd.ZstdDecompressor()
+
+        reader = dctx.stream_reader(source.getvalue())
+        self.assertEqual(reader.read(2), b'fo')
+        self.assertEqual(reader.read(2), b'o')
+        self.assertEqual(reader.read(2), b'ba')
+        self.assertEqual(reader.read(2), b'r')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source)
+        self.assertEqual(reader.read(2), b'fo')
+        self.assertEqual(reader.read(2), b'o')
+        self.assertEqual(reader.read(2), b'ba')
+        self.assertEqual(reader.read(2), b'r')
+
+        reader = dctx.stream_reader(source.getvalue())
+        self.assertEqual(reader.read(3), b'foo')
+        self.assertEqual(reader.read(3), b'bar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source)
+        self.assertEqual(reader.read(3), b'foo')
+        self.assertEqual(reader.read(3), b'bar')
+
+        reader = dctx.stream_reader(source.getvalue())
+        self.assertEqual(reader.read(4), b'foo')
+        self.assertEqual(reader.read(4), b'bar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source)
+        self.assertEqual(reader.read(4), b'foo')
+        self.assertEqual(reader.read(4), b'bar')
+
+        reader = dctx.stream_reader(source.getvalue())
+        self.assertEqual(reader.read(128), b'foo')
+        self.assertEqual(reader.read(128), b'bar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source)
+        self.assertEqual(reader.read(128), b'foo')
+        self.assertEqual(reader.read(128), b'bar')
+
+        # Now tests for reads spanning frames.
+        reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
+        self.assertEqual(reader.read(3), b'foo')
+        self.assertEqual(reader.read(3), b'bar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source, read_across_frames=True)
+        self.assertEqual(reader.read(3), b'foo')
+        self.assertEqual(reader.read(3), b'bar')
+
+        reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
+        self.assertEqual(reader.read(6), b'foobar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source, read_across_frames=True)
+        self.assertEqual(reader.read(6), b'foobar')
+
+        reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
+        self.assertEqual(reader.read(7), b'foobar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source, read_across_frames=True)
+        self.assertEqual(reader.read(7), b'foobar')
+
+        reader = dctx.stream_reader(source.getvalue(), read_across_frames=True)
+        self.assertEqual(reader.read(128), b'foobar')
+
+        source.seek(0)
+        reader = dctx.stream_reader(source, read_across_frames=True)
+        self.assertEqual(reader.read(128), b'foobar')
+
+    def test_readinto(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
+        dctx = zstd.ZstdDecompressor()
+
+        # Attempting to readinto() a non-writable buffer fails.
+        # The exact exception varies based on the backend.
+        reader = dctx.stream_reader(foo)
+        with self.assertRaises(Exception):
+            reader.readinto(b'foobar')
+
+        # readinto() with sufficiently large destination.
+        b = bytearray(1024)
+        reader = dctx.stream_reader(foo)
+        self.assertEqual(reader.readinto(b), 3)
+        self.assertEqual(b[0:3], b'foo')
+        self.assertEqual(reader.readinto(b), 0)
+        self.assertEqual(b[0:3], b'foo')
+
+        # readinto() with small reads.
+        b = bytearray(1024)
+        reader = dctx.stream_reader(foo, read_size=1)
+        self.assertEqual(reader.readinto(b), 3)
+        self.assertEqual(b[0:3], b'foo')
+
+        # Too small destination buffer.
+        b = bytearray(2)
+        reader = dctx.stream_reader(foo)
+        self.assertEqual(reader.readinto(b), 2)
+        self.assertEqual(b[:], b'fo')
+
+    def test_readinto1(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
+        dctx = zstd.ZstdDecompressor()
+
+        reader = dctx.stream_reader(foo)
+        with self.assertRaises(Exception):
+            reader.readinto1(b'foobar')
+
+        # Sufficiently large destination.
+        b = bytearray(1024)
+        reader = dctx.stream_reader(foo)
+        self.assertEqual(reader.readinto1(b), 3)
+        self.assertEqual(b[0:3], b'foo')
+        self.assertEqual(reader.readinto1(b), 0)
+        self.assertEqual(b[0:3], b'foo')
+
+        # readinto() with small reads.
+        b = bytearray(1024)
+        reader = dctx.stream_reader(foo, read_size=1)
+        self.assertEqual(reader.readinto1(b), 3)
+        self.assertEqual(b[0:3], b'foo')
+
+        # Too small destination buffer.
+        b = bytearray(2)
+        reader = dctx.stream_reader(foo)
+        self.assertEqual(reader.readinto1(b), 2)
+        self.assertEqual(b[:], b'fo')
+
+    def test_readall(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
+        dctx = zstd.ZstdDecompressor()
+        reader = dctx.stream_reader(foo)
+
+        self.assertEqual(reader.readall(), b'foo')
+
+    def test_read1(self):
+        cctx = zstd.ZstdCompressor()
+        foo = cctx.compress(b'foo')
+
+        dctx = zstd.ZstdDecompressor()
+
+        b = OpCountingBytesIO(foo)
+        reader = dctx.stream_reader(b)
+
+        self.assertEqual(reader.read1(), b'foo')
+        self.assertEqual(b._read_count, 1)
+
+        b = OpCountingBytesIO(foo)
+        reader = dctx.stream_reader(b)
+
+        self.assertEqual(reader.read1(0), b'')
+        self.assertEqual(reader.read1(2), b'fo')
+        self.assertEqual(b._read_count, 1)
+        self.assertEqual(reader.read1(1), b'o')
+        self.assertEqual(b._read_count, 1)
+        self.assertEqual(reader.read1(1), b'')
+        self.assertEqual(b._read_count, 2)
+
+    def test_read_lines(self):
+        cctx = zstd.ZstdCompressor()
+        source = b'\n'.join(('line %d' % i).encode('ascii') for i in range(1024))
+
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+        reader = dctx.stream_reader(frame)
+        tr = io.TextIOWrapper(reader, encoding='utf-8')
+
+        lines = []
+        for line in tr:
+            lines.append(line.encode('utf-8'))
+
+        self.assertEqual(len(lines), 1024)
+        self.assertEqual(b''.join(lines), source)
+
+        reader = dctx.stream_reader(frame)
+        tr = io.TextIOWrapper(reader, encoding='utf-8')
+
+        lines = tr.readlines()
+        self.assertEqual(len(lines), 1024)
+        self.assertEqual(''.join(lines).encode('utf-8'), source)
+
+        reader = dctx.stream_reader(frame)
+        tr = io.TextIOWrapper(reader, encoding='utf-8')
+
+        lines = []
+        while True:
+            line = tr.readline()
+            if not line:
+                break
+
+            lines.append(line.encode('utf-8'))
+
+        self.assertEqual(len(lines), 1024)
+        self.assertEqual(b''.join(lines), source)
+
 
 @make_cffi
 class TestDecompressor_decompressobj(unittest.TestCase):
@@ -540,6 +772,9 b' class TestDecompressor_decompressobj(uni'
         dctx = zstd.ZstdDecompressor()
         dobj = dctx.decompressobj()
         self.assertEqual(dobj.decompress(data), b'foobar')
+        self.assertIsNone(dobj.flush())
+        self.assertIsNone(dobj.flush(10))
+        self.assertIsNone(dobj.flush(length=100))
 
     def test_input_types(self):
         compressed = zstd.ZstdCompressor(level=1).compress(b'foo')
@@ -557,7 +792,11 b' class TestDecompressor_decompressobj(uni'
 
         for source in sources:
             dobj = dctx.decompressobj()
+            self.assertIsNone(dobj.flush())
+            self.assertIsNone(dobj.flush(10))
+            self.assertIsNone(dobj.flush(length=100))
             self.assertEqual(dobj.decompress(source), b'foo')
+            self.assertIsNone(dobj.flush())
 
     def test_reuse(self):
         data = zstd.ZstdCompressor(level=1).compress(b'foobar')
@@ -568,6 +807,7 b' class TestDecompressor_decompressobj(uni'
 
         with self.assertRaisesRegexp(zstd.ZstdError, 'cannot use a decompressobj'):
             dobj.decompress(data)
+            self.assertIsNone(dobj.flush())
 
     def test_bad_write_size(self):
         dctx = zstd.ZstdDecompressor()
@@ -585,16 +825,141 b' class TestDecompressor_decompressobj(uni'
             dobj = dctx.decompressobj(write_size=i + 1)
             self.assertEqual(dobj.decompress(data), source)
 
+
 def decompress_via_writer(data):
     buffer = io.BytesIO()
     dctx = zstd.ZstdDecompressor()
-    with dctx.stream_writer(buffer) as decompressor:
-        decompressor.write(data)
+    decompressor = dctx.stream_writer(buffer)
+    decompressor.write(data)
+
     return buffer.getvalue()
 
 
 @make_cffi
 class TestDecompressor_stream_writer(unittest.TestCase):
+    def test_io_api(self):
+        buffer = io.BytesIO()
+        dctx = zstd.ZstdDecompressor()
+        writer = dctx.stream_writer(buffer)
+
+        self.assertFalse(writer.closed)
+        self.assertFalse(writer.isatty())
+        self.assertFalse(writer.readable())
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readline(size=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readlines(hint=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.seek(0)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.seek(10, os.SEEK_SET)
+
+        self.assertFalse(writer.seekable())
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.tell()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.truncate(size=42)
+
+        self.assertTrue(writer.writable())
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.writelines([])
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read(42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.read(size=42)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readall()
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.readinto(None)
+
+        with self.assertRaises(io.UnsupportedOperation):
+            writer.fileno()
+
+    def test_fileno_file(self):
+        with tempfile.TemporaryFile('wb') as tf:
+            dctx = zstd.ZstdDecompressor()
+            writer = dctx.stream_writer(tf)
+
+            self.assertEqual(writer.fileno(), tf.fileno())
+
+    def test_close(self):
+        foo = zstd.ZstdCompressor().compress(b'foo')
+
+        buffer = NonClosingBytesIO()
+        dctx = zstd.ZstdDecompressor()
+        writer = dctx.stream_writer(buffer)
+
+        writer.write(foo)
+        self.assertFalse(writer.closed)
+        self.assertFalse(buffer.closed)
+        writer.close()
+        self.assertTrue(writer.closed)
+        self.assertTrue(buffer.closed)
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            writer.write(b'')
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            writer.flush()
+
+        with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+            with writer:
+                pass
+
+        self.assertEqual(buffer.getvalue(), b'foo')
+
+        # Context manager exit should close stream.
+        buffer = NonClosingBytesIO()
+        writer = dctx.stream_writer(buffer)
+
+        with writer:
+            writer.write(foo)
+
+        self.assertTrue(writer.closed)
+        self.assertEqual(buffer.getvalue(), b'foo')
+
+    def test_flush(self):
+        buffer = OpCountingBytesIO()
+        dctx = zstd.ZstdDecompressor()
+        writer = dctx.stream_writer(buffer)
+
+        writer.flush()
+        self.assertEqual(buffer._flush_count, 1)
+        writer.flush()
+        self.assertEqual(buffer._flush_count, 2)
+
     def test_empty_roundtrip(self):
         cctx = zstd.ZstdCompressor()
         empty = cctx.compress(b'')
@@ -616,9 +981,21 b' class TestDecompressor_stream_writer(uni'
         dctx = zstd.ZstdDecompressor()
         for source in sources:
             buffer = io.BytesIO()
+
+            decompressor = dctx.stream_writer(buffer)
+            decompressor.write(source)
+            self.assertEqual(buffer.getvalue(), b'foo')
+
+            buffer = NonClosingBytesIO()
+
             with dctx.stream_writer(buffer) as decompressor:
-                decompressor.write(source)
+                self.assertEqual(decompressor.write(source), 3)
+
+            self.assertEqual(buffer.getvalue(), b'foo')
 
+            buffer = io.BytesIO()
+            writer = dctx.stream_writer(buffer, write_return_read=True)
+            self.assertEqual(writer.write(source), len(source))
             self.assertEqual(buffer.getvalue(), b'foo')
 
     def test_large_roundtrip(self):
@@ -641,7 +1018,7 b' class TestDecompressor_stream_writer(uni'
         cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(orig)
 
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         dctx = zstd.ZstdDecompressor()
         with dctx.stream_writer(buffer) as decompressor:
             pos = 0
@@ -651,6 +1028,17 b' class TestDecompressor_stream_writer(uni'
                 pos += 8192
         self.assertEqual(buffer.getvalue(), orig)
 
+        # Again with write_return_read=True
+        buffer = io.BytesIO()
+        writer = dctx.stream_writer(buffer, write_return_read=True)
+        pos = 0
+        while pos < len(compressed):
+            pos2 = pos + 8192
+            chunk = compressed[pos:pos2]
+            self.assertEqual(writer.write(chunk), len(chunk))
+            pos += 8192
+        self.assertEqual(buffer.getvalue(), orig)
+
     def test_dictionary(self):
         samples = []
         for i in range(128):
@@ -661,7 +1049,7 b' class TestDecompressor_stream_writer(uni'
         d = zstd.train_dictionary(8192, samples)
 
         orig = b'foobar' * 16384
-        buffer = io.BytesIO()
+        buffer = NonClosingBytesIO()
         cctx = zstd.ZstdCompressor(dict_data=d)
         with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(orig), 0)
@@ -670,6 +1058,12 b' class TestDecompressor_stream_writer(uni'
         buffer = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
+        decompressor = dctx.stream_writer(buffer)
+        self.assertEqual(decompressor.write(compressed), len(orig))
+        self.assertEqual(buffer.getvalue(), orig)
+
+        buffer = NonClosingBytesIO()
+
         with dctx.stream_writer(buffer) as decompressor:
             self.assertEqual(decompressor.write(compressed), len(orig))
 
@@ -678,6 +1072,11 b' class TestDecompressor_stream_writer(uni'
     def test_memory_size(self):
         dctx = zstd.ZstdDecompressor()
         buffer = io.BytesIO()
+
+        decompressor = dctx.stream_writer(buffer)
+        size = decompressor.memory_size()
+        self.assertGreater(size, 100000)
+
         with dctx.stream_writer(buffer) as decompressor:
             size = decompressor.memory_size()
 
@@ -810,7 +1209,7 b' class TestDecompressor_read_to_iter(unit'
     @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
     def test_large_input(self):
         bytes = list(struct.Struct('>B').pack(i) for i in range(256))
-        compressed = io.BytesIO()
+        compressed = NonClosingBytesIO()
         input_size = 0
         cctx = zstd.ZstdCompressor(level=1)
         with cctx.stream_writer(compressed) as compressor:
@@ -823,7 +1222,7 b' class TestDecompressor_read_to_iter(unit'
                 if have_compressed and have_raw:
                     break
 
-        compressed.seek(0)
+        compressed = io.BytesIO(compressed.getvalue())
         self.assertGreater(len(compressed.getvalue()),
                            zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE)
 
@@ -861,7 +1260,7 b' class TestDecompressor_read_to_iter(unit'
 
         source = io.BytesIO()
 
-        compressed = io.BytesIO()
+        compressed = NonClosingBytesIO()
         with cctx.stream_writer(compressed) as compressor:
             for i in range(256):
                 chunk = b'\0' * 1024
@@ -874,7 +1273,7 b' class TestDecompressor_read_to_iter(unit'
                                  max_output_size=len(source.getvalue()))
         self.assertEqual(simple, source.getvalue())
 
-        compressed.seek(0)
+        compressed = io.BytesIO(compressed.getvalue())
         streamed = b''.join(dctx.read_to_iter(compressed))
         self.assertEqual(streamed, source.getvalue())
 
@@ -1001,6 +1400,9 b' class TestDecompressor_multi_decompress_'
     def test_invalid_inputs(self):
         dctx = zstd.ZstdDecompressor()
 
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         with self.assertRaises(TypeError):
             dctx.multi_decompress_to_buffer(True)
 
@@ -1020,6 +1422,10 b' class TestDecompressor_multi_decompress_'
         frames = [cctx.compress(d) for d in original]
 
         dctx = zstd.ZstdDecompressor()
+
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         result = dctx.multi_decompress_to_buffer(frames)
 
         self.assertEqual(len(result), len(frames))
@@ -1041,6 +1447,10 b' class TestDecompressor_multi_decompress_'
         sizes = struct.pack('=' + 'Q' * len(original), *map(len, original))
 
         dctx = zstd.ZstdDecompressor()
+
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         result = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
 
         self.assertEqual(len(result), len(frames))
@@ -1057,6 +1467,9 b' class TestDecompressor_multi_decompress_'
 
         dctx = zstd.ZstdDecompressor()
 
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         segments = struct.pack('=QQQQ', 0, len(frames[0]), len(frames[0]), len(frames[1]))
         b = zstd.BufferWithSegments(b''.join(frames), segments)
 
@@ -1074,12 +1487,16 b' class TestDecompressor_multi_decompress_'
         frames = [cctx.compress(d) for d in original]
         sizes = struct.pack('=' + 'Q' * len(original), *map(len, original))
 
+        dctx = zstd.ZstdDecompressor()
+
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         segments = struct.pack('=QQQQQQ', 0, len(frames[0]),
                                len(frames[0]), len(frames[1]),
                                len(frames[0]) + len(frames[1]), len(frames[2]))
         b = zstd.BufferWithSegments(b''.join(frames), segments)
 
-        dctx = zstd.ZstdDecompressor()
         result = dctx.multi_decompress_to_buffer(b, decompressed_sizes=sizes)
 
         self.assertEqual(len(result), len(frames))
@@ -1099,10 +1516,14 b' class TestDecompressor_multi_decompress_'
             b'foo4' * 6,
         ]
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         frames = cctx.multi_compress_to_buffer(original)
 
         # Check round trip.
         dctx = zstd.ZstdDecompressor()
+
         decompressed = dctx.multi_decompress_to_buffer(frames, threads=3)
 
         self.assertEqual(len(decompressed), len(original))
@@ -1138,7 +1559,12 b' class TestDecompressor_multi_decompress_'
         frames = [cctx.compress(s) for s in generate_samples()]
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
+
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         result = dctx.multi_decompress_to_buffer(frames)
+
         self.assertEqual([o.tobytes() for o in result], generate_samples())
 
     def test_multiple_threads(self):
@@ -1149,6 +1575,10 b' class TestDecompressor_multi_decompress_'
         frames.extend(cctx.compress(b'y' * 64) for i in range(256))
 
         dctx = zstd.ZstdDecompressor()
+
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         result = dctx.multi_decompress_to_buffer(frames, threads=-1)
 
         self.assertEqual(len(result), len(frames))
@@ -1164,6 +1594,9 b' class TestDecompressor_multi_decompress_'
 
         dctx = zstd.ZstdDecompressor()
 
+        if not hasattr(dctx, 'multi_decompress_to_buffer'):
+            self.skipTest('multi_decompress_to_buffer not available')
+
         with self.assertRaisesRegexp(zstd.ZstdError,
                                      'error decompressing item 1: ('
                                      'Corrupted block|'

contrib/python-zstandard/tests/test_decompressor_fuzzing.py

@@ -12,6 +12,7 b' import zstandard as zstd'
 
 from . common import (
     make_cffi,
+    NonClosingBytesIO,
     random_input_data,
 )
 
@@ -23,22 +24,200 b' class TestDecompressor_stream_reader_fuz'
         suppress_health_check=[hypothesis.HealthCheck.large_base_example])
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
-                      source_read_size=strategies.integers(1, 16384),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
                       read_sizes=strategies.data())
-    def test_stream_source_read_variance(self, original, level, source_read_size,
-                                         read_sizes):
+    def test_stream_source_read_variance(self, original, level, streaming,
+                                         source_read_size, read_sizes):
         cctx = zstd.ZstdCompressor(level=level)
-        frame = cctx.compress(original)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            source.seek(0)
+        else:
+            frame = cctx.compress(original)
+            source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
-        source = io.BytesIO(frame)
 
         chunks = []
         with dctx.stream_reader(source, read_size=source_read_size) as reader:
             while True:
-                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                read_size = read_sizes.draw(strategies.integers(-1, 131072))
+                chunk = reader.read(read_size)
+                if not chunk and read_size:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    # Similar to above except we have a constant read() size.
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
+                      read_size=strategies.integers(-1, 131072))
+    def test_stream_source_read_size(self, original, level, streaming,
+                                     source_read_size, read_size):
+        if read_size == 0:
+            read_size = 1
+
+        cctx = zstd.ZstdCompressor(level=level)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            source.seek(0)
+        else:
+            frame = cctx.compress(original)
+            source = io.BytesIO(frame)
+
+        dctx = zstd.ZstdDecompressor()
+
+        chunks = []
+        reader = dctx.stream_reader(source, read_size=source_read_size)
+        while True:
+            chunk = reader.read(read_size)
+            if not chunk and read_size:
+                break
+
+            chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
+                      read_sizes=strategies.data())
+    def test_buffer_source_read_variance(self, original, level, streaming,
+                                         source_read_size, read_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            frame = source.getvalue()
+        else:
+            frame = cctx.compress(original)
+
+        dctx = zstd.ZstdDecompressor()
+        chunks = []
+
+        with dctx.stream_reader(frame, read_size=source_read_size) as reader:
+            while True:
+                read_size = read_sizes.draw(strategies.integers(-1, 131072))
                 chunk = reader.read(read_size)
-                if not chunk:
+                if not chunk and read_size:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    # Similar to above except we have a constant read() size.
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
+                      read_size=strategies.integers(-1, 131072))
+    def test_buffer_source_constant_read_size(self, original, level, streaming,
+                                              source_read_size, read_size):
+        if read_size == 0:
+            read_size = -1
+
+        cctx = zstd.ZstdCompressor(level=level)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            frame = source.getvalue()
+        else:
+            frame = cctx.compress(original)
+
+        dctx = zstd.ZstdDecompressor()
+        chunks = []
+
+        reader = dctx.stream_reader(frame, read_size=source_read_size)
+        while True:
+            chunk = reader.read(read_size)
+            if not chunk and read_size:
+                break
+
+            chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576))
+    def test_stream_source_readall(self, original, level, streaming,
+                                         source_read_size):
+        cctx = zstd.ZstdCompressor(level=level)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            source.seek(0)
+        else:
+            frame = cctx.compress(original)
+            source = io.BytesIO(frame)
+
+        dctx = zstd.ZstdDecompressor()
+
+        data = dctx.stream_reader(source, read_size=source_read_size).readall()
+        self.assertEqual(data, original)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
+                      read_sizes=strategies.data())
+    def test_stream_source_read1_variance(self, original, level, streaming,
+                                          source_read_size, read_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            source.seek(0)
+        else:
+            frame = cctx.compress(original)
+            source = io.BytesIO(frame)
+
+        dctx = zstd.ZstdDecompressor()
+
+        chunks = []
+        with dctx.stream_reader(source, read_size=source_read_size) as reader:
+            while True:
+                read_size = read_sizes.draw(strategies.integers(-1, 131072))
+                chunk = reader.read1(read_size)
+                if not chunk and read_size:
                     break
 
                 chunks.append(chunk)
@@ -49,24 +228,36 b' class TestDecompressor_stream_reader_fuz'
         suppress_health_check=[hypothesis.HealthCheck.large_base_example])
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
-                      source_read_size=strategies.integers(1, 16384),
+                      streaming=strategies.booleans(),
+                      source_read_size=strategies.integers(1, 1048576),
                       read_sizes=strategies.data())
-    def test_buffer_source_read_variance(self, original, level, source_read_size,
-                                         read_sizes):
+    def test_stream_source_readinto1_variance(self, original, level, streaming,
+                                          source_read_size, read_sizes):
         cctx = zstd.ZstdCompressor(level=level)
-        frame = cctx.compress(original)
+
+        if streaming:
+            source = io.BytesIO()
+            writer = cctx.stream_writer(source)
+            writer.write(original)
+            writer.flush(zstd.FLUSH_FRAME)
+            source.seek(0)
+        else:
+            frame = cctx.compress(original)
+            source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
+
         chunks = []
-
-        with dctx.stream_reader(frame, read_size=source_read_size) as reader:
+        with dctx.stream_reader(source, read_size=source_read_size) as reader:
             while True:
-                read_size = read_sizes.draw(strategies.integers(1, 16384))
-                chunk = reader.read(read_size)
-                if not chunk:
+                read_size = read_sizes.draw(strategies.integers(1, 131072))
+                b = bytearray(read_size)
+                count = reader.readinto1(b)
+
+                if not count:
                     break
 
-                chunks.append(chunk)
+                chunks.append(bytes(b[0:count]))
 
         self.assertEqual(b''.join(chunks), original)
 
@@ -75,7 +266,7 b' class TestDecompressor_stream_reader_fuz'
     @hypothesis.given(
         original=strategies.sampled_from(random_input_data()),
         level=strategies.integers(min_value=1, max_value=5),
-        source_read_size=strategies.integers(1, 16384),
+        source_read_size=strategies.integers(1, 1048576),
         seek_amounts=strategies.data(),
         read_sizes=strategies.data())
     def test_relative_seeks(self, original, level, source_read_size, seek_amounts,
@@ -99,6 +290,46 b' class TestDecompressor_stream_reader_fuz'
 
                 self.assertEqual(original[offset:offset + len(chunk)], chunk)
 
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(
+        originals=strategies.data(),
+        frame_count=strategies.integers(min_value=2, max_value=10),
+        level=strategies.integers(min_value=1, max_value=5),
+        source_read_size=strategies.integers(1, 1048576),
+        read_sizes=strategies.data())
+    def test_multiple_frames(self, originals, frame_count, level,
+                             source_read_size, read_sizes):
+
+        cctx = zstd.ZstdCompressor(level=level)
+        source = io.BytesIO()
+        buffer = io.BytesIO()
+        writer = cctx.stream_writer(buffer)
+
+        for i in range(frame_count):
+            data = originals.draw(strategies.sampled_from(random_input_data()))
+            source.write(data)
+            writer.write(data)
+            writer.flush(zstd.FLUSH_FRAME)
+
+        dctx = zstd.ZstdDecompressor()
+        buffer.seek(0)
+        reader = dctx.stream_reader(buffer, read_size=source_read_size,
+                                    read_across_frames=True)
+
+        chunks = []
+
+        while True:
+            read_amount = read_sizes.draw(strategies.integers(-1, 16384))
+            chunk = reader.read(read_amount)
+
+            if not chunk and read_amount:
+                break
+
+            chunks.append(chunk)
+
+        self.assertEqual(source.getvalue(), b''.join(chunks))
+
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
@@ -113,7 +344,7 b' class TestDecompressor_stream_writer_fuz'
 
         dctx = zstd.ZstdDecompressor()
         source = io.BytesIO(frame)
-        dest = io.BytesIO()
+        dest = NonClosingBytesIO()
 
         with dctx.stream_writer(dest, write_size=write_size) as decompressor:
             while True:
@@ -234,10 +465,12 b' class TestDecompressor_multi_decompress_'
                                    write_checksum=True,
                                    **kwargs)
 
+        if not hasattr(cctx, 'multi_compress_to_buffer'):
+            self.skipTest('multi_compress_to_buffer not available')
+
         frames_buffer = cctx.multi_compress_to_buffer(original, threads=-1)
 
         dctx = zstd.ZstdDecompressor(**kwargs)
-
         result = dctx.multi_decompress_to_buffer(frames_buffer)
 
         self.assertEqual(len(result), len(original))

contrib/python-zstandard/tests/test_module_attributes.py

@@ -12,9 +12,9 b' from . common import ('
 @make_cffi
 class TestModuleAttributes(unittest.TestCase):
     def test_version(self):
-        self.assertEqual(zstd.ZSTD_VERSION, (1, 3, 6))
+        self.assertEqual(zstd.ZSTD_VERSION, (1, 3, 8))
 
-        self.assertEqual(zstd.__version__, '0.10.1')
+        self.assertEqual(zstd.__version__, '0.11.0')
 
     def test_constants(self):
         self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22)
@@ -29,6 +29,8 b' class TestModuleAttributes(unittest.Test'
             'DECOMPRESSION_RECOMMENDED_INPUT_SIZE',
             'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE',
             'MAGIC_NUMBER',
+            'FLUSH_BLOCK',
+            'FLUSH_FRAME',
             'BLOCKSIZELOG_MAX',
             'BLOCKSIZE_MAX',
             'WINDOWLOG_MIN',
@@ -38,6 +40,8 b' class TestModuleAttributes(unittest.Test'
             'HASHLOG_MIN',
             'HASHLOG_MAX',
             'HASHLOG3_MAX',
+            'MINMATCH_MIN',
+            'MINMATCH_MAX',
             'SEARCHLOG_MIN',
             'SEARCHLOG_MAX',
             'SEARCHLENGTH_MIN',
@@ -55,6 +59,7 b' class TestModuleAttributes(unittest.Test'
             'STRATEGY_BTLAZY2',
             'STRATEGY_BTOPT',
             'STRATEGY_BTULTRA',
+            'STRATEGY_BTULTRA2',
             'DICT_TYPE_AUTO',
             'DICT_TYPE_RAWCONTENT',
             'DICT_TYPE_FULLDICT',

contrib/python-zstandard/zstandard/__init__.py

@@ -35,31 +35,31 b" if _module_policy == 'default':"
         from zstd import *
         backend = 'cext'
     elif platform.python_implementation() in ('PyPy',):
-        from zstd_cffi import *
+        from .cffi import *
         backend = 'cffi'
     else:
         try:
             from zstd import *
             backend = 'cext'
         except ImportError:
-            from zstd_cffi import *
+            from .cffi import *
             backend = 'cffi'
 elif _module_policy == 'cffi_fallback':
     try:
         from zstd import *
         backend = 'cext'
     except ImportError:
-        from zstd_cffi import *
+        from .cffi import *
         backend = 'cffi'
 elif _module_policy == 'cext':
     from zstd import *
     backend = 'cext'
 elif _module_policy == 'cffi':
-    from zstd_cffi import *
+    from .cffi import *
     backend = 'cffi'
 else:
     raise ImportError('unknown module import policy: %s; use default, cffi_fallback, '
                       'cext, or cffi' % _module_policy)
 
 # Keep this in sync with python-zstandard.h.
-__version__ = '0.10.1'
+__version__ = '0.11.0'

contrib/python-zstandard/zstandard/cffi.py

@@ -28,6 +28,8 b' from __future__ import absolute_import, '
     'train_dictionary',
 
     # Constants.
+    'FLUSH_BLOCK',
+    'FLUSH_FRAME',
     'COMPRESSOBJ_FLUSH_FINISH',
     'COMPRESSOBJ_FLUSH_BLOCK',
     'ZSTD_VERSION',
@@ -49,6 +51,8 b' from __future__ import absolute_import, '
     'HASHLOG_MIN',
     'HASHLOG_MAX',
     'HASHLOG3_MAX',
+    'MINMATCH_MIN',
+    'MINMATCH_MAX',
     'SEARCHLOG_MIN',
     'SEARCHLOG_MAX',
     'SEARCHLENGTH_MIN',
@@ -66,6 +70,7 b' from __future__ import absolute_import, '
     'STRATEGY_BTLAZY2',
     'STRATEGY_BTOPT',
     'STRATEGY_BTULTRA',
+    'STRATEGY_BTULTRA2',
     'DICT_TYPE_AUTO',
     'DICT_TYPE_RAWCONTENT',
     'DICT_TYPE_FULLDICT',
@@ -114,10 +119,12 b' CHAINLOG_MAX = lib.ZSTD_CHAINLOG_MAX'
 HASHLOG_MIN = lib.ZSTD_HASHLOG_MIN
 HASHLOG_MAX = lib.ZSTD_HASHLOG_MAX
 HASHLOG3_MAX = lib.ZSTD_HASHLOG3_MAX
+MINMATCH_MIN = lib.ZSTD_MINMATCH_MIN
+MINMATCH_MAX = lib.ZSTD_MINMATCH_MAX
 SEARCHLOG_MIN = lib.ZSTD_SEARCHLOG_MIN
 SEARCHLOG_MAX = lib.ZSTD_SEARCHLOG_MAX
-SEARCHLENGTH_MIN = lib.ZSTD_SEARCHLENGTH_MIN
-SEARCHLENGTH_MAX = lib.ZSTD_SEARCHLENGTH_MAX
+SEARCHLENGTH_MIN = lib.ZSTD_MINMATCH_MIN
+SEARCHLENGTH_MAX = lib.ZSTD_MINMATCH_MAX
 TARGETLENGTH_MIN = lib.ZSTD_TARGETLENGTH_MIN
 TARGETLENGTH_MAX = lib.ZSTD_TARGETLENGTH_MAX
 LDM_MINMATCH_MIN = lib.ZSTD_LDM_MINMATCH_MIN
@@ -132,6 +139,7 b' STRATEGY_LAZY2 = lib.ZSTD_lazy2'
 STRATEGY_BTLAZY2 = lib.ZSTD_btlazy2
 STRATEGY_BTOPT = lib.ZSTD_btopt
 STRATEGY_BTULTRA = lib.ZSTD_btultra
+STRATEGY_BTULTRA2 = lib.ZSTD_btultra2
 
 DICT_TYPE_AUTO = lib.ZSTD_dct_auto
 DICT_TYPE_RAWCONTENT = lib.ZSTD_dct_rawContent
@@ -140,6 +148,9 b' DICT_TYPE_FULLDICT = lib.ZSTD_dct_fullDi'
 FORMAT_ZSTD1 = lib.ZSTD_f_zstd1
 FORMAT_ZSTD1_MAGICLESS = lib.ZSTD_f_zstd1_magicless
 
+FLUSH_BLOCK = 0
+FLUSH_FRAME = 1
+
 COMPRESSOBJ_FLUSH_FINISH = 0
 COMPRESSOBJ_FLUSH_BLOCK = 1
 
@@ -182,27 +193,27 b' def _make_cctx_params(params):'
     res = ffi.gc(res, lib.ZSTD_freeCCtxParams)
 
     attrs = [
-        (lib.ZSTD_p_format, params.format),
-        (lib.ZSTD_p_compressionLevel, params.compression_level),
-        (lib.ZSTD_p_windowLog, params.window_log),
-        (lib.ZSTD_p_hashLog, params.hash_log),
-        (lib.ZSTD_p_chainLog, params.chain_log),
-        (lib.ZSTD_p_searchLog, params.search_log),
-        (lib.ZSTD_p_minMatch, params.min_match),
-        (lib.ZSTD_p_targetLength, params.target_length),
-        (lib.ZSTD_p_compressionStrategy, params.compression_strategy),
-        (lib.ZSTD_p_contentSizeFlag, params.write_content_size),
-        (lib.ZSTD_p_checksumFlag, params.write_checksum),
-        (lib.ZSTD_p_dictIDFlag, params.write_dict_id),
-        (lib.ZSTD_p_nbWorkers, params.threads),
-        (lib.ZSTD_p_jobSize, params.job_size),
-        (lib.ZSTD_p_overlapSizeLog, params.overlap_size_log),
-        (lib.ZSTD_p_forceMaxWindow, params.force_max_window),
-        (lib.ZSTD_p_enableLongDistanceMatching, params.enable_ldm),
-        (lib.ZSTD_p_ldmHashLog, params.ldm_hash_log),
-        (lib.ZSTD_p_ldmMinMatch, params.ldm_min_match),
-        (lib.ZSTD_p_ldmBucketSizeLog, params.ldm_bucket_size_log),
-        (lib.ZSTD_p_ldmHashEveryLog, params.ldm_hash_every_log),
+        (lib.ZSTD_c_format, params.format),
+        (lib.ZSTD_c_compressionLevel, params.compression_level),
+        (lib.ZSTD_c_windowLog, params.window_log),
+        (lib.ZSTD_c_hashLog, params.hash_log),
+        (lib.ZSTD_c_chainLog, params.chain_log),
+        (lib.ZSTD_c_searchLog, params.search_log),
+        (lib.ZSTD_c_minMatch, params.min_match),
+        (lib.ZSTD_c_targetLength, params.target_length),
+        (lib.ZSTD_c_strategy, params.compression_strategy),
+        (lib.ZSTD_c_contentSizeFlag, params.write_content_size),
+        (lib.ZSTD_c_checksumFlag, params.write_checksum),
+        (lib.ZSTD_c_dictIDFlag, params.write_dict_id),
+        (lib.ZSTD_c_nbWorkers, params.threads),
+        (lib.ZSTD_c_jobSize, params.job_size),
+        (lib.ZSTD_c_overlapLog, params.overlap_log),
+        (lib.ZSTD_c_forceMaxWindow, params.force_max_window),
+        (lib.ZSTD_c_enableLongDistanceMatching, params.enable_ldm),
+        (lib.ZSTD_c_ldmHashLog, params.ldm_hash_log),
+        (lib.ZSTD_c_ldmMinMatch, params.ldm_min_match),
+        (lib.ZSTD_c_ldmBucketSizeLog, params.ldm_bucket_size_log),
+        (lib.ZSTD_c_ldmHashRateLog, params.ldm_hash_rate_log),
     ]
 
     for param, value in attrs:
@@ -220,7 +231,7 b' class ZstdCompressionParameters(object):'
             'chain_log': 'chainLog',
             'hash_log': 'hashLog',
             'search_log': 'searchLog',
-            'min_match': 'searchLength',
+            'min_match': 'minMatch',
             'target_length': 'targetLength',
             'compression_strategy': 'strategy',
         }
@@ -233,41 +244,170 b' class ZstdCompressionParameters(object):'
 
     def __init__(self, format=0, compression_level=0, window_log=0, hash_log=0,
                  chain_log=0, search_log=0, min_match=0, target_length=0,
-                 compression_strategy=0, write_content_size=1, write_checksum=0,
-                 write_dict_id=0, job_size=0, overlap_size_log=0,
-                 force_max_window=0, enable_ldm=0, ldm_hash_log=0,
-                 ldm_min_match=0, ldm_bucket_size_log=0, ldm_hash_every_log=0,
-                 threads=0):
+                 strategy=-1, compression_strategy=-1,
+                 write_content_size=1, write_checksum=0,
+                 write_dict_id=0, job_size=0, overlap_log=-1,
+                 overlap_size_log=-1, force_max_window=0, enable_ldm=0,
+                 ldm_hash_log=0, ldm_min_match=0, ldm_bucket_size_log=0,
+                 ldm_hash_rate_log=-1, ldm_hash_every_log=-1, threads=0):
+
+        params = lib.ZSTD_createCCtxParams()
+        if params == ffi.NULL:
+            raise MemoryError()
+
+        params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
+
+        self._params = params
 
         if threads < 0:
             threads = _cpu_count()
 
-        self.format = format
-        self.compression_level = compression_level
-        self.window_log = window_log
-        self.hash_log = hash_log
-        self.chain_log = chain_log
-        self.search_log = search_log
-        self.min_match = min_match
-        self.target_length = target_length
-        self.compression_strategy = compression_strategy
-        self.write_content_size = write_content_size
-        self.write_checksum = write_checksum
-        self.write_dict_id = write_dict_id
-        self.job_size = job_size
-        self.overlap_size_log = overlap_size_log
-        self.force_max_window = force_max_window
-        self.enable_ldm = enable_ldm
-        self.ldm_hash_log = ldm_hash_log
-        self.ldm_min_match = ldm_min_match
-        self.ldm_bucket_size_log = ldm_bucket_size_log
-        self.ldm_hash_every_log = ldm_hash_every_log
-        self.threads = threads
-
-        self.params = _make_cctx_params(self)
+        # We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
+        # because setting ZSTD_c_nbWorkers resets the other parameters.
+        _set_compression_parameter(params, lib.ZSTD_c_nbWorkers, threads)
+
+        _set_compression_parameter(params, lib.ZSTD_c_format, format)
+        _set_compression_parameter(params, lib.ZSTD_c_compressionLevel, compression_level)
+        _set_compression_parameter(params, lib.ZSTD_c_windowLog, window_log)
+        _set_compression_parameter(params, lib.ZSTD_c_hashLog, hash_log)
+        _set_compression_parameter(params, lib.ZSTD_c_chainLog, chain_log)
+        _set_compression_parameter(params, lib.ZSTD_c_searchLog, search_log)
+        _set_compression_parameter(params, lib.ZSTD_c_minMatch, min_match)
+        _set_compression_parameter(params, lib.ZSTD_c_targetLength, target_length)
+
+        if strategy != -1 and compression_strategy != -1:
+            raise ValueError('cannot specify both compression_strategy and strategy')
+
+        if compression_strategy != -1:
+            strategy = compression_strategy
+        elif strategy == -1:
+            strategy = 0
+
+        _set_compression_parameter(params, lib.ZSTD_c_strategy, strategy)
+        _set_compression_parameter(params, lib.ZSTD_c_contentSizeFlag, write_content_size)
+        _set_compression_parameter(params, lib.ZSTD_c_checksumFlag, write_checksum)
+        _set_compression_parameter(params, lib.ZSTD_c_dictIDFlag, write_dict_id)
+        _set_compression_parameter(params, lib.ZSTD_c_jobSize, job_size)
+
+        if overlap_log != -1 and overlap_size_log != -1:
+            raise ValueError('cannot specify both overlap_log and overlap_size_log')
+
+        if overlap_size_log != -1:
+            overlap_log = overlap_size_log
+        elif overlap_log == -1:
+            overlap_log = 0
+
+        _set_compression_parameter(params, lib.ZSTD_c_overlapLog, overlap_log)
+        _set_compression_parameter(params, lib.ZSTD_c_forceMaxWindow, force_max_window)
+        _set_compression_parameter(params, lib.ZSTD_c_enableLongDistanceMatching, enable_ldm)
+        _set_compression_parameter(params, lib.ZSTD_c_ldmHashLog, ldm_hash_log)
+        _set_compression_parameter(params, lib.ZSTD_c_ldmMinMatch, ldm_min_match)
+        _set_compression_parameter(params, lib.ZSTD_c_ldmBucketSizeLog, ldm_bucket_size_log)
+
+        if ldm_hash_rate_log != -1 and ldm_hash_every_log != -1:
+            raise ValueError('cannot specify both ldm_hash_rate_log and ldm_hash_every_log')
+
+        if ldm_hash_every_log != -1:
+            ldm_hash_rate_log = ldm_hash_every_log
+        elif ldm_hash_rate_log == -1:
+            ldm_hash_rate_log = 0
+
+        _set_compression_parameter(params, lib.ZSTD_c_ldmHashRateLog, ldm_hash_rate_log)
+
+    @property
+    def format(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_format)
+
+    @property
+    def compression_level(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_compressionLevel)
+
+    @property
+    def window_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_windowLog)
+
+    @property
+    def hash_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_hashLog)
+
+    @property
+    def chain_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_chainLog)
+
+    @property
+    def search_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_searchLog)
+
+    @property
+    def min_match(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_minMatch)
+
+    @property
+    def target_length(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_targetLength)
+
+    @property
+    def compression_strategy(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_strategy)
+
+    @property
+    def write_content_size(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_contentSizeFlag)
+
+    @property
+    def write_checksum(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_checksumFlag)
+
+    @property
+    def write_dict_id(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_dictIDFlag)
+
+    @property
+    def job_size(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_jobSize)
+
+    @property
+    def overlap_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_overlapLog)
+
+    @property
+    def overlap_size_log(self):
+        return self.overlap_log
+
+    @property
+    def force_max_window(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_forceMaxWindow)
+
+    @property
+    def enable_ldm(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_enableLongDistanceMatching)
+
+    @property
+    def ldm_hash_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashLog)
+
+    @property
+    def ldm_min_match(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmMinMatch)
+
+    @property
+    def ldm_bucket_size_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmBucketSizeLog)
+
+    @property
+    def ldm_hash_rate_log(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_ldmHashRateLog)
+
+    @property
+    def ldm_hash_every_log(self):
+        return self.ldm_hash_rate_log
+
+    @property
+    def threads(self):
+        return _get_compression_parameter(self._params, lib.ZSTD_c_nbWorkers)
 
     def estimated_compression_context_size(self):
-        return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self.params)
+        return lib.ZSTD_estimateCCtxSize_usingCCtxParams(self._params)
 
 CompressionParameters = ZstdCompressionParameters
 
@@ -276,31 +416,53 b' def estimate_decompression_context_size('
 
 
 def _set_compression_parameter(params, param, value):
-    zresult = lib.ZSTD_CCtxParam_setParameter(params, param,
-                                              ffi.cast('unsigned', value))
+    zresult = lib.ZSTD_CCtxParam_setParameter(params, param, value)
     if lib.ZSTD_isError(zresult):
         raise ZstdError('unable to set compression context parameter: %s' %
                         _zstd_error(zresult))
 
+
+def _get_compression_parameter(params, param):
+    result = ffi.new('int *')
+
+    zresult = lib.ZSTD_CCtxParam_getParameter(params, param, result)
+    if lib.ZSTD_isError(zresult):
+        raise ZstdError('unable to get compression context parameter: %s' %
+                        _zstd_error(zresult))
+
+    return result[0]
+
+
 class ZstdCompressionWriter(object):
-    def __init__(self, compressor, writer, source_size, write_size):
+    def __init__(self, compressor, writer, source_size, write_size,
+                 write_return_read):
         self._compressor = compressor
         self._writer = writer
-        self._source_size = source_size
         self._write_size = write_size
+        self._write_return_read = bool(write_return_read)
         self._entered = False
+        self._closed = False
         self._bytes_compressed = 0
 
-    def __enter__(self):
-        if self._entered:
-            raise ZstdError('cannot __enter__ multiple times')
-
-        zresult = lib.ZSTD_CCtx_setPledgedSrcSize(self._compressor._cctx,
-                                                  self._source_size)
+        self._dst_buffer = ffi.new('char[]', write_size)
+        self._out_buffer = ffi.new('ZSTD_outBuffer *')
+        self._out_buffer.dst = self._dst_buffer
+        self._out_buffer.size = len(self._dst_buffer)
+        self._out_buffer.pos = 0
+
+        zresult = lib.ZSTD_CCtx_setPledgedSrcSize(compressor._cctx,
+                                                  source_size)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
+    def __enter__(self):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if self._entered:
+            raise ZstdError('cannot __enter__ multiple times')
+
         self._entered = True
         return self
 
@@ -308,50 +470,79 b' class ZstdCompressionWriter(object):'
         self._entered = False
 
         if not exc_type and not exc_value and not exc_tb:
-            dst_buffer = ffi.new('char[]', self._write_size)
-
-            out_buffer = ffi.new('ZSTD_outBuffer *')
-            in_buffer = ffi.new('ZSTD_inBuffer *')
-
-            out_buffer.dst = dst_buffer
-            out_buffer.size = len(dst_buffer)
-            out_buffer.pos = 0
-
-            in_buffer.src = ffi.NULL
-            in_buffer.size = 0
-            in_buffer.pos = 0
-
-            while True:
-                zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                    out_buffer, in_buffer,
-                                                    lib.ZSTD_e_end)
-
-                if lib.ZSTD_isError(zresult):
-                    raise ZstdError('error ending compression stream: %s' %
-                                    _zstd_error(zresult))
-
-                if out_buffer.pos:
-                    self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)[:])
-                    out_buffer.pos = 0
-
-                if zresult == 0:
-                    break
+            self.close()
 
         self._compressor = None
 
         return False
 
     def memory_size(self):
-        if not self._entered:
-            raise ZstdError('cannot determine size of an inactive compressor; '
-                            'call when a context manager is active')
-
         return lib.ZSTD_sizeof_CCtx(self._compressor._cctx)
 
+    def fileno(self):
+        f = getattr(self._writer, 'fileno', None)
+        if f:
+            return f()
+        else:
+            raise OSError('fileno not available on underlying writer')
+
+    def close(self):
+        if self._closed:
+            return
+
+        try:
+            self.flush(FLUSH_FRAME)
+        finally:
+            self._closed = True
+
+        # Call close() on underlying stream as well.
+        f = getattr(self._writer, 'close', None)
+        if f:
+            f()
+
+    @property
+    def closed(self):
+        return self._closed
+
+    def isatty(self):
+        return False
+
+    def readable(self):
+        return False
+
+    def readline(self, size=-1):
+        raise io.UnsupportedOperation()
+
+    def readlines(self, hint=-1):
+        raise io.UnsupportedOperation()
+
+    def seek(self, offset, whence=None):
+        raise io.UnsupportedOperation()
+
+    def seekable(self):
+        return False
+
+    def truncate(self, size=None):
+        raise io.UnsupportedOperation()
+
+    def writable(self):
+        return True
+
+    def writelines(self, lines):
+        raise NotImplementedError('writelines() is not yet implemented')
+
+    def read(self, size=-1):
+        raise io.UnsupportedOperation()
+
+    def readall(self):
+        raise io.UnsupportedOperation()
+
+    def readinto(self, b):
+        raise io.UnsupportedOperation()
+
     def write(self, data):
-        if not self._entered:
-            raise ZstdError('write() must be called from an active context '
-                            'manager')
+        if self._closed:
+            raise ValueError('stream is closed')
 
         total_write = 0
 
@@ -362,16 +553,13 b' class ZstdCompressionWriter(object):'
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
-        out_buffer = ffi.new('ZSTD_outBuffer *')
-        dst_buffer = ffi.new('char[]', self._write_size)
-        out_buffer.dst = dst_buffer
-        out_buffer.size = self._write_size
+        out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         while in_buffer.pos < in_buffer.size:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                out_buffer, in_buffer,
-                                                lib.ZSTD_e_continue)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               out_buffer, in_buffer,
+                                               lib.ZSTD_e_continue)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
@@ -382,18 +570,25 b' class ZstdCompressionWriter(object):'
                 self._bytes_compressed += out_buffer.pos
                 out_buffer.pos = 0
 
-        return total_write
-
-    def flush(self):
-        if not self._entered:
-            raise ZstdError('flush must be called from an active context manager')
+        if self._write_return_read:
+            return in_buffer.pos
+        else:
+            return total_write
+
+    def flush(self, flush_mode=FLUSH_BLOCK):
+        if flush_mode == FLUSH_BLOCK:
+            flush = lib.ZSTD_e_flush
+        elif flush_mode == FLUSH_FRAME:
+            flush = lib.ZSTD_e_end
+        else:
+            raise ValueError('unknown flush_mode: %r' % flush_mode)
+
+        if self._closed:
+            raise ValueError('stream is closed')
 
         total_write = 0
 
-        out_buffer = ffi.new('ZSTD_outBuffer *')
-        dst_buffer = ffi.new('char[]', self._write_size)
-        out_buffer.dst = dst_buffer
-        out_buffer.size = self._write_size
+        out_buffer = self._out_buffer
         out_buffer.pos = 0
 
         in_buffer = ffi.new('ZSTD_inBuffer *')
@@ -402,9 +597,9 b' class ZstdCompressionWriter(object):'
         in_buffer.pos = 0
 
         while True:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                out_buffer, in_buffer,
-                                                lib.ZSTD_e_flush)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               out_buffer, in_buffer,
+                                               flush)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
@@ -438,10 +633,10 b' class ZstdCompressionObj(object):'
         chunks = []
 
         while source.pos < len(data):
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                self._out,
-                                                source,
-                                                lib.ZSTD_e_continue)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               self._out,
+                                               source,
+                                               lib.ZSTD_e_continue)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' %
                                 _zstd_error(zresult))
@@ -477,10 +672,10 b' class ZstdCompressionObj(object):'
         chunks = []
 
         while True:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                self._out,
-                                                in_buffer,
-                                                z_flush_mode)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               self._out,
+                                               in_buffer,
+                                               z_flush_mode)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
@@ -528,10 +723,10 b' class ZstdCompressionChunker(object):'
         self._in.pos = 0
 
         while self._in.pos < self._in.size:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                self._out,
-                                                self._in,
-                                                lib.ZSTD_e_continue)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               self._out,
+                                               self._in,
+                                               lib.ZSTD_e_continue)
 
             if self._in.pos == self._in.size:
                 self._in.src = ffi.NULL
@@ -555,9 +750,9 b' class ZstdCompressionChunker(object):'
                             'previous operation')
 
         while True:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                self._out, self._in,
-                                                lib.ZSTD_e_flush)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               self._out, self._in,
+                                               lib.ZSTD_e_flush)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' % _zstd_error(zresult))
 
@@ -577,9 +772,9 b' class ZstdCompressionChunker(object):'
                             'previous operation')
 
         while True:
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                self._out, self._in,
-                                                lib.ZSTD_e_end)
+            zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                               self._out, self._in,
+                                               lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd compress error: %s' % _zstd_error(zresult))
 
@@ -592,7 +787,7 b' class ZstdCompressionChunker(object):'
                 return
 
 
-class CompressionReader(object):
+class ZstdCompressionReader(object):
     def __init__(self, compressor, source, read_size):
         self._compressor = compressor
         self._source = source
@@ -661,7 +856,16 b' class CompressionReader(object):'
         return self._bytes_compressed
 
     def readall(self):
-        raise NotImplementedError()
+        chunks = []
+
+        while True:
+            chunk = self.read(1048576)
+            if not chunk:
+                break
+
+            chunks.append(chunk)
+
+        return b''.join(chunks)
 
     def __iter__(self):
         raise io.UnsupportedOperation()
@@ -671,16 +875,67 b' class CompressionReader(object):'
 
     next = __next__
 
+    def _read_input(self):
+        if self._finished_input:
+            return
+
+        if hasattr(self._source, 'read'):
+            data = self._source.read(self._read_size)
+
+            if not data:
+                self._finished_input = True
+                return
+
+            self._source_buffer = ffi.from_buffer(data)
+            self._in_buffer.src = self._source_buffer
+            self._in_buffer.size = len(self._source_buffer)
+            self._in_buffer.pos = 0
+        else:
+            self._source_buffer = ffi.from_buffer(self._source)
+            self._in_buffer.src = self._source_buffer
+            self._in_buffer.size = len(self._source_buffer)
+            self._in_buffer.pos = 0
+
+    def _compress_into_buffer(self, out_buffer):
+        if self._in_buffer.pos >= self._in_buffer.size:
+            return
+
+        old_pos = out_buffer.pos
+
+        zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                           out_buffer, self._in_buffer,
+                                           lib.ZSTD_e_continue)
+
+        self._bytes_compressed += out_buffer.pos - old_pos
+
+        if self._in_buffer.pos == self._in_buffer.size:
+            self._in_buffer.src = ffi.NULL
+            self._in_buffer.pos = 0
+            self._in_buffer.size = 0
+            self._source_buffer = None
+
+            if not hasattr(self._source, 'read'):
+                self._finished_input = True
+
+        if lib.ZSTD_isError(zresult):
+            raise ZstdError('zstd compress error: %s',
+                            _zstd_error(zresult))
+
+        return out_buffer.pos and out_buffer.pos == out_buffer.size
+
     def read(self, size=-1):
         if self._closed:
             raise ValueError('stream is closed')
 
-        if self._finished_output:
+        if size < -1:
+            raise ValueError('cannot read negative amounts less than -1')
+
+        if size == -1:
+            return self.readall()
+
+        if self._finished_output or size == 0:
             return b''
 
-        if size < 1:
-            raise ValueError('cannot read negative or size 0 amounts')
-
         # Need a dedicated ref to dest buffer otherwise it gets collected.
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
@@ -688,71 +943,21 b' class CompressionReader(object):'
         out_buffer.size = size
         out_buffer.pos = 0
 
-        def compress_input():
-            if self._in_buffer.pos >= self._in_buffer.size:
-                return
-
-            old_pos = out_buffer.pos
-
-            zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                                out_buffer, self._in_buffer,
-                                                lib.ZSTD_e_continue)
-
-            self._bytes_compressed += out_buffer.pos - old_pos
-
-            if self._in_buffer.pos == self._in_buffer.size:
-                self._in_buffer.src = ffi.NULL
-                self._in_buffer.pos = 0
-                self._in_buffer.size = 0
-                self._source_buffer = None
-
-                if not hasattr(self._source, 'read'):
-                    self._finished_input = True
-
-            if lib.ZSTD_isError(zresult):
-                raise ZstdError('zstd compress error: %s',
-                                _zstd_error(zresult))
-
-            if out_buffer.pos and out_buffer.pos == out_buffer.size:
-                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
-
-        def get_input():
-            if self._finished_input:
-                return
-
-            if hasattr(self._source, 'read'):
-                data = self._source.read(self._read_size)
-
-                if not data:
-                    self._finished_input = True
-                    return
-
-                self._source_buffer = ffi.from_buffer(data)
-                self._in_buffer.src = self._source_buffer
-                self._in_buffer.size = len(self._source_buffer)
-                self._in_buffer.pos = 0
-            else:
-                self._source_buffer = ffi.from_buffer(self._source)
-                self._in_buffer.src = self._source_buffer
-                self._in_buffer.size = len(self._source_buffer)
-                self._in_buffer.pos = 0
-
-        result = compress_input()
-        if result:
-            return result
+        if self._compress_into_buffer(out_buffer):
+            return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         while not self._finished_input:
-            get_input()
-            result = compress_input()
-            if result:
-                return result
+            self._read_input()
+
+            if self._compress_into_buffer(out_buffer):
+                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
         # EOF
         old_pos = out_buffer.pos
 
-        zresult = lib.ZSTD_compress_generic(self._compressor._cctx,
-                                            out_buffer, self._in_buffer,
-                                            lib.ZSTD_e_end)
+        zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                           out_buffer, self._in_buffer,
+                                           lib.ZSTD_e_end)
 
         self._bytes_compressed += out_buffer.pos - old_pos
 
@@ -765,6 +970,159 b' class CompressionReader(object):'
 
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
+    def read1(self, size=-1):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if size < -1:
+            raise ValueError('cannot read negative amounts less than -1')
+
+        if self._finished_output or size == 0:
+            return b''
+
+        # -1 returns arbitrary number of bytes.
+        if size == -1:
+            size = COMPRESSION_RECOMMENDED_OUTPUT_SIZE
+
+        dst_buffer = ffi.new('char[]', size)
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dst_buffer
+        out_buffer.size = size
+        out_buffer.pos = 0
+
+        # read1() dictates that we can perform at most 1 call to the
+        # underlying stream to get input. However, we can't satisfy this
+        # restriction with compression because not all input generates output.
+        # It is possible to perform a block flush in order to ensure output.
+        # But this may not be desirable behavior. So we allow multiple read()
+        # to the underlying stream. But unlike read(), we stop once we have
+        # any output.
+
+        self._compress_into_buffer(out_buffer)
+        if out_buffer.pos:
+            return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+        while not self._finished_input:
+            self._read_input()
+
+            # If we've filled the output buffer, return immediately.
+            if self._compress_into_buffer(out_buffer):
+                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+            # If we've populated the output buffer and we're not at EOF,
+            # also return, as we've satisfied the read1() limits.
+            if out_buffer.pos and not self._finished_input:
+                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+            # Else if we're at EOS and we have room left in the buffer,
+            # fall through to below and try to add more data to the output.
+
+        # EOF.
+        old_pos = out_buffer.pos
+
+        zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                           out_buffer, self._in_buffer,
+                                           lib.ZSTD_e_end)
+
+        self._bytes_compressed += out_buffer.pos - old_pos
+
+        if lib.ZSTD_isError(zresult):
+            raise ZstdError('error ending compression stream: %s' %
+                            _zstd_error(zresult))
+
+        if zresult == 0:
+            self._finished_output = True
+
+        return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+    def readinto(self, b):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if self._finished_output:
+            return 0
+
+        # TODO use writable=True once we require CFFI >= 1.12.
+        dest_buffer = ffi.from_buffer(b)
+        ffi.memmove(b, b'', 0)
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dest_buffer
+        out_buffer.size = len(dest_buffer)
+        out_buffer.pos = 0
+
+        if self._compress_into_buffer(out_buffer):
+            return out_buffer.pos
+
+        while not self._finished_input:
+            self._read_input()
+            if self._compress_into_buffer(out_buffer):
+                return out_buffer.pos
+
+        # EOF.
+        old_pos = out_buffer.pos
+        zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                           out_buffer, self._in_buffer,
+                                           lib.ZSTD_e_end)
+
+        self._bytes_compressed += out_buffer.pos - old_pos
+
+        if lib.ZSTD_isError(zresult):
+            raise ZstdError('error ending compression stream: %s',
+                            _zstd_error(zresult))
+
+        if zresult == 0:
+            self._finished_output = True
+
+        return out_buffer.pos
+
+    def readinto1(self, b):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if self._finished_output:
+            return 0
+
+        # TODO use writable=True once we require CFFI >= 1.12.
+        dest_buffer = ffi.from_buffer(b)
+        ffi.memmove(b, b'', 0)
+
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dest_buffer
+        out_buffer.size = len(dest_buffer)
+        out_buffer.pos = 0
+
+        self._compress_into_buffer(out_buffer)
+        if out_buffer.pos:
+            return out_buffer.pos
+
+        while not self._finished_input:
+            self._read_input()
+
+            if self._compress_into_buffer(out_buffer):
+                return out_buffer.pos
+
+            if out_buffer.pos and not self._finished_input:
+                return out_buffer.pos
+
+        # EOF.
+        old_pos = out_buffer.pos
+
+        zresult = lib.ZSTD_compressStream2(self._compressor._cctx,
+                                           out_buffer, self._in_buffer,
+                                           lib.ZSTD_e_end)
+
+        self._bytes_compressed += out_buffer.pos - old_pos
+
+        if lib.ZSTD_isError(zresult):
+            raise ZstdError('error ending compression stream: %s' %
+                            _zstd_error(zresult))
+
+        if zresult == 0:
+            self._finished_output = True
+
+        return out_buffer.pos
+
+
 class ZstdCompressor(object):
     def __init__(self, level=3, dict_data=None, compression_params=None,
                  write_checksum=None, write_content_size=None,
@@ -803,25 +1161,25 b' class ZstdCompressor(object):'
             self._params = ffi.gc(params, lib.ZSTD_freeCCtxParams)
 
             _set_compression_parameter(self._params,
-                                       lib.ZSTD_p_compressionLevel,
+                                       lib.ZSTD_c_compressionLevel,
                                        level)
 
             _set_compression_parameter(
                 self._params,
-                lib.ZSTD_p_contentSizeFlag,
+                lib.ZSTD_c_contentSizeFlag,
                 write_content_size if write_content_size is not None else 1)
 
             _set_compression_parameter(self._params,
-                                       lib.ZSTD_p_checksumFlag,
+                                       lib.ZSTD_c_checksumFlag,
                                        1 if write_checksum else 0)
 
             _set_compression_parameter(self._params,
-                                       lib.ZSTD_p_dictIDFlag,
+                                       lib.ZSTD_c_dictIDFlag,
                                        1 if write_dict_id else 0)
 
             if threads:
                 _set_compression_parameter(self._params,
-                                           lib.ZSTD_p_nbWorkers,
+                                           lib.ZSTD_c_nbWorkers,
                                            threads)
 
         cctx = lib.ZSTD_createCCtx()
@@ -864,7 +1222,7 b' class ZstdCompressor(object):'
         return lib.ZSTD_sizeof_CCtx(self._cctx)
 
     def compress(self, data):
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         data_buffer = ffi.from_buffer(data)
 
@@ -887,10 +1245,10 b' class ZstdCompressor(object):'
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
-        zresult = lib.ZSTD_compress_generic(self._cctx,
-                                            out_buffer,
-                                            in_buffer,
-                                            lib.ZSTD_e_end)
+        zresult = lib.ZSTD_compressStream2(self._cctx,
+                                           out_buffer,
+                                           in_buffer,
+                                           lib.ZSTD_e_end)
 
         if lib.ZSTD_isError(zresult):
             raise ZstdError('cannot compress: %s' %
@@ -901,7 +1259,7 b' class ZstdCompressor(object):'
         return ffi.buffer(out, out_buffer.pos)[:]
 
     def compressobj(self, size=-1):
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
@@ -923,7 +1281,7 b' class ZstdCompressor(object):'
         return cobj
 
     def chunker(self, size=-1, chunk_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
@@ -944,7 +1302,7 b' class ZstdCompressor(object):'
         if not hasattr(ofh, 'write'):
             raise ValueError('second argument must have a write() method')
 
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
@@ -976,10 +1334,10 b' class ZstdCompressor(object):'
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
-                zresult = lib.ZSTD_compress_generic(self._cctx,
-                                                    out_buffer,
-                                                    in_buffer,
-                                                    lib.ZSTD_e_continue)
+                zresult = lib.ZSTD_compressStream2(self._cctx,
+                                                   out_buffer,
+                                                   in_buffer,
+                                                   lib.ZSTD_e_continue)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd compress error: %s' %
                                     _zstd_error(zresult))
@@ -991,10 +1349,10 b' class ZstdCompressor(object):'
 
         # We've finished reading. Flush the compressor.
         while True:
-            zresult = lib.ZSTD_compress_generic(self._cctx,
-                                                out_buffer,
-                                                in_buffer,
-                                                lib.ZSTD_e_end)
+            zresult = lib.ZSTD_compressStream2(self._cctx,
+                                               out_buffer,
+                                               in_buffer,
+                                               lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
@@ -1011,7 +1369,7 b' class ZstdCompressor(object):'
 
     def stream_reader(self, source, size=-1,
                       read_size=COMPRESSION_RECOMMENDED_INPUT_SIZE):
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         try:
             size = len(source)
@@ -1026,20 +1384,22 b' class ZstdCompressor(object):'
             raise ZstdError('error setting source size: %s' %
                             _zstd_error(zresult))
 
-        return CompressionReader(self, source, read_size)
+        return ZstdCompressionReader(self, source, read_size)
 
     def stream_writer(self, writer, size=-1,
-                 write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+                 write_size=COMPRESSION_RECOMMENDED_OUTPUT_SIZE,
+                 write_return_read=False):
 
         if not hasattr(writer, 'write'):
             raise ValueError('must pass an object with a write() method')
 
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
 
-        return ZstdCompressionWriter(self, writer, size, write_size)
+        return ZstdCompressionWriter(self, writer, size, write_size,
+                                     write_return_read)
 
     write_to = stream_writer
 
@@ -1056,7 +1416,7 b' class ZstdCompressor(object):'
             raise ValueError('must pass an object with a read() method or '
                              'conforms to buffer protocol')
 
-        lib.ZSTD_CCtx_reset(self._cctx)
+        lib.ZSTD_CCtx_reset(self._cctx, lib.ZSTD_reset_session_only)
 
         if size < 0:
             size = lib.ZSTD_CONTENTSIZE_UNKNOWN
@@ -1104,8 +1464,8 b' class ZstdCompressor(object):'
             in_buffer.pos = 0
 
             while in_buffer.pos < in_buffer.size:
-                zresult = lib.ZSTD_compress_generic(self._cctx, out_buffer, in_buffer,
-                                                    lib.ZSTD_e_continue)
+                zresult = lib.ZSTD_compressStream2(self._cctx, out_buffer, in_buffer,
+                                                   lib.ZSTD_e_continue)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd compress error: %s' %
                                     _zstd_error(zresult))
@@ -1124,10 +1484,10 b' class ZstdCompressor(object):'
         # remains.
         while True:
             assert out_buffer.pos == 0
-            zresult = lib.ZSTD_compress_generic(self._cctx,
-                                                out_buffer,
-                                                in_buffer,
-                                                lib.ZSTD_e_end)
+            zresult = lib.ZSTD_compressStream2(self._cctx,
+                                               out_buffer,
+                                               in_buffer,
+                                               lib.ZSTD_e_end)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('error ending compression stream: %s' %
                                 _zstd_error(zresult))
@@ -1234,7 +1594,7 b' class ZstdCompressionDict(object):'
             cparams = ffi.new('ZSTD_compressionParameters')
             cparams.chainLog = compression_params.chain_log
             cparams.hashLog = compression_params.hash_log
-            cparams.searchLength = compression_params.min_match
+            cparams.minMatch = compression_params.min_match
             cparams.searchLog = compression_params.search_log
             cparams.strategy = compression_params.compression_strategy
             cparams.targetLength = compression_params.target_length
@@ -1345,6 +1705,10 b' class ZstdDecompressionObj(object):'
         out_buffer = ffi.new('ZSTD_outBuffer *')
 
         data_buffer = ffi.from_buffer(data)
+
+        if len(data_buffer) == 0:
+            return b''
+
         in_buffer.src = data_buffer
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
@@ -1357,8 +1721,8 b' class ZstdDecompressionObj(object):'
         chunks = []
 
         while True:
-            zresult = lib.ZSTD_decompress_generic(self._decompressor._dctx,
-                                                  out_buffer, in_buffer)
+            zresult = lib.ZSTD_decompressStream(self._decompressor._dctx,
+                                                out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd decompressor error: %s' %
                                 _zstd_error(zresult))
@@ -1378,12 +1742,16 b' class ZstdDecompressionObj(object):'
 
         return b''.join(chunks)
 
-
-class DecompressionReader(object):
-    def __init__(self, decompressor, source, read_size):
+    def flush(self, length=0):
+        pass
+
+
+class ZstdDecompressionReader(object):
+    def __init__(self, decompressor, source, read_size, read_across_frames):
         self._decompressor = decompressor
         self._source = source
         self._read_size = read_size
+        self._read_across_frames = bool(read_across_frames)
         self._entered = False
         self._closed = False
         self._bytes_decompressed = 0
@@ -1418,10 +1786,10 b' class DecompressionReader(object):'
         return True
 
     def readline(self):
-        raise NotImplementedError()
+        raise io.UnsupportedOperation()
 
     def readlines(self):
-        raise NotImplementedError()
+        raise io.UnsupportedOperation()
 
     def write(self, data):
         raise io.UnsupportedOperation()
@@ -1447,25 +1815,158 b' class DecompressionReader(object):'
         return self._bytes_decompressed
 
     def readall(self):
-        raise NotImplementedError()
+        chunks = []
+
+        while True:
+            chunk = self.read(1048576)
+            if not chunk:
+                break
+
+            chunks.append(chunk)
+
+        return b''.join(chunks)
 
     def __iter__(self):
-        raise NotImplementedError()
+        raise io.UnsupportedOperation()
 
     def __next__(self):
-        raise NotImplementedError()
+        raise io.UnsupportedOperation()
 
     next = __next__
 
-    def read(self, size):
+    def _read_input(self):
+        # We have data left over in the input buffer. Use it.
+        if self._in_buffer.pos < self._in_buffer.size:
+            return
+
+        # All input data exhausted. Nothing to do.
+        if self._finished_input:
+            return
+
+        # Else populate the input buffer from our source.
+        if hasattr(self._source, 'read'):
+            data = self._source.read(self._read_size)
+
+            if not data:
+                self._finished_input = True
+                return
+
+            self._source_buffer = ffi.from_buffer(data)
+            self._in_buffer.src = self._source_buffer
+            self._in_buffer.size = len(self._source_buffer)
+            self._in_buffer.pos = 0
+        else:
+            self._source_buffer = ffi.from_buffer(self._source)
+            self._in_buffer.src = self._source_buffer
+            self._in_buffer.size = len(self._source_buffer)
+            self._in_buffer.pos = 0
+
+    def _decompress_into_buffer(self, out_buffer):
+        """Decompress available input into an output buffer.
+
+        Returns True if data in output buffer should be emitted.
+        """
+        zresult = lib.ZSTD_decompressStream(self._decompressor._dctx,
+                                            out_buffer, self._in_buffer)
+
+        if self._in_buffer.pos == self._in_buffer.size:
+            self._in_buffer.src = ffi.NULL
+            self._in_buffer.pos = 0
+            self._in_buffer.size = 0
+            self._source_buffer = None
+
+            if not hasattr(self._source, 'read'):
+                self._finished_input = True
+
+        if lib.ZSTD_isError(zresult):
+            raise ZstdError('zstd decompress error: %s' %
+                            _zstd_error(zresult))
+
+        # Emit data if there is data AND either:
+        # a) output buffer is full (read amount is satisfied)
+        # b) we're at end of a frame and not in frame spanning mode
+        return (out_buffer.pos and
+                (out_buffer.pos == out_buffer.size or
+                 zresult == 0 and not self._read_across_frames))
+
+    def read(self, size=-1):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if size < -1:
+            raise ValueError('cannot read negative amounts less than -1')
+
+        if size == -1:
+            # This is recursive. But it gets the job done.
+            return self.readall()
+
+        if self._finished_output or size == 0:
+            return b''
+
+        # We /could/ call into readinto() here. But that introduces more
+        # overhead.
+        dst_buffer = ffi.new('char[]', size)
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dst_buffer
+        out_buffer.size = size
+        out_buffer.pos = 0
+
+        self._read_input()
+        if self._decompress_into_buffer(out_buffer):
+            self._bytes_decompressed += out_buffer.pos
+            return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+        while not self._finished_input:
+            self._read_input()
+            if self._decompress_into_buffer(out_buffer):
+                self._bytes_decompressed += out_buffer.pos
+                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+        self._bytes_decompressed += out_buffer.pos
+        return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
+
+    def readinto(self, b):
         if self._closed:
             raise ValueError('stream is closed')
 
         if self._finished_output:
+            return 0
+
+        # TODO use writable=True once we require CFFI >= 1.12.
+        dest_buffer = ffi.from_buffer(b)
+        ffi.memmove(b, b'', 0)
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dest_buffer
+        out_buffer.size = len(dest_buffer)
+        out_buffer.pos = 0
+
+        self._read_input()
+        if self._decompress_into_buffer(out_buffer):
+            self._bytes_decompressed += out_buffer.pos
+            return out_buffer.pos
+
+        while not self._finished_input:
+            self._read_input()
+            if self._decompress_into_buffer(out_buffer):
+                self._bytes_decompressed += out_buffer.pos
+                return out_buffer.pos
+
+        self._bytes_decompressed += out_buffer.pos
+        return out_buffer.pos
+
+    def read1(self, size=-1):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if size < -1:
+            raise ValueError('cannot read negative amounts less than -1')
+
+        if self._finished_output or size == 0:
             return b''
 
-        if size < 1:
-            raise ValueError('cannot read negative or size 0 amounts')
+        # -1 returns arbitrary number of bytes.
+        if size == -1:
+            size = DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE
 
         dst_buffer = ffi.new('char[]', size)
         out_buffer = ffi.new('ZSTD_outBuffer *')
@@ -1473,64 +1974,46 b' class DecompressionReader(object):'
         out_buffer.size = size
         out_buffer.pos = 0
 
-        def decompress():
-            zresult = lib.ZSTD_decompress_generic(self._decompressor._dctx,
-                                                  out_buffer, self._in_buffer)
-
-            if self._in_buffer.pos == self._in_buffer.size:
-                self._in_buffer.src = ffi.NULL
-                self._in_buffer.pos = 0
-                self._in_buffer.size = 0
-                self._source_buffer = None
-
-                if not hasattr(self._source, 'read'):
-                    self._finished_input = True
-
-            if lib.ZSTD_isError(zresult):
-                raise ZstdError('zstd decompress error: %s',
-                                _zstd_error(zresult))
-            elif zresult == 0:
-                self._finished_output = True
-
-            if out_buffer.pos and out_buffer.pos == out_buffer.size:
-                self._bytes_decompressed += out_buffer.size
-                return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
-
-        def get_input():
-            if self._finished_input:
-                return
-
-            if hasattr(self._source, 'read'):
-                data = self._source.read(self._read_size)
-
-                if not data:
-                    self._finished_input = True
-                    return
-
-                self._source_buffer = ffi.from_buffer(data)
-                self._in_buffer.src = self._source_buffer
-                self._in_buffer.size = len(self._source_buffer)
-                self._in_buffer.pos = 0
-            else:
-                self._source_buffer = ffi.from_buffer(self._source)
-                self._in_buffer.src = self._source_buffer
-                self._in_buffer.size = len(self._source_buffer)
-                self._in_buffer.pos = 0
-
-        get_input()
-        result = decompress()
-        if result:
-            return result
-
+        # read1() dictates that we can perform at most 1 call to underlying
+        # stream to get input. However, we can't satisfy this restriction with
+        # decompression because not all input generates output. So we allow
+        # multiple read(). But unlike read(), we stop once we have any output.
         while not self._finished_input:
-            get_input()
-            result = decompress()
-            if result:
-                return result
+            self._read_input()
+            self._decompress_into_buffer(out_buffer)
+
+            if out_buffer.pos:
+                break
 
         self._bytes_decompressed += out_buffer.pos
         return ffi.buffer(out_buffer.dst, out_buffer.pos)[:]
 
+    def readinto1(self, b):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        if self._finished_output:
+            return 0
+
+        # TODO use writable=True once we require CFFI >= 1.12.
+        dest_buffer = ffi.from_buffer(b)
+        ffi.memmove(b, b'', 0)
+
+        out_buffer = ffi.new('ZSTD_outBuffer *')
+        out_buffer.dst = dest_buffer
+        out_buffer.size = len(dest_buffer)
+        out_buffer.pos = 0
+
+        while not self._finished_input and not self._finished_output:
+            self._read_input()
+            self._decompress_into_buffer(out_buffer)
+
+            if out_buffer.pos:
+                break
+
+        self._bytes_decompressed += out_buffer.pos
+        return out_buffer.pos
+
     def seek(self, pos, whence=os.SEEK_SET):
         if self._closed:
             raise ValueError('stream is closed')
@@ -1569,34 +2052,108 b' class DecompressionReader(object):'
         return self._bytes_decompressed
 
 class ZstdDecompressionWriter(object):
-    def __init__(self, decompressor, writer, write_size):
+    def __init__(self, decompressor, writer, write_size, write_return_read):
+        decompressor._ensure_dctx()
+
         self._decompressor = decompressor
         self._writer = writer
         self._write_size = write_size
+        self._write_return_read = bool(write_return_read)
         self._entered = False
+        self._closed = False
 
     def __enter__(self):
+        if self._closed:
+            raise ValueError('stream is closed')
+
         if self._entered:
             raise ZstdError('cannot __enter__ multiple times')
 
-        self._decompressor._ensure_dctx()
         self._entered = True
 
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._entered = False
+        self.close()
 
     def memory_size(self):
-        if not self._decompressor._dctx:
-            raise ZstdError('cannot determine size of inactive decompressor '
-                            'call when context manager is active')
-
         return lib.ZSTD_sizeof_DCtx(self._decompressor._dctx)
 
+    def close(self):
+        if self._closed:
+            return
+
+        try:
+            self.flush()
+        finally:
+            self._closed = True
+
+        f = getattr(self._writer, 'close', None)
+        if f:
+            f()
+
+    @property
+    def closed(self):
+        return self._closed
+
+    def fileno(self):
+        f = getattr(self._writer, 'fileno', None)
+        if f:
+            return f()
+        else:
+            raise OSError('fileno not available on underlying writer')
+
+    def flush(self):
+        if self._closed:
+            raise ValueError('stream is closed')
+
+        f = getattr(self._writer, 'flush', None)
+        if f:
+            return f()
+
+    def isatty(self):
+        return False
+
+    def readable(self):
+        return False
+
+    def readline(self, size=-1):
+        raise io.UnsupportedOperation()
+
+    def readlines(self, hint=-1):
+        raise io.UnsupportedOperation()
+
+    def seek(self, offset, whence=None):
+        raise io.UnsupportedOperation()
+
+    def seekable(self):
+        return False
+
+    def tell(self):
+        raise io.UnsupportedOperation()
+
+    def truncate(self, size=None):
+        raise io.UnsupportedOperation()
+
+    def writable(self):
+        return True
+
+    def writelines(self, lines):
+        raise io.UnsupportedOperation()
+
+    def read(self, size=-1):
+        raise io.UnsupportedOperation()
+
+    def readall(self):
+        raise io.UnsupportedOperation()
+
+    def readinto(self, b):
+        raise io.UnsupportedOperation()
+
     def write(self, data):
-        if not self._entered:
-            raise ZstdError('write must be called from an active context manager')
+        if self._closed:
+            raise ValueError('stream is closed')
 
         total_write = 0
 
@@ -1616,7 +2173,7 b' class ZstdDecompressionWriter(object):'
         dctx = self._decompressor._dctx
 
         while in_buffer.pos < in_buffer.size:
-            zresult = lib.ZSTD_decompress_generic(dctx, out_buffer, in_buffer)
+            zresult = lib.ZSTD_decompressStream(dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('zstd decompress error: %s' %
                                 _zstd_error(zresult))
@@ -1626,7 +2183,10 b' class ZstdDecompressionWriter(object):'
                 total_write += out_buffer.pos
                 out_buffer.pos = 0
 
-        return total_write
+        if self._write_return_read:
+            return in_buffer.pos
+        else:
+            return total_write
 
 
 class ZstdDecompressor(object):
@@ -1684,7 +2244,7 b' class ZstdDecompressor(object):'
         in_buffer.size = len(data_buffer)
         in_buffer.pos = 0
 
-        zresult = lib.ZSTD_decompress_generic(self._dctx, out_buffer, in_buffer)
+        zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('decompression error: %s' %
                             _zstd_error(zresult))
@@ -1696,9 +2256,10 b' class ZstdDecompressor(object):'
 
         return ffi.buffer(result_buffer, out_buffer.pos)[:]
 
-    def stream_reader(self, source, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE):
+    def stream_reader(self, source, read_size=DECOMPRESSION_RECOMMENDED_INPUT_SIZE,
+                      read_across_frames=False):
         self._ensure_dctx()
-        return DecompressionReader(self, source, read_size)
+        return ZstdDecompressionReader(self, source, read_size, read_across_frames)
 
     def decompressobj(self, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
         if write_size < 1:
@@ -1767,7 +2328,7 b' class ZstdDecompressor(object):'
             while in_buffer.pos < in_buffer.size:
                 assert out_buffer.pos == 0
 
-                zresult = lib.ZSTD_decompress_generic(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd decompress error: %s' %
                                     _zstd_error(zresult))
@@ -1787,11 +2348,13 b' class ZstdDecompressor(object):'
 
     read_from = read_to_iter
 
-    def stream_writer(self, writer, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
+    def stream_writer(self, writer, write_size=DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
+                      write_return_read=False):
         if not hasattr(writer, 'write'):
             raise ValueError('must pass an object with a write() method')
 
-        return ZstdDecompressionWriter(self, writer, write_size)
+        return ZstdDecompressionWriter(self, writer, write_size,
+                                       write_return_read)
 
     write_to = stream_writer
 
@@ -1829,7 +2392,7 b' class ZstdDecompressor(object):'
 
             # Flush all read data to output.
             while in_buffer.pos < in_buffer.size:
-                zresult = lib.ZSTD_decompress_generic(self._dctx, out_buffer, in_buffer)
+                zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
                 if lib.ZSTD_isError(zresult):
                     raise ZstdError('zstd decompressor error: %s' %
                                     _zstd_error(zresult))
@@ -1881,7 +2444,7 b' class ZstdDecompressor(object):'
         in_buffer.size = len(chunk_buffer)
         in_buffer.pos = 0
 
-        zresult = lib.ZSTD_decompress_generic(self._dctx, out_buffer, in_buffer)
+        zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
         if lib.ZSTD_isError(zresult):
             raise ZstdError('could not decompress chunk 0: %s' %
                             _zstd_error(zresult))
@@ -1918,7 +2481,7 b' class ZstdDecompressor(object):'
             in_buffer.size = len(chunk_buffer)
             in_buffer.pos = 0
 
-            zresult = lib.ZSTD_decompress_generic(self._dctx, out_buffer, in_buffer)
+            zresult = lib.ZSTD_decompressStream(self._dctx, out_buffer, in_buffer)
             if lib.ZSTD_isError(zresult):
                 raise ZstdError('could not decompress chunk %d: %s' %
                                 _zstd_error(zresult))
@@ -1931,7 +2494,7 b' class ZstdDecompressor(object):'
         return ffi.buffer(last_buffer, len(last_buffer))[:]
 
     def _ensure_dctx(self, load_dict=True):
-        lib.ZSTD_DCtx_reset(self._dctx)
+        lib.ZSTD_DCtx_reset(self._dctx, lib.ZSTD_reset_session_only)
 
         if self._max_window_size:
             zresult = lib.ZSTD_DCtx_setMaxWindowSize(self._dctx,

contrib/python-zstandard/zstd.c

@@ -210,7 +210,7 b' void zstd_module_init(PyObject* m) {'
 	   We detect this mismatch here and refuse to load the module if this
 	   scenario is detected.
 	*/
-	if (ZSTD_VERSION_NUMBER != 10306 || ZSTD_versionNumber() != 10306) {
+	if (ZSTD_VERSION_NUMBER != 10308 || ZSTD_versionNumber() != 10308) {
 		PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
 		return;
 	}

contrib/python-zstandard/zstd/common/bitstream.h

@@ -339,17 +339,10 b' MEM_STATIC size_t BIT_getUpperBits(size_'
 
 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
-#  if defined(__x86_64__)
-    if (sizeof(bitContainer)==8)
-        return _bextr_u64(bitContainer, start, nbBits);
-    else
-#  endif
-        return _bextr_u32(bitContainer, start, nbBits);
-#else
+    U32 const regMask = sizeof(bitContainer)*8 - 1;
+    /* if start > regMask, bitstream is corrupted, and result is undefined */
     assert(nbBits < BIT_MASK_SIZE);
-    return (bitContainer >> start) & BIT_mask[nbBits];
-#endif
+    return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
 }
 
 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
@@ -366,9 +359,13 b' MEM_STATIC size_t BIT_getLowerBits(size_'
  * @return : value extracted */
 MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
+    /* arbitrate between double-shift and shift+mask */
+#if 1
+    /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
+     * bitstream is likely corrupted, and result is undefined */
     return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
 #else
+    /* this code path is slower on my os-x laptop */
     U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
     return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
 #endif
@@ -392,7 +389,7 b' MEM_STATIC void BIT_skipBits(BIT_DStream'
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
-MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
@@ -401,7 +398,7 b' MEM_STATIC size_t BIT_readBits(BIT_DStre'
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works only if nbBits >= 1 */
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);

contrib/python-zstandard/zstd/common/compiler.h

@@ -15,6 +15,8 b''
 *  Compiler specifics
 *********************************************************/
 /* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
 #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
 #else
@@ -29,6 +31,13 b''
 #  define FORCE_INLINE_ATTR
 #endif
 
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
 /**
  * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
  * parameters. They must be inlined for the compiler to elimininate the constant
@@ -89,23 +98,21 b''
 #endif
 
 /* prefetch
- * can be disabled, by declaring NO_PREFETCH macro
- * All prefetch invocations use a single default locality 2,
- * generating instruction prefetcht1,
- * which, according to Intel, means "load data into L2 cache".
- * This is a good enough "middle ground" for the time being,
- * though in theory, it would be better to specialize locality depending on data being prefetched.
- * Tests could not determine any sensible difference based on locality value. */
+ * can be disabled, by declaring NO_PREFETCH build macro */
 #if defined(NO_PREFETCH)
-#  define PREFETCH(ptr)     (void)(ptr)  /* disabled */
+#  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
+#  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
 #else
 #  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
-#    define PREFETCH(ptr)   _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
 #  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
-#    define PREFETCH(ptr)   __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
 #  else
-#    define PREFETCH(ptr)   (void)(ptr)  /* disabled */
+#    define PREFETCH_L1(ptr) (void)(ptr)  /* disabled */
+#    define PREFETCH_L2(ptr) (void)(ptr)  /* disabled */
 #  endif
 #endif  /* NO_PREFETCH */
 
@@ -116,7 +123,7 b''
     size_t const _size = (size_t)(s);     \
     size_t _pos;                          \
     for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) {  \
-        PREFETCH(_ptr + _pos);            \
+        PREFETCH_L2(_ptr + _pos);         \
     }                                     \
 }
 

contrib/python-zstandard/zstd/common/cpu.h

@@ -78,7 +78,7 b' MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void)'
       __asm__(
           "pushl %%ebx\n\t"
           "cpuid\n\t"
-          "movl %%ebx, %%eax\n\r"
+          "movl %%ebx, %%eax\n\t"
           "popl %%ebx"
           : "=a"(f7b), "=c"(f7c)
           : "a"(7), "c"(0)

contrib/python-zstandard/zstd/common/debug.h

@@ -57,9 +57,9 b' extern "C" {'
 #endif
 
 
-/* static assert is triggered at compile time, leaving no runtime artefact,
- * but can only work with compile-time constants.
- * This variant can only be used inside a function. */
+/* static assert is triggered at compile time, leaving no runtime artefact.
+ * static assert only works with compile-time constants.
+ * Also, this variant can only be used inside a function. */
 #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
 
 
@@ -70,9 +70,19 b' extern "C" {'
 #  define DEBUGLEVEL 0
 #endif
 
+
+/* DEBUGFILE can be defined externally,
+ * typically through compiler command line.
+ * note : currently useless.
+ * Value must be stderr or stdout */
+#ifndef DEBUGFILE
+#  define DEBUGFILE stderr
+#endif
+
+
 /* recommended values for DEBUGLEVEL :
- * 0 : no debug, all run-time functions disabled
- * 1 : no display, enables assert() only
+ * 0 : release mode, no debug, all run-time checks disabled
+ * 1 : enables assert() only, no display
  * 2 : reserved, for currently active debug path
  * 3 : events once per object lifetime (CCtx, CDict, etc.)
  * 4 : events once per frame
@@ -81,7 +91,7 b' extern "C" {'
  * 7+: events at every position (*very* verbose)
  *
  * It's generally inconvenient to output traces > 5.
- * In which case, it's possible to selectively enable higher verbosity levels
+ * In which case, it's possible to selectively trigger high verbosity levels
  * by modifying g_debug_level.
  */
 
@@ -95,11 +105,12 b' extern "C" {'
 
 #if (DEBUGLEVEL>=2)
 #  include <stdio.h>
-extern int g_debuglevel; /* here, this variable is only declared,
-                           it actually lives in debug.c,
-                           and is shared by the whole process.
-                           It's typically used to enable very verbose levels
-                           on selective conditions (such as position in src) */
+extern int g_debuglevel; /* the variable is only declared,
+                            it actually lives in debug.c,
+                            and is shared by the whole process.
+                            It's not thread-safe.
+                            It's useful when enabling very verbose levels
+                            on selective conditions (such as position in src) */
 
 #  define RAWLOG(l, ...) {                                      \
                 if (l<=g_debuglevel) {                          \

contrib/python-zstandard/zstd/common/error_private.c

@@ -14,6 +14,10 b''
 
 const char* ERR_getErrorString(ERR_enum code)
 {
+#ifdef ZSTD_STRIP_ERROR_STRINGS
+    (void)code;
+    return "Error strings stripped";
+#else
     static const char* const notErrorCode = "Unspecified error code";
     switch( code )
     {
@@ -39,10 +43,12 b' const char* ERR_getErrorString(ERR_enum '
     case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
+    case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
         /* following error codes are not stable and may be removed or changed in a future version */
     case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
+#endif
 }

contrib/python-zstandard/zstd/common/fse.h

@@ -512,7 +512,7 b' MEM_STATIC void FSE_initCState(FSE_CStat'
     const U32 tableLog = MEM_read16(ptr);
     statePtr->value = (ptrdiff_t)1<<tableLog;
     statePtr->stateTable = u16ptr+2;
-    statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
+    statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
     statePtr->stateLog = tableLog;
 }
 
@@ -531,7 +531,7 b' MEM_STATIC void FSE_initCState2(FSE_CSta'
     }
 }
 
-MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
+MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
 {
     FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
     const U16* const stateTable = (const U16*)(statePtr->stateTable);

contrib/python-zstandard/zstd/common/huf.h

@@ -173,15 +173,19 b' typedef U32 HUF_DTable;'
 *  Advanced decompression functions
 ******************************************/
 size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
+#endif
 
 size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
 size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
+#endif
 
 
 /* ****************************************
@@ -228,7 +232,7 b' size_t HUF_compress4X_repeat(void* dst, '
 #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
 size_t HUF_buildCTable_wksp (HUF_CElt* tree,
-                       const U32* count, U32 maxSymbolValue, U32 maxNbBits,
+                       const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
                              void* workSpace, size_t wkspSize);
 
 /*! HUF_readStats() :
@@ -277,14 +281,22 b' U32 HUF_selectDecoder (size_t dstSize, s'
 #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
 
+#ifndef HUF_FORCE_DECOMPRESS_X2
 size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
 size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
 size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
+#endif
 
 size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+#ifndef HUF_FORCE_DECOMPRESS_X2
 size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+#endif
 
 
 /* ====================== */
@@ -306,24 +318,36 b' size_t HUF_compress1X_repeat(void* dst, '
                        HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
 
 size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
+#endif
 
 size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
 size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
+#ifndef HUF_FORCE_DECOMPRESS_X2
 size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
 size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
+#endif
 
 size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);   /**< automatic selection of sing or double symbol decoder, based on DTable */
+#ifndef HUF_FORCE_DECOMPRESS_X2
 size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+#endif
+#ifndef HUF_FORCE_DECOMPRESS_X1
 size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+#endif
 
 /* BMI2 variants.
  * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
  */
 size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+#ifndef HUF_FORCE_DECOMPRESS_X2
 size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+#endif
 size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
 size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
 

contrib/python-zstandard/zstd/common/mem.h

@@ -39,6 +39,10 b' extern "C" {'
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif
 
+#ifndef __has_builtin
+#  define __has_builtin(x) 0  /* compat. with non-clang compilers */
+#endif
+
 /* code only tested on 32 and 64 bits systems */
 #define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
 MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
@@ -198,7 +202,8 b' MEM_STATIC U32 MEM_swap32(U32 in)'
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_ulong(in);
-#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
     return __builtin_bswap32(in);
 #else
     return  ((in << 24) & 0xff000000 ) |
@@ -212,7 +217,8 b' MEM_STATIC U64 MEM_swap64(U64 in)'
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_uint64(in);
-#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+  || (defined(__clang__) && __has_builtin(__builtin_bswap64))
     return __builtin_bswap64(in);
 #else
     return  ((in << 56) & 0xff00000000000000ULL) |

contrib/python-zstandard/zstd/common/pool.c

@@ -88,8 +88,8 b' static void* POOL_thread(void* opaque) {'
             ctx->numThreadsBusy++;
             ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
             /* Unlock the mutex, signal a pusher, and run the job */
+            ZSTD_pthread_cond_signal(&ctx->queuePushCond);
             ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
-            ZSTD_pthread_cond_signal(&ctx->queuePushCond);
 
             job.function(job.opaque);
 

contrib/python-zstandard/zstd/common/zstd_common.c

@@ -30,8 +30,10 b' const char* ZSTD_versionString(void) { r'
 /*-****************************************
 *  ZSTD Error Management
 ******************************************/
+#undef ZSTD_isError   /* defined within zstd_internal.h */
 /*! ZSTD_isError() :
- *  tells if a return value is an error code */
+ *  tells if a return value is an error code
+ *  symbol is required for external callers */
 unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
 
 /*! ZSTD_getErrorName() :

contrib/python-zstandard/zstd/common/zstd_errors.h

@@ -72,6 +72,7 b' typedef enum {'
   ZSTD_error_workSpace_tooSmall= 66,
   ZSTD_error_dstSize_tooSmall = 70,
   ZSTD_error_srcSize_wrong    = 72,
+  ZSTD_error_dstBuffer_null   = 74,
   /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
   ZSTD_error_frameIndex_tooLarge = 100,
   ZSTD_error_seekableIO          = 102,

contrib/python-zstandard/zstd/common/zstd_internal.h

@@ -41,6 +41,9 b' extern "C" {'
 
 /* ---- static assert (debug) --- */
 #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
+#define ZSTD_isError ERR_isError   /* for inlining */
+#define FSE_isError  ERR_isError
+#define HUF_isError  ERR_isError
 
 
 /*-*************************************
@@ -75,7 +78,6 b' static const U32 repStartValue[ZSTD_REP_'
 #define BIT0   1
 
 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
-#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
 static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 
@@ -242,7 +244,7 b' typedef struct {'
     blockType_e blockType;
     U32 lastBlock;
     U32 origSize;
-} blockProperties_t;
+} blockProperties_t;   /* declared here for decompress and fullbench */
 
 /*! ZSTD_getcBlockSize() :
  *  Provides the size of compressed block from block header `src` */
@@ -250,6 +252,13 b' typedef struct {'
 size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
                           blockProperties_t* bpPtr);
 
+/*! ZSTD_decodeSeqHeaders() :
+ *  decode sequence header from src */
+/* Used by: decompress, fullbench (does not get its definition from here) */
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
+                       const void* src, size_t srcSize);
+
+
 #if defined (__cplusplus)
 }
 #endif

contrib/python-zstandard/zstd/compress/fse_compress.c

@@ -115,7 +115,7 b' size_t FSE_buildCTable_wksp(FSE_CTable* '
     /* symbol start positions */
     {   U32 u;
         cumul[0] = 0;
-        for (u=1; u<=maxSymbolValue+1; u++) {
+        for (u=1; u <= maxSymbolValue+1; u++) {
             if (normalizedCounter[u-1]==-1) {  /* Low proba symbol */
                 cumul[u] = cumul[u-1] + 1;
                 tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
@@ -658,7 +658,7 b' size_t FSE_compress_wksp (void* dst, siz'
     BYTE* op = ostart;
     BYTE* const oend = ostart + dstSize;
 
-    U32   count[FSE_MAX_SYMBOL_VALUE+1];
+    unsigned count[FSE_MAX_SYMBOL_VALUE+1];
     S16   norm[FSE_MAX_SYMBOL_VALUE+1];
     FSE_CTable* CTable = (FSE_CTable*)workSpace;
     size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
@@ -672,7 +672,7 b' size_t FSE_compress_wksp (void* dst, siz'
     if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
 
     /* Scan input and build symbol stats */
-    {   CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) );
+    {   CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
         if (maxCount == srcSize) return 1;   /* only a single symbol in src : rle */
         if (maxCount == 1) return 0;         /* each symbol present maximum once => not compressible */
         if (maxCount < (srcSize >> 7)) return 0;   /* Heuristic : not compressible enough */