zstandard: vendor python-zstandard 0.9.0

This was just released. It features a number of goodies. More info at
https://gregoryszorc.com/blog/2018/04/09/release-of-python-zstandard-0.9/.

The clang-format ignore list was updated to reflect the new source
of files.

The project contains a vendored copy of zstandard 1.3.4. The old
version was 1.1.3. One of the changes between those versions is that
zstandard is now dual licensed BSD + GPLv2 and the patent rights grant
has been removed. Good riddance.

The API should be backwards compatible. So no changes in core
should be needed. However, there were a number of changes in the
library that we'll want to adapt to. Those will be addressed in
subsequent commits.

Differential Revision: https://phab.mercurial-scm.org/D3198

contrib/python-zstandard/c-ext/compressionreader.c

@@ -0,0 +1,405 b''
+/**
+* Copyright (c) 2017-present, Gregory Szorc
+* All rights reserved.
+*
+* This software may be modified and distributed under the terms
+* of the BSD license. See the LICENSE file for details.
+*/
+
+#include "python-zstandard.h"
+
+extern PyObject* ZstdError;
+
+static void set_unsupported_operation(void) {
+	PyObject* iomod;
+	PyObject* exc;
+
+	iomod = PyImport_ImportModule("io");
+	if (NULL == iomod) {
+		return;
+	}
+
+	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
+	if (NULL == exc) {
+		Py_DECREF(iomod);
+		return;
+	}
+
+	PyErr_SetNone(exc);
+	Py_DECREF(exc);
+	Py_DECREF(iomod);
+}
+
+static void reader_dealloc(ZstdCompressionReader* self) {
+	Py_XDECREF(self->compressor);
+	Py_XDECREF(self->reader);
+
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+		memset(&self->buffer, 0, sizeof(self->buffer));
+	}
+
+	PyObject_Del(self);
+}
+
+static ZstdCompressionReader* reader_enter(ZstdCompressionReader* self) {
+	size_t zresult;
+
+	if (self->entered) {
+		PyErr_SetString(PyExc_ValueError, "cannot __enter__ multiple times");
+		return NULL;
+	}
+
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->compressor->cctx, self->sourceSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
+	}
+
+	self->entered = 1;
+
+	Py_INCREF(self);
+	return self;
+}
+
+static PyObject* reader_exit(ZstdCompressionReader* self, PyObject* args) {
+	PyObject* exc_type;
+	PyObject* exc_value;
+	PyObject* exc_tb;
+
+	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
+		return NULL;
+	}
+
+	self->entered = 0;
+	self->closed = 1;
+
+	/* Release resources associated with source. */
+	Py_CLEAR(self->reader);
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+		memset(&self->buffer, 0, sizeof(self->buffer));
+	}
+
+    Py_CLEAR(self->compressor);
+
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_readable(ZstdCompressionReader* self) {
+	Py_RETURN_TRUE;
+}
+
+static PyObject* reader_writable(ZstdCompressionReader* self) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_seekable(ZstdCompressionReader* self) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_readline(PyObject* self, PyObject* args) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyObject* reader_readlines(PyObject* self, PyObject* args) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyObject* reader_write(PyObject* self, PyObject* args) {
+	PyErr_SetString(PyExc_OSError, "stream is not writable");
+	return NULL;
+}
+
+static PyObject* reader_writelines(PyObject* self, PyObject* args) {
+	PyErr_SetString(PyExc_OSError, "stream is not writable");
+	return NULL;
+}
+
+static PyObject* reader_isatty(PyObject* self) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_flush(PyObject* self) {
+	Py_RETURN_NONE;
+}
+
+static PyObject* reader_close(ZstdCompressionReader* self) {
+	self->closed = 1;
+	Py_RETURN_NONE;
+}
+
+static PyObject* reader_closed(ZstdCompressionReader* self) {
+	if (self->closed) {
+		Py_RETURN_TRUE;
+	}
+	else {
+		Py_RETURN_FALSE;
+	}
+}
+
+static PyObject* reader_tell(ZstdCompressionReader* self) {
+	/* TODO should this raise OSError since stream isn't seekable? */
+	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
+}
+
+static PyObject* reader_read(ZstdCompressionReader* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"size",
+		NULL
+	};
+
+	Py_ssize_t size = -1;
+	PyObject* result = NULL;
+	char* resultBuffer;
+	Py_ssize_t resultSize;
+	size_t zresult;
+	size_t oldPos;
+
+	if (!self->entered) {
+		PyErr_SetString(ZstdError, "read() must be called from an active context manager");
+		return NULL;
+	}
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "n", kwlist, &size)) {
+		return NULL;
+	}
+
+	if (size < 1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative or size 0 amounts");
+		return NULL;
+	}
+
+	result = PyBytes_FromStringAndSize(NULL, size);
+	if (NULL == result) {
+		return NULL;
+	}
+
+	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
+
+	self->output.dst = resultBuffer;
+	self->output.size = resultSize;
+	self->output.pos = 0;
+
+readinput:
+
+	/* If we have data left over, consume it. */
+	if (self->input.pos < self->input.size) {
+		oldPos = self->output.pos;
+
+		Py_BEGIN_ALLOW_THREADS
+		zresult = ZSTD_compress_generic(self->compressor->cctx,
+			&self->output, &self->input, ZSTD_e_continue);
+
+		Py_END_ALLOW_THREADS
+
+		self->bytesCompressed += self->output.pos - oldPos;
+
+		/* Input exhausted. Clear out state tracking. */
+		if (self->input.pos == self->input.size) {
+			memset(&self->input, 0, sizeof(self->input));
+			Py_CLEAR(self->readResult);
+
+			if (self->buffer.buf) {
+				self->finishedInput = 1;
+			}
+		}
+
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
+			return NULL;
+		}
+
+		if (self->output.pos) {
+			/* If no more room in output, emit it. */
+			if (self->output.pos == self->output.size) {
+				memset(&self->output, 0, sizeof(self->output));
+				return result;
+			}
+
+			/*
+			 * There is room in the output. We fall through to below, which will either
+			 * get more input for us or will attempt to end the stream.
+			 */
+		}
+
+		/* Fall through to gather more input. */
+	}
+
+	if (!self->finishedInput) {
+		if (self->reader) {
+			Py_buffer buffer;
+
+			assert(self->readResult == NULL);
+			self->readResult = PyObject_CallMethod(self->reader, "read",
+				"k", self->readSize);
+			if (self->readResult == NULL) {
+				return NULL;
+			}
+
+			memset(&buffer, 0, sizeof(buffer));
+
+			if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
+				return NULL;
+			}
+
+			/* EOF */
+			if (0 == buffer.len) {
+				self->finishedInput = 1;
+				Py_CLEAR(self->readResult);
+			}
+			else {
+				self->input.src = buffer.buf;
+				self->input.size = buffer.len;
+				self->input.pos = 0;
+			}
+
+			PyBuffer_Release(&buffer);
+		}
+		else {
+			assert(self->buffer.buf);
+
+			self->input.src = self->buffer.buf;
+			self->input.size = self->buffer.len;
+			self->input.pos = 0;
+		}
+	}
+
+	if (self->input.size) {
+		goto readinput;
+	}
+
+	/* Else EOF */
+	oldPos = self->output.pos;
+
+	zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+		&self->input, ZSTD_e_end);
+
+	self->bytesCompressed += self->output.pos - oldPos;
+
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error ending compression stream: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
+	}
+
+	assert(self->output.pos);
+
+	if (0 == zresult) {
+		self->finishedOutput = 1;
+	}
+
+	if (safe_pybytes_resize(&result, self->output.pos)) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+
+	memset(&self->output, 0, sizeof(self->output));
+
+	return result;
+}
+
+static PyObject* reader_readall(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* reader_iter(PyObject* self) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyObject* reader_iternext(PyObject* self) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyMethodDef reader_methods[] = {
+	{ "__enter__", (PyCFunction)reader_enter, METH_NOARGS,
+	PyDoc_STR("Enter a compression context") },
+	{ "__exit__", (PyCFunction)reader_exit, METH_VARARGS,
+	PyDoc_STR("Exit a compression context") },
+	{ "close", (PyCFunction)reader_close, METH_NOARGS,
+	PyDoc_STR("Close the stream so it cannot perform any more operations") },
+	{ "closed", (PyCFunction)reader_closed, METH_NOARGS,
+	PyDoc_STR("Whether stream is closed") },
+	{ "flush", (PyCFunction)reader_flush, METH_NOARGS, PyDoc_STR("no-ops") },
+	{ "isatty", (PyCFunction)reader_isatty, METH_NOARGS, PyDoc_STR("Returns False") },
+	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
+	PyDoc_STR("Returns True") },
+	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read compressed data") },
+	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
+	{ "readline", (PyCFunction)reader_readline, METH_VARARGS, PyDoc_STR("Not implemented") },
+	{ "readlines", (PyCFunction)reader_readlines, METH_VARARGS, PyDoc_STR("Not implemented") },
+	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,
+	PyDoc_STR("Returns False") },
+	{ "tell", (PyCFunction)reader_tell, METH_NOARGS,
+	PyDoc_STR("Returns current number of bytes compressed") },
+	{ "writable", (PyCFunction)reader_writable, METH_NOARGS,
+	PyDoc_STR("Returns False") },
+	{ "write", reader_write, METH_VARARGS, PyDoc_STR("Raises OSError") },
+	{ "writelines", reader_writelines, METH_VARARGS, PyDoc_STR("Not implemented") },
+	{ NULL, NULL }
+};
+
+PyTypeObject ZstdCompressionReaderType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	"zstd.ZstdCompressionReader", /* tp_name */
+	sizeof(ZstdCompressionReader), /* tp_basicsize */
+	0, /* tp_itemsize */
+	(destructor)reader_dealloc, /* tp_dealloc */
+	0, /* tp_print */
+	0, /* tp_getattr */
+	0, /* tp_setattr */
+	0, /* tp_compare */
+	0, /* tp_repr */
+	0, /* tp_as_number */
+	0, /* tp_as_sequence */
+	0, /* tp_as_mapping */
+	0, /* tp_hash */
+	0, /* tp_call */
+	0, /* tp_str */
+	0, /* tp_getattro */
+	0, /* tp_setattro */
+	0, /* tp_as_buffer */
+	Py_TPFLAGS_DEFAULT, /* tp_flags */
+	0, /* tp_doc */
+	0, /* tp_traverse */
+	0, /* tp_clear */
+	0, /* tp_richcompare */
+	0, /* tp_weaklistoffset */
+	reader_iter, /* tp_iter */
+	reader_iternext, /* tp_iternext */
+	reader_methods, /* tp_methods */
+	0, /* tp_members */
+	0, /* tp_getset */
+	0, /* tp_base */
+	0, /* tp_dict */
+	0, /* tp_descr_get */
+	0, /* tp_descr_set */
+	0, /* tp_dictoffset */
+	0, /* tp_init */
+	0, /* tp_alloc */
+	PyType_GenericNew, /* tp_new */
+};
+
+void compressionreader_module_init(PyObject* mod) {
+	/* TODO make reader a sub-class of io.RawIOBase */
+
+	Py_TYPE(&ZstdCompressionReaderType) = &PyType_Type;
+	if (PyType_Ready(&ZstdCompressionReaderType) < 0) {
+		return;
+	}
+}

contrib/python-zstandard/c-ext/decompressionreader.c

@@ -0,0 +1,459 b''
+/**
+* Copyright (c) 2017-present, Gregory Szorc
+* All rights reserved.
+*
+* This software may be modified and distributed under the terms
+* of the BSD license. See the LICENSE file for details.
+*/
+
+#include "python-zstandard.h"
+
+extern PyObject* ZstdError;
+
+static void set_unsupported_operation(void) {
+	PyObject* iomod;
+	PyObject* exc;
+
+	iomod = PyImport_ImportModule("io");
+	if (NULL == iomod) {
+		return;
+	}
+
+	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
+	if (NULL == exc) {
+		Py_DECREF(iomod);
+		return;
+	}
+
+	PyErr_SetNone(exc);
+	Py_DECREF(exc);
+	Py_DECREF(iomod);
+}
+
+static void reader_dealloc(ZstdDecompressionReader* self) {
+	Py_XDECREF(self->decompressor);
+	Py_XDECREF(self->reader);
+
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+	}
+
+	PyObject_Del(self);
+}
+
+static ZstdDecompressionReader* reader_enter(ZstdDecompressionReader* self) {
+	if (self->entered) {
+		PyErr_SetString(PyExc_ValueError, "cannot __enter__ multiple times");
+		return NULL;
+	}
+
+	if (ensure_dctx(self->decompressor, 1)) {
+		return NULL;
+	}
+
+	self->entered = 1;
+
+	Py_INCREF(self);
+	return self;
+}
+
+static PyObject* reader_exit(ZstdDecompressionReader* self, PyObject* args) {
+	PyObject* exc_type;
+	PyObject* exc_value;
+	PyObject* exc_tb;
+
+	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
+		return NULL;
+	}
+
+	self->entered = 0;
+	self->closed = 1;
+
+	/* Release resources. */
+	Py_CLEAR(self->reader);
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+		memset(&self->buffer, 0, sizeof(self->buffer));
+	}
+
+	Py_CLEAR(self->decompressor);
+
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_readable(PyObject* self) {
+	Py_RETURN_TRUE;
+}
+
+static PyObject* reader_writable(PyObject* self) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_seekable(PyObject* self) {
+	Py_RETURN_TRUE;
+}
+
+static PyObject* reader_close(ZstdDecompressionReader* self) {
+	self->closed = 1;
+	Py_RETURN_NONE;
+}
+
+static PyObject* reader_closed(ZstdDecompressionReader* self) {
+	if (self->closed) {
+		Py_RETURN_TRUE;
+	}
+	else {
+		Py_RETURN_FALSE;
+	}
+}
+
+static PyObject* reader_flush(PyObject* self) {
+	Py_RETURN_NONE;
+}
+
+static PyObject* reader_isatty(PyObject* self) {
+	Py_RETURN_FALSE;
+}
+
+static PyObject* reader_read(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"size",
+		NULL
+	};
+
+	Py_ssize_t size = -1;
+	PyObject* result = NULL;
+	char* resultBuffer;
+	Py_ssize_t resultSize;
+	ZSTD_outBuffer output;
+	size_t zresult;
+
+	if (!self->entered) {
+		PyErr_SetString(ZstdError, "read() must be called from an active context manager");
+		return NULL;
+	}
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (self->finishedOutput) {
+		return PyBytes_FromStringAndSize("", 0);
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "n", kwlist, &size)) {
+		return NULL;
+	}
+
+	if (size < 1) {
+		PyErr_SetString(PyExc_ValueError, "cannot read negative or size 0 amounts");
+		return NULL;
+	}
+
+	result = PyBytes_FromStringAndSize(NULL, size);
+	if (NULL == result) {
+		return NULL;
+	}
+
+	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
+
+	output.dst = resultBuffer;
+	output.size = resultSize;
+	output.pos = 0;
+
+readinput:
+
+	/* Consume input data left over from last time. */
+	if (self->input.pos < self->input.size) {
+		Py_BEGIN_ALLOW_THREADS
+		zresult = ZSTD_decompress_generic(self->decompressor->dctx,
+			&output, &self->input);
+		Py_END_ALLOW_THREADS
+
+		/* Input exhausted. Clear our state tracking. */
+		if (self->input.pos == self->input.size) {
+			memset(&self->input, 0, sizeof(self->input));
+			Py_CLEAR(self->readResult);
+
+			if (self->buffer.buf) {
+				self->finishedInput = 1;
+			}
+		}
+
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "zstd decompress error: %s", ZSTD_getErrorName(zresult));
+			return NULL;
+		}
+		else if (0 == zresult) {
+			self->finishedOutput = 1;
+		}
+
+		/* We fulfilled the full read request. Emit it. */
+		if (output.pos && output.pos == output.size) {
+			self->bytesDecompressed += output.size;
+			return result;
+		}
+
+		/*
+		 * There is more room in the output. Fall through to try to collect
+		 * more data so we can try to fill the output.
+		 */
+	}
+
+	if (!self->finishedInput) {
+		if (self->reader) {
+			Py_buffer buffer;
+
+			assert(self->readResult == NULL);
+			self->readResult = PyObject_CallMethod(self->reader, "read",
+				"k", self->readSize);
+			if (NULL == self->readResult) {
+				return NULL;
+			}
+
+			memset(&buffer, 0, sizeof(buffer));
+
+			if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
+				return NULL;
+			}
+
+			/* EOF */
+			if (0 == buffer.len) {
+				self->finishedInput = 1;
+				Py_CLEAR(self->readResult);
+			}
+			else {
+				self->input.src = buffer.buf;
+				self->input.size = buffer.len;
+				self->input.pos = 0;
+			}
+
+			PyBuffer_Release(&buffer);
+		}
+		else {
+			assert(self->buffer.buf);
+			/*
+			 * We should only get here once since above block will exhaust
+			 * source buffer until finishedInput is set.
+			 */
+			assert(self->input.src == NULL);
+
+			self->input.src = self->buffer.buf;
+			self->input.size = self->buffer.len;
+			self->input.pos = 0;
+		}
+	}
+
+	if (self->input.size) {
+		goto readinput;
+	}
+
+	/* EOF */
+	self->bytesDecompressed += output.pos;
+
+	if (safe_pybytes_resize(&result, output.pos)) {
+		Py_XDECREF(result);
+		return NULL;
+	}
+
+	return result;
+}
+
+static PyObject* reader_readall(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* reader_readline(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* reader_readlines(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* reader_seek(ZstdDecompressionReader* self, PyObject* args) {
+	Py_ssize_t pos;
+	int whence = 0;
+	unsigned long long readAmount = 0;
+	size_t defaultOutSize = ZSTD_DStreamOutSize();
+
+	if (!self->entered) {
+		PyErr_SetString(ZstdError, "seek() must be called from an active context manager");
+		return NULL;
+	}
+
+	if (self->closed) {
+		PyErr_SetString(PyExc_ValueError, "stream is closed");
+		return NULL;
+	}
+
+	if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &whence)) {
+		return NULL;
+	}
+
+	if (whence == SEEK_SET) {
+		if (pos < 0) {
+			PyErr_SetString(PyExc_ValueError,
+				"cannot seek to negative position with SEEK_SET");
+			return NULL;
+		}
+
+		if ((unsigned long long)pos < self->bytesDecompressed) {
+			PyErr_SetString(PyExc_ValueError,
+				"cannot seek zstd decompression stream backwards");
+			return NULL;
+		}
+
+		readAmount = pos - self->bytesDecompressed;
+	}
+	else if (whence == SEEK_CUR) {
+		if (pos < 0) {
+			PyErr_SetString(PyExc_ValueError,
+				"cannot seek zstd decompression stream backwards");
+			return NULL;
+		}
+
+		readAmount = pos;
+	}
+	else if (whence == SEEK_END) {
+		/* We /could/ support this with pos==0. But let's not do that until someone
+		   needs it. */
+		PyErr_SetString(PyExc_ValueError,
+			"zstd decompression streams cannot be seeked with SEEK_END");
+		return NULL;
+	}
+
+	/* It is a bit inefficient to do this via the Python API. But since there
+	   is a bit of state tracking involved to read from this type, it is the
+	   easiest to implement. */
+	while (readAmount) {
+		Py_ssize_t readSize;
+		PyObject* readResult = PyObject_CallMethod((PyObject*)self, "read", "K",
+			readAmount < defaultOutSize ? readAmount : defaultOutSize);
+
+		if (!readResult) {
+			return NULL;
+		}
+
+		readSize = PyBytes_GET_SIZE(readResult);
+
+		/* Empty read means EOF. */
+		if (!readSize) {
+			break;
+		}
+
+		readAmount -= readSize;
+	}
+
+	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
+}
+
+static PyObject* reader_tell(ZstdDecompressionReader* self) {
+	/* TODO should this raise OSError since stream isn't seekable? */
+	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
+}
+
+static PyObject* reader_write(PyObject* self, PyObject* args) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyObject* reader_writelines(PyObject* self, PyObject* args) {
+	set_unsupported_operation();
+	return NULL;
+}
+
+static PyObject* reader_iter(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyObject* reader_iternext(PyObject* self) {
+	PyErr_SetNone(PyExc_NotImplementedError);
+	return NULL;
+}
+
+static PyMethodDef reader_methods[] = {
+	{ "__enter__", (PyCFunction)reader_enter, METH_NOARGS,
+	PyDoc_STR("Enter a compression context") },
+	{ "__exit__", (PyCFunction)reader_exit, METH_VARARGS,
+	PyDoc_STR("Exit a compression context") },
+	{ "close", (PyCFunction)reader_close, METH_NOARGS,
+	PyDoc_STR("Close the stream so it cannot perform any more operations") },
+	{ "closed", (PyCFunction)reader_closed, METH_NOARGS,
+	PyDoc_STR("Whether stream is closed") },
+	{ "flush", (PyCFunction)reader_flush, METH_NOARGS, PyDoc_STR("no-ops") },
+	{ "isatty", (PyCFunction)reader_isatty, METH_NOARGS, PyDoc_STR("Returns False") },
+	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
+	PyDoc_STR("Returns True") },
+	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS,
+	PyDoc_STR("read compressed data") },
+	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
+	{ "readline", (PyCFunction)reader_readline, METH_NOARGS, PyDoc_STR("Not implemented") },
+	{ "readlines", (PyCFunction)reader_readlines, METH_NOARGS, PyDoc_STR("Not implemented") },
+	{ "seek", (PyCFunction)reader_seek, METH_VARARGS, PyDoc_STR("Seek the stream") },
+	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,
+	PyDoc_STR("Returns True") },
+	{ "tell", (PyCFunction)reader_tell, METH_NOARGS,
+	PyDoc_STR("Returns current number of bytes compressed") },
+	{ "writable", (PyCFunction)reader_writable, METH_NOARGS,
+	PyDoc_STR("Returns False") },
+	{ "write", (PyCFunction)reader_write, METH_VARARGS, PyDoc_STR("unsupported operation") },
+	{ "writelines", (PyCFunction)reader_writelines, METH_VARARGS, PyDoc_STR("unsupported operation") },
+	{ NULL, NULL }
+};
+
+PyTypeObject ZstdDecompressionReaderType = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	"zstd.ZstdDecompressionReader", /* tp_name */
+	sizeof(ZstdDecompressionReader), /* tp_basicsize */
+	0, /* tp_itemsize */
+	(destructor)reader_dealloc, /* tp_dealloc */
+	0, /* tp_print */
+	0, /* tp_getattr */
+	0, /* tp_setattr */
+	0, /* tp_compare */
+	0, /* tp_repr */
+	0, /* tp_as_number */
+	0, /* tp_as_sequence */
+	0, /* tp_as_mapping */
+	0, /* tp_hash */
+	0, /* tp_call */
+	0, /* tp_str */
+	0, /* tp_getattro */
+	0, /* tp_setattro */
+	0, /* tp_as_buffer */
+	Py_TPFLAGS_DEFAULT, /* tp_flags */
+	0, /* tp_doc */
+	0, /* tp_traverse */
+	0, /* tp_clear */
+	0, /* tp_richcompare */
+	0, /* tp_weaklistoffset */
+	reader_iter, /* tp_iter */
+	reader_iternext, /* tp_iternext */
+	reader_methods, /* tp_methods */
+	0, /* tp_members */
+	0, /* tp_getset */
+	0, /* tp_base */
+	0, /* tp_dict */
+	0, /* tp_descr_get */
+	0, /* tp_descr_set */
+	0, /* tp_dictoffset */
+	0, /* tp_init */
+	0, /* tp_alloc */
+	PyType_GenericNew, /* tp_new */
+};
+
+
+void decompressionreader_module_init(PyObject* mod) {
+	/* TODO make reader a sub-class of io.RawIOBase */
+
+	Py_TYPE(&ZstdDecompressionReaderType) = &PyType_Type;
+	if (PyType_Ready(&ZstdDecompressionReaderType) < 0) {
+		return;
+	}
+}

contrib/python-zstandard/zstandard/__init__.py

@@ -0,0 +1,62 b''
+# Copyright (c) 2017-present, Gregory Szorc
+# All rights reserved.
+#
+# This software may be modified and distributed under the terms
+# of the BSD license. See the LICENSE file for details.
+
+"""Python interface to the Zstandard (zstd) compression library."""
+
+from __future__ import absolute_import, unicode_literals
+
+# This module serves 2 roles:
+#
+# 1) Export the C or CFFI "backend" through a central module.
+# 2) Implement additional functionality built on top of C or CFFI backend.
+
+import os
+import platform
+
+# Some Python implementations don't support C extensions. That's why we have
+# a CFFI implementation in the first place. The code here import one of our
+# "backends" then re-exports the symbols from this module. For convenience,
+# we support falling back to the CFFI backend if the C extension can't be
+# imported. But for performance reasons, we only do this on unknown Python
+# implementation. Notably, for CPython we require the C extension by default.
+# Because someone will inevitably want special behavior, the behavior is
+# configurable via an environment variable. A potentially better way to handle
+# this is to import a special ``__importpolicy__`` module or something
+# defining a variable and `setup.py` could write the file with whatever
+# policy was specified at build time. Until someone needs it, we go with
+# the hacky but simple environment variable approach.
+_module_policy = os.environ.get('PYTHON_ZSTANDARD_IMPORT_POLICY', 'default')
+
+if _module_policy == 'default':
+    if platform.python_implementation() in ('CPython',):
+        from zstd import *
+        backend = 'cext'
+    elif platform.python_implementation() in ('PyPy',):
+        from zstd_cffi import *
+        backend = 'cffi'
+    else:
+        try:
+            from zstd import *
+            backend = 'cext'
+        except ImportError:
+            from zstd_cffi import *
+            backend = 'cffi'
+elif _module_policy == 'cffi_fallback':
+    try:
+        from zstd import *
+        backend = 'cext'
+    except ImportError:
+        from zstd_cffi import *
+        backend = 'cffi'
+elif _module_policy == 'cext':
+    from zstd import *
+    backend = 'cext'
+elif _module_policy == 'cffi':
+    from zstd_cffi import *
+    backend = 'cffi'
+else:
+    raise ImportError('unknown module import policy: %s; use default, cffi_fallback, '
+                      'cext, or cffi' % _module_policy)

contrib/python-zstandard/zstd/COPYING

@@ -0,0 +1,339 b''
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
  No newline at end of file

contrib/python-zstandard/zstd/common/compiler.h

@@ -0,0 +1,111 b''
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMPILER_H
+#define ZSTD_COMPILER_H
+
+/*-*******************************************************
+*  Compiler specifics
+*********************************************************/
+/* force inlining */
+#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#  define INLINE_KEYWORD inline
+#else
+#  define INLINE_KEYWORD
+#endif
+
+#if defined(__GNUC__)
+#  define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#  define FORCE_INLINE_ATTR __forceinline
+#else
+#  define FORCE_INLINE_ATTR
+#endif
+
+/**
+ * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
+ * parameters. They must be inlined for the compiler to elimininate the constant
+ * branches.
+ */
+#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
+/**
+ * HINT_INLINE is used to help the compiler generate better code. It is *not*
+ * used for "templates", so it can be tweaked based on the compilers
+ * performance.
+ *
+ * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
+ * always_inline attribute.
+ *
+ * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
+ * attribute.
+ */
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
+#  define HINT_INLINE static INLINE_KEYWORD
+#else
+#  define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
+#endif
+
+/* force no inlining */
+#ifdef _MSC_VER
+#  define FORCE_NOINLINE static __declspec(noinline)
+#else
+#  ifdef __GNUC__
+#    define FORCE_NOINLINE static __attribute__((__noinline__))
+#  else
+#    define FORCE_NOINLINE static
+#  endif
+#endif
+
+/* target attribute */
+#ifndef __has_attribute
+  #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
+#endif
+#if defined(__GNUC__)
+#  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+#  define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+  #if (defined(__clang__) && __has_attribute(__target__)) \
+      || (defined(__GNUC__) \
+          && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) \
+      && (defined(__x86_64__) || defined(_M_X86)) \
+      && !defined(__BMI2__)
+  #  define DYNAMIC_BMI2 1
+  #else
+  #  define DYNAMIC_BMI2 0
+  #endif
+#endif
+
+/* prefetch */
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
+#  include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#  define PREFETCH(ptr)   _mm_prefetch((const char*)ptr, _MM_HINT_T0)
+#elif defined(__GNUC__)
+#  define PREFETCH(ptr)   __builtin_prefetch(ptr, 0, 0)
+#else
+#  define PREFETCH(ptr)   /* disabled */
+#endif
+
+/* disable warnings */
+#ifdef _MSC_VER    /* Visual Studio */
+#  include <intrin.h>                    /* For Visual 2005 */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
+#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#endif
+
+#endif /* ZSTD_COMPILER_H */

contrib/python-zstandard/zstd/common/cpu.h

@@ -0,0 +1,216 b''
+/*
+ * Copyright (c) 2018-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMMON_CPU_H
+#define ZSTD_COMMON_CPU_H
+
+/**
+ * Implementation taken from folly/CpuId.h
+ * https://github.com/facebook/folly/blob/master/folly/CpuId.h
+ */
+
+#include <string.h>
+
+#include "mem.h"
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+typedef struct {
+    U32 f1c;
+    U32 f1d;
+    U32 f7b;
+    U32 f7c;
+} ZSTD_cpuid_t;
+
+MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
+    U32 f1c = 0;
+    U32 f1d = 0;
+    U32 f7b = 0;
+    U32 f7c = 0;
+#ifdef _MSC_VER
+    int reg[4];
+    __cpuid((int*)reg, 0);
+    {
+        int const n = reg[0];
+        if (n >= 1) {
+            __cpuid((int*)reg, 1);
+            f1c = (U32)reg[2];
+            f1d = (U32)reg[3];
+        }
+        if (n >= 7) {
+            __cpuidex((int*)reg, 7, 0);
+            f7b = (U32)reg[1];
+            f7c = (U32)reg[2];
+        }
+    }
+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
+    /* The following block like the normal cpuid branch below, but gcc
+     * reserves ebx for use of its pic register so we must specially
+     * handle the save and restore to avoid clobbering the register
+     */
+    U32 n;
+    __asm__(
+        "pushl %%ebx\n\t"
+        "cpuid\n\t"
+        "popl %%ebx\n\t"
+        : "=a"(n)
+        : "a"(0)
+        : "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "popl %%ebx\n\t"
+          : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+          : "a"(1)
+          :);
+    }
+    if (n >= 7) {
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "movl %%ebx, %%eax\n\r"
+          "popl %%ebx"
+          : "=a"(f7b), "=c"(f7c)
+          : "a"(7), "c"(0)
+          : "edx");
+    }
+#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
+    U32 n;
+    __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
+    }
+    if (n >= 7) {
+      U32 f7a;
+      __asm__("cpuid"
+              : "=a"(f7a), "=b"(f7b), "=c"(f7c)
+              : "a"(7), "c"(0)
+              : "edx");
+    }
+#endif
+    {
+        ZSTD_cpuid_t cpuid;
+        cpuid.f1c = f1c;
+        cpuid.f1d = f1d;
+        cpuid.f7b = f7b;
+        cpuid.f7c = f7c;
+        return cpuid;
+    }
+}
+
+#define X(name, r, bit)                                                        \
+  MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) {                 \
+    return ((cpuid.r) & (1U << bit)) != 0;                                     \
+  }
+
+/* cpuid(1): Processor Info and Feature Bits. */
+#define C(name, bit) X(name, f1c, bit)
+  C(sse3, 0)
+  C(pclmuldq, 1)
+  C(dtes64, 2)
+  C(monitor, 3)
+  C(dscpl, 4)
+  C(vmx, 5)
+  C(smx, 6)
+  C(eist, 7)
+  C(tm2, 8)
+  C(ssse3, 9)
+  C(cnxtid, 10)
+  C(fma, 12)
+  C(cx16, 13)
+  C(xtpr, 14)
+  C(pdcm, 15)
+  C(pcid, 17)
+  C(dca, 18)
+  C(sse41, 19)
+  C(sse42, 20)
+  C(x2apic, 21)
+  C(movbe, 22)
+  C(popcnt, 23)
+  C(tscdeadline, 24)
+  C(aes, 25)
+  C(xsave, 26)
+  C(osxsave, 27)
+  C(avx, 28)
+  C(f16c, 29)
+  C(rdrand, 30)
+#undef C
+#define D(name, bit) X(name, f1d, bit)
+  D(fpu, 0)
+  D(vme, 1)
+  D(de, 2)
+  D(pse, 3)
+  D(tsc, 4)
+  D(msr, 5)
+  D(pae, 6)
+  D(mce, 7)
+  D(cx8, 8)
+  D(apic, 9)
+  D(sep, 11)
+  D(mtrr, 12)
+  D(pge, 13)
+  D(mca, 14)
+  D(cmov, 15)
+  D(pat, 16)
+  D(pse36, 17)
+  D(psn, 18)
+  D(clfsh, 19)
+  D(ds, 21)
+  D(acpi, 22)
+  D(mmx, 23)
+  D(fxsr, 24)
+  D(sse, 25)
+  D(sse2, 26)
+  D(ss, 27)
+  D(htt, 28)
+  D(tm, 29)
+  D(pbe, 31)
+#undef D
+
+/* cpuid(7): Extended Features. */
+#define B(name, bit) X(name, f7b, bit)
+  B(bmi1, 3)
+  B(hle, 4)
+  B(avx2, 5)
+  B(smep, 7)
+  B(bmi2, 8)
+  B(erms, 9)
+  B(invpcid, 10)
+  B(rtm, 11)
+  B(mpx, 14)
+  B(avx512f, 16)
+  B(avx512dq, 17)
+  B(rdseed, 18)
+  B(adx, 19)
+  B(smap, 20)
+  B(avx512ifma, 21)
+  B(pcommit, 22)
+  B(clflushopt, 23)
+  B(clwb, 24)
+  B(avx512pf, 26)
+  B(avx512er, 27)
+  B(avx512cd, 28)
+  B(sha, 29)
+  B(avx512bw, 30)
+  B(avx512vl, 31)
+#undef B
+#define C(name, bit) X(name, f7c, bit)
+  C(prefetchwt1, 0)
+  C(avx512vbmi, 1)
+#undef C
+
+#undef X
+
+#endif /* ZSTD_COMMON_CPU_H */

contrib/clang-format-ignorelist

@@ -8,11 +8,13 b' mercurial/cext/revlog.c'
 contrib/python-zstandard/c-ext/bufferutil.c
 contrib/python-zstandard/c-ext/compressiondict.c
 contrib/python-zstandard/c-ext/compressionparams.c
+contrib/python-zstandard/c-ext/compressionreader.c
 contrib/python-zstandard/c-ext/compressionwriter.c
 contrib/python-zstandard/c-ext/compressobj.c
 contrib/python-zstandard/c-ext/compressor.c
 contrib/python-zstandard/c-ext/compressoriterator.c
 contrib/python-zstandard/c-ext/constants.c
+contrib/python-zstandard/c-ext/decompressionreader.c
 contrib/python-zstandard/c-ext/decompressionwriter.c
 contrib/python-zstandard/c-ext/decompressobj.c
 contrib/python-zstandard/c-ext/decompressor.c
@@ -21,11 +23,13 b' contrib/python-zstandard/c-ext/framepara'
 contrib/python-zstandard/c-ext/python-zstandard.h
 contrib/python-zstandard/zstd.c
 contrib/python-zstandard/zstd/common/bitstream.h
+contrib/python-zstandard/zstd/common/compiler.h
+contrib/python-zstandard/zstd/common/cpu.h
 contrib/python-zstandard/zstd/common/entropy_common.c
 contrib/python-zstandard/zstd/common/error_private.c
 contrib/python-zstandard/zstd/common/error_private.h
+contrib/python-zstandard/zstd/common/fse_decompress.c
 contrib/python-zstandard/zstd/common/fse.h
-contrib/python-zstandard/zstd/common/fse_decompress.c
 contrib/python-zstandard/zstd/common/huf.h
 contrib/python-zstandard/zstd/common/mem.h
 contrib/python-zstandard/zstd/common/pool.c
@@ -40,11 +44,25 b' contrib/python-zstandard/zstd/common/zst'
 contrib/python-zstandard/zstd/compress/fse_compress.c
 contrib/python-zstandard/zstd/compress/huf_compress.c
 contrib/python-zstandard/zstd/compress/zstd_compress.c
-contrib/python-zstandard/zstd/compress/zstd_opt.h
+contrib/python-zstandard/zstd/compress/zstd_compress_internal.h
+contrib/python-zstandard/zstd/compress/zstd_double_fast.c
+contrib/python-zstandard/zstd/compress/zstd_double_fast.h
+contrib/python-zstandard/zstd/compress/zstd_fast.c
+contrib/python-zstandard/zstd/compress/zstd_fast.h
+contrib/python-zstandard/zstd/compress/zstd_lazy.c
+contrib/python-zstandard/zstd/compress/zstd_lazy.h
+contrib/python-zstandard/zstd/compress/zstd_ldm.c
+contrib/python-zstandard/zstd/compress/zstd_ldm.h
 contrib/python-zstandard/zstd/compress/zstdmt_compress.c
 contrib/python-zstandard/zstd/compress/zstdmt_compress.h
+contrib/python-zstandard/zstd/compress/zstd_opt.c
+contrib/python-zstandard/zstd/compress/zstd_opt.h
 contrib/python-zstandard/zstd/decompress/huf_decompress.c
 contrib/python-zstandard/zstd/decompress/zstd_decompress.c
+contrib/python-zstandard/zstd/deprecated/zbuff_common.c
+contrib/python-zstandard/zstd/deprecated/zbuff_compress.c
+contrib/python-zstandard/zstd/deprecated/zbuff_decompress.c
+contrib/python-zstandard/zstd/deprecated/zbuff.h
 contrib/python-zstandard/zstd/dictBuilder/cover.c
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.c
 contrib/python-zstandard/zstd/dictBuilder/divsufsort.h

contrib/python-zstandard/MANIFEST.in

@@ -1,5 +1,7 b''
 graft c-ext
 graft zstd
+graft tests
 include make_cffi.py
 include setup_zstd.py
 include zstd.c
+include LICENSE

contrib/python-zstandard/NEWS.rst

@@ -1,13 +1,201 b''
+===============
 Version History
 ===============
 
+1.0.0 (not yet released)
+========================
+
+Actions Blocking Release
+------------------------
+
+* compression and decompression APIs that support ``io.rawIOBase`` interface
+  (#13).
+* Refactor module names so C and CFFI extensions live under ``zstandard``
+  package.
+* Overall API design review.
+* Use Python allocator where possible.
+* Figure out what to do about experimental APIs not implemented by CFFI.
+* APIs for auto adjusting compression parameters based on input size. e.g.
+  clamping the window log so it isn't too large for input.
+* Consider allowing compressor and decompressor instances to be thread safe,
+  support concurrent operations. Or track when an operation is in progress and
+  refuse to let concurrent operations use the same instance.
+* Support for magic-less frames for all decompression operations (``decompress()``
+  doesn't work due to sniffing the content size and the lack of a ZSTD API to
+  sniff magic-less frames - this should be fixed in 1.3.5.).
+* Audit for complete flushing when ending compression streams.
+* Deprecate legacy APIs.
+* Audit for ability to control read/write sizes on all APIs.
+* Detect memory leaks via bench.py.
+* Remove low-level compression parameters from ``ZstdCompressor.__init__`` and
+  require use of ``CompressionParameters``.
+* Expose ``ZSTD_getFrameProgression()`` from more compressor types.
+
+Other Actions Not Blocking Release
+---------------------------------------
+
+* Support for block compression APIs.
+* API for ensuring max memory ceiling isn't exceeded.
+* Move off nose for testing.
+
+0.9.0 (released 2018-04-08)
+===========================
+
+Backwards Compatibility Notes
+-----------------------------
+
+* CFFI 1.11 or newer is now required (previous requirement was 1.8).
+* The primary module is now ``zstandard``. Please change imports of ``zstd``
+  and ``zstd_cffi`` to ``import zstandard``. See the README for more. Support
+  for importing the old names will be dropped in the next release.
+* ``ZstdCompressor.read_from()`` and ``ZstdDecompressor.read_from()`` have
+  been renamed to ``read_to_iter()``. ``read_from()`` is aliased to the new
+  name and will be deleted in a future release.
+* Support for Python 2.6 has been removed.
+* Support for Python 3.3 has been removed.
+* The ``selectivity`` argument to ``train_dictionary()`` has been removed, as
+  the feature disappeared from zstd 1.3.
+* Support for legacy dictionaries has been removed. Cover dictionaries are now
+  the default. ``train_cover_dictionary()`` has effectively been renamed to
+  ``train_dictionary()``.
+* The ``allow_empty`` argument from ``ZstdCompressor.compress()`` has been
+  deleted and the method now allows empty inputs to be compressed by default.
+* ``estimate_compression_context_size()`` has been removed. Use
+  ``CompressionParameters.estimated_compression_context_size()`` instead.
+* ``get_compression_parameters()`` has been removed. Use
+  ``CompressionParameters.from_level()`` instead.
+* The arguments to ``CompressionParameters.__init__()`` have changed. If you
+  were using positional arguments before, the positions now map to different
+  arguments. It is recommended to use keyword arguments to construct
+  ``CompressionParameters`` instances.
+* ``TARGETLENGTH_MAX`` constant has been removed (it disappeared from zstandard
+  1.3.4).
+* ``ZstdCompressor.write_to()`` and ``ZstdDecompressor.write_to()`` have been
+  renamed to ``ZstdCompressor.stream_writer()`` and
+  ``ZstdDecompressor.stream_writer()``, respectively. The old names are still
+  aliased, but will be removed in the next major release.
+* Content sizes are written into frame headers by default
+  (``ZstdCompressor(write_content_size=True)`` is now the default).
+* ``CompressionParameters`` has been renamed to ``ZstdCompressionParameters``
+  for consistency with other types. The old name is an alias and will be removed
+  in the next major release.
+
+Bug Fixes
+---------
+
+* Fixed memory leak in ``ZstdCompressor.copy_stream()`` (#40) (from 0.8.2).
+* Fixed memory leak in ``ZstdDecompressor.copy_stream()`` (#35) (from 0.8.2).
+* Fixed memory leak of ``ZSTD_DDict`` instances in CFFI's ``ZstdDecompressor``.
+
+New Features
+------------
+
+* Bundlded zstandard library upgraded from 1.1.3 to 1.3.4. This delivers various
+  bug fixes and performance improvements. It also gives us access to newer
+  features.
+* Support for negative compression levels.
+* Support for *long distance matching* (facilitates compression ratios that approach
+  LZMA).
+* Supporting for reading empty zstandard frames (with an embedded content size
+  of 0).
+* Support for writing and partial support for reading zstandard frames without a
+  magic header.
+* New ``stream_reader()`` API that exposes the ``io.RawIOBase`` interface (allows
+  you to ``.read()`` from a file-like object).
+* Several minor features, bug fixes, and performance enhancements.
+* Wheels for Linux and macOS are now provided with releases.
+
+Changes
+-------
+
+* Functions accepting bytes data now use the buffer protocol and can accept
+  more types (like ``memoryview`` and ``bytearray``) (#26).
+* Add #includes so compilation on OS X and BSDs works (#20).
+* New ``ZstdDecompressor.stream_reader()`` API to obtain a read-only i/o stream
+  of decompressed data for a source.
+* New ``ZstdCompressor.stream_reader()`` API to obtain a read-only i/o stream of
+  compressed data for a source.
+* Renamed ``ZstdDecompressor.read_from()`` to ``ZstdDecompressor.read_to_iter()``.
+  The old name is still available.
+* Renamed ``ZstdCompressor.read_from()`` to ``ZstdCompressor.read_to_iter()``.
+  ``read_from()`` is still available at its old location.
+* Introduce the ``zstandard`` module to import and re-export the C or CFFI
+  *backend* as appropriate. Behavior can be controlled via the
+  ``PYTHON_ZSTANDARD_IMPORT_POLICY`` environment variable. See README for
+  usage info.
+* Vendored version of zstd upgraded to 1.3.4.
+* Added module constants ``CONTENTSIZE_UNKNOWN`` and ``CONTENTSIZE_ERROR``.
+* Add ``STRATEGY_BTULTRA`` compression strategy constant.
+* Switch from deprecated ``ZSTD_getDecompressedSize()`` to
+  ``ZSTD_getFrameContentSize()`` replacement.
+* ``ZstdCompressor.compress()`` can now compress empty inputs without requiring
+  special handling.
+* ``ZstdCompressor`` and ``ZstdDecompressor`` now have a ``memory_size()``
+  method for determining the current memory utilization of the underlying zstd
+  primitive.
+* ``train_dictionary()`` has new arguments and functionality for trying multiple
+  variations of COVER parameters and selecting the best one.
+* Added module constants ``LDM_MINMATCH_MIN``, ``LDM_MINMATCH_MAX``, and
+  ``LDM_BUCKETSIZELOG_MAX``.
+* Converted all consumers to the zstandard *new advanced API*, which uses
+  ``ZSTD_compress_generic()``
+* ``CompressionParameters.__init__`` now accepts several more arguments,
+  including support for *long distance matching*.
+* ``ZstdCompressionDict.__init__`` now accepts a ``dict_type`` argument that
+  controls how the dictionary should be interpreted. This can be used to
+  force the use of *content-only* dictionaries or to require the presence
+  of the dictionary magic header.
+* ``ZstdCompressionDict.precompute_compress()`` can be used to precompute the
+  compression dictionary so it can efficiently be used with multiple
+  ``ZstdCompressor`` instances.
+* Digested dictionaries are now stored in ``ZstdCompressionDict`` instances,
+  created automatically on first use, and automatically reused by all
+  ``ZstdDecompressor`` instances bound to that dictionary.
+* All meaningful functions now accept keyword arguments.
+* ``ZstdDecompressor.decompressobj()`` now accepts a ``write_size`` argument
+  to control how much work to perform on every decompressor invocation.
+* ``ZstdCompressor.write_to()`` now exposes a ``tell()``, which exposes the
+  total number of bytes written so far.
+* ``ZstdDecompressor.stream_reader()`` now supports ``seek()`` when moving
+  forward in the stream.
+* Removed ``TARGETLENGTH_MAX`` constant.
+* Added ``frame_header_size(data)`` function.
+* Added ``frame_content_size(data)`` function.
+* Consumers of ``ZSTD_decompress*`` have been switched to the new *advanced
+  decompression* API.
+* ``ZstdCompressor`` and ``ZstdCompressionParams`` can now be constructed with
+  negative compression levels.
+* ``ZstdDecompressor`` now accepts a ``max_window_size`` argument to limit the
+  amount of memory required for decompression operations.
+* ``FORMAT_ZSTD1`` and ``FORMAT_ZSTD1_MAGICLESS`` constants to be used with
+  the ``format`` compression parameter to control whether the frame magic
+  header is written.
+* ``ZstdDecompressor`` now accepts a ``format`` argument to control the
+  expected frame format.
+* ``ZstdCompressor`` now has a ``frame_progression()`` method to return
+  information about the current compression operation.
+* Error messages in CFFI no longer have ``b''`` literals.
+* Compiler warnings and underlying overflow issues on 32-bit platforms have been
+  fixed.
+* Builds in CI now build with compiler warnings as errors. This should hopefully
+  fix new compiler warnings from being introduced.
+* Make ``ZstdCompressor(write_content_size=True)`` and
+  ``CompressionParameters(write_content_size=True)`` the default.
+* ``CompressionParameters`` has been renamed to ``ZstdCompressionParameters``.
+
+0.8.2 (released 2018-02-22)
+---------------------------
+
+* Fixed memory leak in ``ZstdCompressor.copy_stream()`` (#40).
+* Fixed memory leak in ``ZstdDecompressor.copy_stream()`` (#35).
+
 0.8.1 (released 2017-04-08)
 ---------------------------
 
 * Add #includes so compilation on OS X and BSDs works (#20).
 
 0.8.0 (released 2017-03-08)
----------------------------
+===========================
 
 * CompressionParameters now has a estimated_compression_context_size() method.
   zstd.estimate_compression_context_size() is now deprecated and slated for
@@ -35,7 +223,7 b' 0.8.0 (released 2017-03-08)'
   DictParameters instance to control dictionary generation.
 
 0.7.0 (released 2017-02-07)
----------------------------
+===========================
 
 * Added zstd.get_frame_parameters() to obtain info about a zstd frame.
 * Added ZstdDecompressor.decompress_content_dict_chain() for efficient
@@ -62,7 +250,7 b' 0.7.0 (released 2017-02-07)'
 * DictParameters instances now expose their values as attributes.
 
 0.6.0 (released 2017-01-14)
----------------------------
+===========================
 
 * Support for legacy zstd protocols (build time opt in feature).
 * Automation improvements to test against Python 3.6, latest versions
@@ -79,17 +267,17 b' 0.6.0 (released 2017-01-14)'
 * Disallow compress(b'') when writing content sizes by default (issue #11).
 
 0.5.2 (released 2016-11-12)
----------------------------
+===========================
 
 * more packaging fixes for source distribution
 
 0.5.1 (released 2016-11-12)
----------------------------
+===========================
 
 * setup_zstd.py is included in the source distribution
 
 0.5.0 (released 2016-11-10)
----------------------------
+===========================
 
 * Vendored version of zstd updated to 1.1.1.
 * Continuous integration for Python 3.6 and 3.7
@@ -114,8 +302,8 b' 0.5.0 (released 2016-11-10)'
 * The monolithic ``zstd.c`` file has been split into a header file defining
   types and separate ``.c`` source files for the implementation.
 
-History of the Project
-======================
+Older History
+=============
 
 2016-08-31 - Zstandard 1.0.0 is released and Gregory starts hacking on a
 Python extension for use by the Mercurial project. A very hacky prototype

contrib/python-zstandard/README.rst

@@ -11,69 +11,18 b' underlying C API through a Pythonic inte'
 performance. This means exposing most of the features and flexibility
 of the C API while not sacrificing usability or safety that Python provides.
 
-The canonical home for this project is
+The canonical home for this project lives in a Mercurial repository run by
+the author. For convenience, that repository is frequently synchronized to
 https://github.com/indygreg/python-zstandard.
 
 |  |ci-status| |win-ci-status|
 
-State of Project
-================
-
-The project is officially in beta state. The author is reasonably satisfied
-that functionality works as advertised. **There will be some backwards
-incompatible changes before 1.0, probably in the 0.9 release.** This may
-involve renaming the main module from *zstd* to *zstandard* and renaming
-various types and methods. Pin the package version to prevent unwanted
-breakage when this change occurs!
-
-This project is vendored and distributed with Mercurial 4.1, where it is
-used in a production capacity.
-
-There is continuous integration for Python versions 2.6, 2.7, and 3.3+
-on Linux x86_x64 and Windows x86 and x86_64. The author is reasonably
-confident the extension is stable and works as advertised on these
-platforms.
-
-The CFFI bindings are mostly feature complete. Where a feature is implemented
-in CFFI, unit tests run against both C extension and CFFI implementation to
-ensure behavior parity.
-
-Expected Changes
-----------------
-
-The author is reasonably confident in the current state of what's
-implemented on the ``ZstdCompressor`` and ``ZstdDecompressor`` types.
-Those APIs likely won't change significantly. Some low-level behavior
-(such as naming and types expected by arguments) may change.
-
-There will likely be arguments added to control the input and output
-buffer sizes (currently, certain operations read and write in chunk
-sizes using zstd's preferred defaults).
-
-There should be an API that accepts an object that conforms to the buffer
-interface and returns an iterator over compressed or decompressed output.
-
-There should be an API that exposes an ``io.RawIOBase`` interface to
-compressor and decompressor streams, like how ``gzip.GzipFile`` from
-the standard library works (issue 13).
-
-The author is on the fence as to whether to support the extremely
-low level compression and decompression APIs. It could be useful to
-support compression without the framing headers. But the author doesn't
-believe it a high priority at this time.
-
-There will likely be a refactoring of the module names. Currently,
-``zstd`` is a C extension and ``zstd_cffi`` is the CFFI interface.
-This means that all code for the C extension must be implemented in
-C. ``zstd`` may be converted to a Python module so code can be reused
-between CFFI and C and so not all code in the C extension has to be C.
-
 Requirements
 ============
 
-This extension is designed to run with Python 2.6, 2.7, 3.3, 3.4, 3.5, and
-3.6 on common platforms (Linux, Windows, and OS X). Only x86_64 is
-currently well-tested as an architecture.
+This extension is designed to run with Python 2.7, 3.4, 3.5, and 3.6
+on common platforms (Linux, Windows, and OS X). x86 and x86_64 are well-tested
+on Windows. Only x86_64 is well-tested on Linux and macOS.
 
 Installing
 ==========
@@ -96,114 +45,82 b' this package with ``conda``.'
 Performance
 ===========
 
-Very crude and non-scientific benchmarking (most benchmarks fall in this
-category because proper benchmarking is hard) show that the Python bindings
-perform within 10% of the native C implementation.
-
-The following table compares the performance of compressing and decompressing
-a 1.1 GB tar file comprised of the files in a Firefox source checkout. Values
-obtained with the ``zstd`` program are on the left. The remaining columns detail
-performance of various compression APIs in the Python bindings.
+zstandard is a highly tunable compression algorithm. In its default settings
+(compression level 3), it will be faster at compression and decompression and
+will have better compression ratios than zlib on most data sets. When tuned
+for speed, it approaches lz4's speed and ratios. When tuned for compression
+ratio, it approaches lzma ratios and compression speed, but decompression
+speed is much faster. See the official zstandard documentation for more.
 
-+-------+-----------------+-----------------+-----------------+---------------+
-| Level | Native          | Simple          | Stream In       | Stream Out    |
-|       | Comp / Decomp   | Comp / Decomp   | Comp / Decomp   | Comp          |
-+=======+=================+=================+=================+===============+
-|   1   | 490 / 1338 MB/s | 458 / 1266 MB/s | 407 / 1156 MB/s |  405 MB/s     |
-+-------+-----------------+-----------------+-----------------+---------------+
-|   2   | 412 / 1288 MB/s | 381 / 1203 MB/s | 345 / 1128 MB/s |  349 MB/s     |
-+-------+-----------------+-----------------+-----------------+---------------+
-|   3   | 342 / 1312 MB/s | 319 / 1182 MB/s | 285 / 1165 MB/s |  287 MB/s     |
-+-------+-----------------+-----------------+-----------------+---------------+
-|  11   |  64 / 1506 MB/s |  66 / 1436 MB/s |  56 / 1342 MB/s |   57 MB/s     |
-+-------+-----------------+-----------------+-----------------+---------------+
-
-Again, these are very unscientific. But it shows that Python is capable of
-compressing at several hundred MB/s and decompressing at over 1 GB/s.
-
-Comparison to Other Python Bindings
-===================================
-
-https://pypi.python.org/pypi/zstd is an alternate Python binding to
-Zstandard. At the time this was written, the latest release of that
-package (1.1.2) only exposed the simple APIs for compression and decompression.
-This package exposes much more of the zstd API, including streaming and
-dictionary compression. This package also has CFFI support.
-
-Bundling of Zstandard Source Code
-=================================
-
-The source repository for this project contains a vendored copy of the
-Zstandard source code. This is done for a few reasons.
+zstandard and this library support multi-threaded compression. There is a
+mechanism to compress large inputs using multiple threads.
 
-First, Zstandard is relatively new and not yet widely available as a system
-package. Providing a copy of the source code enables the Python C extension
-to be compiled without requiring the user to obtain the Zstandard source code
-separately.
-
-Second, Zstandard has both a stable *public* API and an *experimental* API.
-The *experimental* API is actually quite useful (contains functionality for
-training dictionaries for example), so it is something we wish to expose to
-Python. However, the *experimental* API is only available via static linking.
-Furthermore, the *experimental* API can change at any time. So, control over
-the exact version of the Zstandard library linked against is important to
-ensure known behavior.
-
-Instructions for Building and Testing
-=====================================
-
-Once you have the source code, the extension can be built via setup.py::
-
-   $ python setup.py build_ext
-
-We recommend testing with ``nose``::
-
-   $ nosetests
+The performance of this library is usually very similar to what the zstandard
+C API can deliver. Overhead in this library is due to general Python overhead
+and can't easily be avoided by *any* zstandard Python binding. This library
+exposes multiple APIs for performing compression and decompression so callers
+can pick an API suitable for their need. Contrast with the compression
+modules in Python's standard library (like ``zlib``), which only offer limited
+mechanisms for performing operations. The API flexibility means consumers can
+choose to use APIs that facilitate zero copying or minimize Python object
+creation and garbage collection overhead.
 
-A Tox configuration is present to test against multiple Python versions::
-
-   $ tox
-
-Tests use the ``hypothesis`` Python package to perform fuzzing. If you
-don't have it, those tests won't run. Since the fuzzing tests take longer
-to execute than normal tests, you'll need to opt in to running them by
-setting the ``ZSTD_SLOW_TESTS`` environment variable. This is set
-automatically when using ``tox``.
-
-The ``cffi`` Python package needs to be installed in order to build the CFFI
-bindings. If it isn't present, the CFFI bindings won't be built.
-
-To create a virtualenv with all development dependencies, do something
-like the following::
-
-  # Python 2
-  $ virtualenv venv
-
-  # Python 3
-  $ python3 -m venv venv
-
-  $ source venv/bin/activate
-  $ pip install cffi hypothesis nose tox
+This library is capable of single-threaded throughputs well over 1 GB/s. For
+exact numbers, measure yourself. The source code repository has a ``bench.py``
+script that can be used to measure things.
 
 API
 ===
 
-The compiled C extension provides a ``zstd`` Python module. The CFFI
-bindings provide a ``zstd_cffi`` module. Both provide an identical API
-interface. The types, functions, and attributes exposed by these modules
+To interface with Zstandard, simply import the ``zstandard`` module::
+
+   import zstandard
+
+It is a popular convention to alias the module as a different name for
+brevity::
+
+   import zstandard as zstd
+
+This module attempts to import and use either the C extension or CFFI
+implementation. On Python platforms known to support C extensions (like
+CPython), it raises an ImportError if the C extension cannot be imported.
+On Python platforms known to not support C extensions (like PyPy), it only
+attempts to import the CFFI implementation and raises ImportError if that
+can't be done. On other platforms, it first tries to import the C extension
+then falls back to CFFI if that fails and raises ImportError if CFFI fails.
+
+To change the module import behavior, a ``PYTHON_ZSTANDARD_IMPORT_POLICY``
+environment variable can be set. The following values are accepted:
+
+default
+   The behavior described above.
+cffi_fallback
+   Always try to import the C extension then fall back to CFFI if that
+   fails.
+cext
+   Only attempt to import the C extension.
+cffi
+   Only attempt to import the CFFI implementation.
+
+In addition, the ``zstandard`` module exports a ``backend`` attribute
+containing the string name of the backend being used. It will be one
+of ``cext`` or ``cffi`` (for *C extension* and *cffi*, respectively).
+
+The types, functions, and attributes exposed by the ``zstandard`` module
 are documented in the sections below.
 
 .. note::
 
    The documentation in this section makes references to various zstd
-   concepts and functionality. The ``Concepts`` section below explains
-   these concepts in more detail.
+   concepts and functionality. The source repository contains a
+   ``docs/concepts.rst`` file explaining these in more detail.
 
 ZstdCompressor
 --------------
 
 The ``ZstdCompressor`` class provides an interface for performing
-compression operations.
+compression operations. Each instance is essentially a wrapper around a
+``ZSTD_CCtx`` from the C API.
 
 Each instance is associated with parameters that control compression
 behavior. These come from the following named arguments (all optional):
@@ -214,21 +131,21 b' dict_data'
    Compression dictionary to use.
 
    Note: When using dictionary data and ``compress()`` is called multiple
-   times, the ``CompressionParameters`` derived from an integer compression
-   ``level`` and the first compressed data's size will be reused for all
-   subsequent operations. This may not be desirable if source data size
-   varies significantly.
+   times, the ``ZstdCompressionParameters`` derived from an integer
+   compression ``level`` and the first compressed data's size will be reused
+   for all subsequent operations. This may not be desirable if source data
+   size varies significantly.
 compression_params
-   A ``CompressionParameters`` instance (overrides the ``level`` value).
+   A ``ZstdCompressionParameters`` instance defining compression settings.
 write_checksum
    Whether a 4 byte checksum should be written with the compressed data.
    Defaults to False. If True, the decompressor can verify that decompressed
    data matches the original input data.
 write_content_size
    Whether the size of the uncompressed data will be written into the
-   header of compressed data. Defaults to False. The data will only be
+   header of compressed data. Defaults to True. The data will only be
    written if the compressor knows the size of the input data. This is
-   likely not true for streaming compression.
+   often not true for streaming compression.
 write_dict_id
    Whether to write the dictionary ID into the compressed data.
    Defaults to True. The dictionary ID is only written if a dictionary
@@ -242,10 +159,25 b' threads'
    data. APIs that spawn multiple threads for working on multiple pieces of
    data have their own ``threads`` argument.
 
+``compression_params`` is mutually exclusive with ``level``, ``write_checksum``,
+``write_content_size``, ``write_dict_id``, and ``threads``.
+
 Unless specified otherwise, assume that no two methods of ``ZstdCompressor``
 instances can be called from multiple Python threads simultaneously. In other
 words, assume instances are not thread safe unless stated otherwise.
 
+Utility Methods
+^^^^^^^^^^^^^^^
+
+``frame_progression()`` returns a 3-tuple containing the number of bytes
+ingested, consumed, and produced by the current compression operation.
+
+``memory_size()`` obtains the memory utilization of the underlying zstd
+compression context, in bytes.::
+
+    cctx = zstd.ZstdCompressor()
+    memory = cctx.memory_size()
+
 Simple API
 ^^^^^^^^^^
 
@@ -256,40 +188,75 b' Simple API'
 
 The ``data`` argument can be any object that implements the *buffer protocol*.
 
-Unless ``compression_params`` or ``dict_data`` are passed to the
-``ZstdCompressor``, each invocation of ``compress()`` will calculate the
-optimal compression parameters for the configured compression ``level`` and
-input data size (some parameters are fine-tuned for small input sizes).
+Stream Reader API
+^^^^^^^^^^^^^^^^^
+
+``stream_reader(source)`` can be used to obtain an object conforming to the
+``io.RawIOBase`` interface for reading compressed output as a stream::
+
+   with open(path, 'rb') as fh:
+       cctx = zstd.ZstdCompressor()
+       with cctx.stream_reader(fh) as reader:
+           while True:
+               chunk = reader.read(16384)
+               if not chunk:
+                   break
+
+               # Do something with compressed chunk.
+
+The stream can only be read within a context manager. When the context
+manager exits, the stream is closed and the underlying resource is
+released and future operations against the compression stream stream will fail.
+
+The ``source`` argument to ``stream_reader()`` can be any object with a
+``read(size)`` method or any object implementing the *buffer protocol*.
 
-If a compression dictionary is being used, the compression parameters
-determined from the first input's size will be reused for subsequent
-operations.
+``stream_reader()`` accepts a ``size`` argument specifying how large the input
+stream is. This is used to adjust compression parameters so they are
+tailored to the source size.::
+
+   with open(path, 'rb') as fh:
+       cctx = zstd.ZstdCompressor()
+       with cctx.stream_reader(fh, size=os.stat(path).st_size) as reader:
+           ...
+
+If the ``source`` is a stream, you can specify how large ``read()`` requests
+to that stream should be via the ``read_size`` argument. It defaults to
+``zstandard.COMPRESSION_RECOMMENDED_INPUT_SIZE``.::
 
-There is currently a deficiency in zstd's C APIs that makes it difficult
-to round trip empty inputs when ``write_content_size=True``. Attempting
-this will raise a ``ValueError`` unless ``allow_empty=True`` is passed
-to ``compress()``.
+   with open(path, 'rb') as fh:
+       cctx = zstd.ZstdCompressor()
+       # Will perform fh.read(8192) when obtaining data to feed into the
+       # compressor.
+       with cctx.stream_reader(fh, read_size=8192) as reader:
+           ...
+
+The stream returned by ``stream_reader()`` is neither writable nor seekable
+(even if the underlying source is seekable). ``readline()`` and
+``readlines()`` are not implemented because they don't make sense for
+compressed data. ``tell()`` returns the number of compressed bytes
+emitted so far.
 
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
-``write_to(fh)`` (which behaves as a context manager) allows you to *stream*
+``stream_writer(fh)`` (which behaves as a context manager) allows you to *stream*
 data into a compressor.::
 
    cctx = zstd.ZstdCompressor(level=10)
-   with cctx.write_to(fh) as compressor:
+   with cctx.stream_writer(fh) as compressor:
        compressor.write(b'chunk 0')
        compressor.write(b'chunk 1')
        ...
 
-The argument to ``write_to()`` must have a ``write(data)`` method. As
+The argument to ``stream_writer()`` must have a ``write(data)`` method. As
 compressed data is available, ``write()`` will be called with the compressed
 data as its argument. Many common Python types implement ``write()``, including
 open file handles and ``io.BytesIO``.
 
-``write_to()`` returns an object representing a streaming compressor instance.
-It **must** be used as a context manager. That object's ``write(data)`` method
-is used to feed data into the compressor.
+``stream_writer()`` returns an object representing a streaming compressor
+instance. It **must** be used as a context manager. That object's
+``write(data)`` method is used to feed data into the compressor.
 
 A ``flush()`` method can be called to evict whatever data remains within the
 compressor's internal state into the output object. This may result in 0 or
@@ -303,7 +270,7 b' If the size of the data being fed to thi'
 you can declare it before compression begins::
 
    cctx = zstd.ZstdCompressor()
-   with cctx.write_to(fh, size=data_len) as compressor:
+   with cctx.stream_writer(fh, size=data_len) as compressor:
        compressor.write(chunk0)
        compressor.write(chunk1)
        ...
@@ -315,29 +282,35 b' content size being written into the fram'
 The size of chunks being ``write()`` to the destination can be specified::
 
     cctx = zstd.ZstdCompressor()
-    with cctx.write_to(fh, write_size=32768) as compressor:
+    with cctx.stream_writer(fh, write_size=32768) as compressor:
         ...
 
 To see how much memory is being used by the streaming compressor::
 
     cctx = zstd.ZstdCompressor()
-    with cctx.write_to(fh) as compressor:
+    with cctx.stream_writer(fh) as compressor:
         ...
         byte_size = compressor.memory_size()
 
+Thte total number of bytes written so far are exposed via ``tell()``::
+
+    cctx = zstd.ZstdCompressor()
+    with cctx.stream_writer(fh) as compressor:
+        ...
+        total_written = compressor.tell()
+
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
-``read_from(reader)`` provides a mechanism to stream data out of a compressor
-as an iterator of data chunks.::
+``read_to_iter(reader)`` provides a mechanism to stream data out of a
+compressor as an iterator of data chunks.::
 
    cctx = zstd.ZstdCompressor()
-   for chunk in cctx.read_from(fh):
+   for chunk in cctx.read_to_iter(fh):
         # Do something with emitted data.
 
-``read_from()`` accepts an object that has a ``read(size)`` method or conforms
-to the buffer protocol. (``bytes`` and ``memoryview`` are 2 common types that
-provide the buffer protocol.)
+``read_to_iter()`` accepts an object that has a ``read(size)`` method or
+conforms to the buffer protocol.
 
 Uncompressed data is fetched from the source either by calling ``read(size)``
 or by fetching a slice of data from the object directly (in the case where
@@ -348,23 +321,24 b' If reading from the source via ``read()`'
 it raises or returns an empty bytes (``b''``). It is perfectly valid for
 the source to deliver fewer bytes than were what requested by ``read(size)``.
 
-Like ``write_to()``, ``read_from()`` also accepts a ``size`` argument
+Like ``stream_writer()``, ``read_to_iter()`` also accepts a ``size`` argument
 declaring the size of the input stream::
 
     cctx = zstd.ZstdCompressor()
-    for chunk in cctx.read_from(fh, size=some_int):
+    for chunk in cctx.read_to_iter(fh, size=some_int):
         pass
 
 You can also control the size that data is ``read()`` from the source and
 the ideal size of output chunks::
 
     cctx = zstd.ZstdCompressor()
-    for chunk in cctx.read_from(fh, read_size=16384, write_size=8192):
+    for chunk in cctx.read_to_iter(fh, read_size=16384, write_size=8192):
         pass
 
-Unlike ``write_to()``, ``read_from()`` does not give direct control over the
-sizes of chunks fed into the compressor. Instead, chunk sizes will be whatever
-the object being read from delivers. These will often be of a uniform size.
+Unlike ``stream_writer()``, ``read_to_iter()`` does not give direct control
+over the sizes of chunks fed into the compressor. Instead, chunk sizes will
+be whatever the object being read from delivers. These will often be of a
+uniform size.
 
 Stream Copying API
 ^^^^^^^^^^^^^^^^^^
@@ -404,7 +378,7 b' Compressor API'
 ``flush()`` methods. Each returns compressed data or an empty bytes.
 
 The purpose of ``compressobj()`` is to provide an API-compatible interface
-with ``zlib.compressobj`` and ``bz2.BZ2Compressor``. This allows callers to
+with ``zlib.compressobj``, ``bz2.BZ2Compressor``, etc. This allows callers to
 swap in different compressor objects while using the same API.
 
 ``flush()`` accepts an optional argument indicating how to end the stream.
@@ -485,13 +459,23 b' ZstdDecompressor'
 ----------------
 
 The ``ZstdDecompressor`` class provides an interface for performing
-decompression.
+decompression. It is effectively a wrapper around the ``ZSTD_DCtx`` type from
+the C API.
 
 Each instance is associated with parameters that control decompression. These
 come from the following named arguments (all optional):
 
 dict_data
    Compression dictionary to use.
+max_window_size
+   Sets an uppet limit on the window size for decompression operations in
+   kibibytes. This setting can be used to prevent large memory allocations
+   for inputs using large compression windows.
+format
+   Set the format of data for the decoder. By default, this is
+   ``zstd.FORMAT_ZSTD1``. It can be set to ``zstd.FORMAT_ZSTD1_MAGICLESS`` to
+   allow decoding frames without the 4 byte magic header. Not all decompression
+   APIs support this mode.
 
 The interface of this class is very similar to ``ZstdCompressor`` (by design).
 
@@ -499,6 +483,15 b' Unless specified otherwise, assume that '
 instances can be called from multiple Python threads simultaneously. In other
 words, assume instances are not thread safe unless stated otherwise.
 
+Utility Methods
+^^^^^^^^^^^^^^^
+
+``memory_size()`` obtains the size of the underlying zstd decompression context,
+in bytes.::
+
+    dctx = zstd.ZstdDecompressor()
+    size = dctx.memory_size()
+
 Simple API
 ^^^^^^^^^^
 
@@ -509,9 +502,10 b' frame in a single operation.::'
     decompressed = dctx.decompress(data)
 
 By default, ``decompress(data)`` will only work on data written with the content
-size encoded in its header. This can be achieved by creating a
-``ZstdCompressor`` with ``write_content_size=True``. If compressed data without
-an embedded content size is seen, ``zstd.ZstdError`` will be raised.
+size encoded in its header (this is the default behavior of
+``ZstdCompressor().compress()`` but may not be true for streaming compression). If
+compressed data without an embedded content size is seen, ``zstd.ZstdError`` will
+be raised.
 
 If the compressed data doesn't have its content size embedded within it,
 decompression can be attempted by specifying the ``max_output_size``
@@ -534,17 +528,67 b' performed every time the method is calle'
 result in a lot of work for the memory allocator and may result in
 ``MemoryError`` being raised if the allocation fails.
 
-If the exact size of decompressed data is unknown, it is **strongly**
-recommended to use a streaming API.
+.. important::
+
+   If the exact size of decompressed data is unknown (not passed in explicitly
+   and not stored in the zstandard frame), for performance reasons it is
+   encouraged to use a streaming API.
+
+Stream Reader API
+^^^^^^^^^^^^^^^^^
+
+``stream_reader(source)`` can be used to obtain an object conforming to the
+``io.RawIOBase`` interface for reading decompressed output as a stream::
+
+   with open(path, 'rb') as fh:
+       dctx = zstd.ZstdDecompressor()
+       with dctx.stream_reader(fh) as reader:
+           while True:
+               chunk = reader.read(16384)
+               if not chunk:
+                   break
+
+               # Do something with decompressed chunk.
+
+The stream can only be read within a context manager. When the context
+manager exits, the stream is closed and the underlying resource is
+released and future operations against the stream will fail.
+
+The ``source`` argument to ``stream_reader()`` can be any object with a
+``read(size)`` method or any object implementing the *buffer protocol*.
+
+If the ``source`` is a stream, you can specify how large ``read()`` requests
+to that stream should be via the ``read_size`` argument. It defaults to
+``zstandard.DECOMPRESSION_RECOMMENDED_INPUT_SIZE``.::
+
+   with open(path, 'rb') as fh:
+       dctx = zstd.ZstdDecompressor()
+       # Will perform fh.read(8192) when obtaining data for the decompressor.
+       with dctx.stream_reader(fh, read_size=8192) as reader:
+           ...
+
+The stream returned by ``stream_reader()`` is not writable.
+
+The stream returned by ``stream_reader()`` is *partially* seekable.
+Absolute and relative positions (``SEEK_SET`` and ``SEEK_CUR``) forward
+of the current position are allowed. Offsets behind the current read
+position and offsets relative to the end of stream are not allowed and
+will raise ``ValueError`` if attempted.
+
+``tell()`` returns the number of decompressed bytes read so far.
+
+Not all I/O methods are implemented. Notably missing is support for
+``readline()``, ``readlines()``, and linewise iteration support. Support for
+these is planned for a future release.
 
 Streaming Input API
 ^^^^^^^^^^^^^^^^^^^
 
-``write_to(fh)`` can be used to incrementally send compressed data to a
+``stream_writer(fh)`` can be used to incrementally send compressed data to a
 decompressor.::
 
     dctx = zstd.ZstdDecompressor()
-    with dctx.write_to(fh) as decompressor:
+    with dctx.stream_writer(fh) as decompressor:
         decompressor.write(compressed_data)
 
 This behaves similarly to ``zstd.ZstdCompressor``: compressed data is written to
@@ -558,54 +602,56 b' of ``0`` are possible.'
 The size of chunks being ``write()`` to the destination can be specified::
 
     dctx = zstd.ZstdDecompressor()
-    with dctx.write_to(fh, write_size=16384) as decompressor:
+    with dctx.stream_writer(fh, write_size=16384) as decompressor:
         pass
 
 You can see how much memory is being used by the decompressor::
 
     dctx = zstd.ZstdDecompressor()
-    with dctx.write_to(fh) as decompressor:
+    with dctx.stream_writer(fh) as decompressor:
         byte_size = decompressor.memory_size()
 
 Streaming Output API
 ^^^^^^^^^^^^^^^^^^^^
 
-``read_from(fh)`` provides a mechanism to stream decompressed data out of a
+``read_to_iter(fh)`` provides a mechanism to stream decompressed data out of a
 compressed source as an iterator of data chunks.:: 
 
     dctx = zstd.ZstdDecompressor()
-    for chunk in dctx.read_from(fh):
+    for chunk in dctx.read_to_iter(fh):
         # Do something with original data.
 
-``read_from()`` accepts a) an object with a ``read(size)`` method that will
-return  compressed bytes b) an object conforming to the buffer protocol that
-can expose its data as a contiguous range of bytes. The ``bytes`` and
-``memoryview`` types expose this buffer protocol.
+``read_to_iter()`` accepts an object with a ``read(size)`` method that will
+return  compressed bytes or an object conforming to the buffer protocol that
+can expose its data as a contiguous range of bytes.
 
-``read_from()`` returns an iterator whose elements are chunks of the
+``read_to_iter()`` returns an iterator whose elements are chunks of the
 decompressed data.
 
 The size of requested ``read()`` from the source can be specified::
 
     dctx = zstd.ZstdDecompressor()
-    for chunk in dctx.read_from(fh, read_size=16384):
+    for chunk in dctx.read_to_iter(fh, read_size=16384):
         pass
 
 It is also possible to skip leading bytes in the input data::
 
     dctx = zstd.ZstdDecompressor()
-    for chunk in dctx.read_from(fh, skip_bytes=1):
+    for chunk in dctx.read_to_iter(fh, skip_bytes=1):
         pass
 
-Skipping leading bytes is useful if the source data contains extra
-*header* data but you want to avoid the overhead of making a buffer copy
-or allocating a new ``memoryview`` object in order to decompress the data.
+.. tip::
 
-Similarly to ``ZstdCompressor.read_from()``, the consumer of the iterator
+   Skipping leading bytes is useful if the source data contains extra
+   *header* data. Traditionally, you would need to create a slice or
+   ``memoryview`` of the data you want to decompress. This would create
+   overhead. It is more efficient to pass the offset into this API.
+
+Similarly to ``ZstdCompressor.read_to_iter()``, the consumer of the iterator
 controls when data is decompressed. If the iterator isn't consumed,
 decompression is put on hold.
 
-When ``read_from()`` is passed an object conforming to the buffer protocol,
+When ``read_to_iter()`` is passed an object conforming to the buffer protocol,
 the behavior may seem similar to what occurs when the simple decompression
 API is used. However, this API works when the decompressed size is unknown.
 Furthermore, if feeding large inputs, the decompressor will work in chunks
@@ -636,7 +682,7 b' Decompressor API'
 ^^^^^^^^^^^^^^^^
 
 ``decompressobj()`` returns an object that exposes a ``decompress(data)``
-methods. Compressed data chunks are fed into ``decompress(data)`` and
+method. Compressed data chunks are fed into ``decompress(data)`` and
 uncompressed output (or an empty bytes) is returned. Output from subsequent
 calls needs to be concatenated to reassemble the full decompressed byte
 sequence.
@@ -650,11 +696,25 b' can no longer be called.'
 
 Here is how this API should be used::
 
-   dctx = zstd.ZstdDeompressor()
-   dobj = cctx.decompressobj()
+   dctx = zstd.ZstdDecompressor()
+   dobj = dctx.decompressobj()
    data = dobj.decompress(compressed_chunk_0)
    data = dobj.decompress(compressed_chunk_1)
 
+By default, calls to ``decompress()`` write output data in chunks of size
+``DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE``. These chunks are concatenated
+before being returned to the caller. It is possible to define the size of
+these temporary chunks by passing ``write_size`` to ``decompressobj()``::
+
+   dctx = zstd.ZstdDecompressor()
+   dobj = dctx.decompressobj(write_size=1048576)
+
+.. note::
+
+   Because calls to ``decompress()`` may need to perform multiple
+   memory (re)allocations, this streaming decompression API isn't as
+   efficient as other APIs.
+
 Batch Decompression API
 ^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -671,9 +731,12 b' conform to the buffer protocol. For best'
 minimal input validation will be done for that type. If calling from
 Python (as opposed to C), constructing one of these instances may add
 overhead cancelling out the performance overhead of validation for list
-inputs.
+inputs.::
 
-The decompressed size of each frame must be discoverable. It can either be
+    dctx = zstd.ZstdDecompressor()
+    results = dctx.multi_decompress_to_buffer([b'...', b'...'])
+
+The decompressed size of each frame MUST be discoverable. It can either be
 embedded within the zstd frame (``write_content_size=True`` argument to
 ``ZstdCompressor``) or passed in via the ``decompressed_sizes`` argument.
 
@@ -681,7 +744,13 b' The ``decompressed_sizes`` argument is a'
 protocol which holds an array of 64-bit unsigned integers in the machine's
 native format defining the decompressed sizes of each frame. If this argument
 is passed, it avoids having to scan each frame for its decompressed size.
-This frame scanning can add noticeable overhead in some scenarios.
+This frame scanning can add noticeable overhead in some scenarios.::
+
+    frames = [...]
+    sizes = struct.pack('=QQQQ', len0, len1, len2, len3)
+
+    dctx = zstd.ZstdDecompressor()
+    results = dctx.multi_decompress_to_buffer(frames, decompressed_sizes=sizes)
 
 The ``threads`` argument controls the number of threads to use to perform
 decompression operations. The default (``0``) or the value ``1`` means to
@@ -701,22 +770,23 b' This function exists to perform decompre'
 as possible by having as little overhead as possible. Since decompression is
 performed as a single operation and since the decompressed output is stored in
 a single buffer, extra memory allocations, Python objects, and Python function
-calls are avoided. This is ideal for scenarios where callers need to access
-decompressed data for multiple frames.
+calls are avoided. This is ideal for scenarios where callers know up front that
+they need to access data for multiple frames, such as when  *delta chains* are
+being used.
 
 Currently, the implementation always spawns multiple threads when requested,
 even if the amount of work to do is small. In the future, it will be smarter
 about avoiding threads and their associated overhead when the amount of
 work to do is small.
 
-Content-Only Dictionary Chain Decompression
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Prefix Dictionary Chain Decompression
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 ``decompress_content_dict_chain(frames)`` performs decompression of a list of
-zstd frames produced using chained *content-only* dictionary compression. Such
+zstd frames produced using chained *prefix* dictionary compression. Such
 a list of frames is produced by compressing discrete inputs where each
-non-initial input is compressed with a *content-only* dictionary consisting
-of the content of the previous input.
+non-initial input is compressed with a *prefix* dictionary consisting of the
+content of the previous input.
 
 For example, say you have the following inputs::
 
@@ -725,25 +795,25 b' For example, say you have the following '
 The zstd frame chain consists of:
 
 1. ``b'input 1'`` compressed in standalone/discrete mode
-2. ``b'input 2'`` compressed using ``b'input 1'`` as a *content-only* dictionary
-3. ``b'input 3'`` compressed using ``b'input 2'`` as a *content-only* dictionary
+2. ``b'input 2'`` compressed using ``b'input 1'`` as a *prefix* dictionary
+3. ``b'input 3'`` compressed using ``b'input 2'`` as a *prefix* dictionary
 
 Each zstd frame **must** have the content size written.
 
-The following Python code can be used to produce a *content-only dictionary
-chain*::
+The following Python code can be used to produce a *prefix dictionary chain*::
 
     def make_chain(inputs):
         frames = []
 
         # First frame is compressed in standalone/discrete mode.
-        zctx = zstd.ZstdCompressor(write_content_size=True)
+        zctx = zstd.ZstdCompressor()
         frames.append(zctx.compress(inputs[0]))
 
-        # Subsequent frames use the previous fulltext as a content-only dictionary
+        # Subsequent frames use the previous fulltext as a prefix dictionary
         for i, raw in enumerate(inputs[1:]):
-            dict_data = zstd.ZstdCompressionDict(inputs[i])
-            zctx = zstd.ZstdCompressor(write_content_size=True, dict_data=dict_data)
+            dict_data = zstd.ZstdCompressionDict(
+                inputs[i], dict_type=zstd.DICT_TYPE_RAWCONTENT)
+            zctx = zstd.ZstdCompressor(dict_data=dict_data)
             frames.append(zctx.compress(raw))
 
         return frames
@@ -751,10 +821,13 b' chain*::'
 ``decompress_content_dict_chain()`` returns the uncompressed data of the last
 element in the input chain.
 
-It is possible to implement *content-only dictionary chain* decompression
-on top of other Python APIs. However, this function will likely be significantly
-faster, especially for long input chains, as it avoids the overhead of
-instantiating and passing around intermediate objects between C and Python.
+
+.. note::
+
+   It is possible to implement *prefix dictionary chain* decompression
+   on top of other APIs. However, this function will likely be faster -
+   especially for long input chains - as it avoids the overhead of instantiating
+   and passing around intermediate objects between C and Python.
 
 Multi-Threaded Compression
 --------------------------
@@ -764,9 +837,15 b' of threads to use for compression. The w'
 into segments and each segment is fed into a worker pool for compression. Once
 a segment is compressed, it is flushed/appended to the output.
 
+.. note::
+
+   These threads are created at the C layer and are not Python threads. So they
+   work outside the GIL. It is therefore possible to CPU saturate multiple cores
+   from Python.
+
 The segment size for multi-threaded compression is chosen from the window size
 of the compressor. This is derived from the ``window_log`` attribute of a
-``CompressionParameters`` instance. By default, segment sizes are in the 1+MB
+``ZstdCompressionParameters`` instance. By default, segment sizes are in the 1+MB
 range.
 
 If multi-threaded compression is requested and the input is smaller than the
@@ -785,31 +864,33 b' than non-multi-threaded compression. The'
 there is a CPU/wall time versus size trade off that may warrant investigation.
 
 Output from multi-threaded compression does not require any special handling
-on the decompression side. In other words, any zstd decompressor should be able
-to consume data produced with multi-threaded compression.
+on the decompression side. To the decompressor, data generated with single
+threaded compressor looks the same as data generated by a multi-threaded
+compressor and does not require any special handling or additional resource
+requirements.
 
 Dictionary Creation and Management
 ----------------------------------
 
-Compression dictionaries are represented as the ``ZstdCompressionDict`` type.
+Compression dictionaries are represented with the ``ZstdCompressionDict`` type.
 
 Instances can be constructed from bytes::
 
    dict_data = zstd.ZstdCompressionDict(data)
 
-It is possible to construct a dictionary from *any* data. Unless the
-data begins with a magic header, the dictionary will be treated as
-*content-only*. *Content-only* dictionaries allow compression operations
-that follow to reference raw data within the content. For one use of
-*content-only* dictionaries, see
-``ZstdDecompressor.decompress_content_dict_chain()``.
+It is possible to construct a dictionary from *any* data. If the data doesn't
+begin with a magic header, it will be treated as a *prefix* dictionary.
+*Prefix* dictionaries allow compression operations to reference raw data
+within the dictionary.
 
-More interestingly, instances can be created by *training* on sample data::
+It is possible to force the use of *prefix* dictionaries or to require a
+dictionary header:
 
-   dict_data = zstd.train_dictionary(size, samples)
+   dict_data = zstd.ZstdCompressionDict(data,
+                                        dict_type=zstd.DICT_TYPE_RAWCONTENT)
 
-This takes a list of bytes instances and creates and returns a
-``ZstdCompressionDict``.
+   dict_data = zstd.ZstdCompressionDict(data,
+                                        dict_type=zstd.DICT_TYPE_FULLDICT)
 
 You can see how many bytes are in the dictionary by calling ``len()``::
 
@@ -819,7 +900,7 b' You can see how many bytes are in the di'
 Once you have a dictionary, you can pass it to the objects performing
 compression and decompression::
 
-   dict_data = zstd.train_dictionary(16384, samples)
+   dict_data = zstd.train_dictionary(131072, samples)
 
    cctx = zstd.ZstdCompressor(dict_data=dict_data)
    for source_data in input_data:
@@ -829,7 +910,7 b' compression and decompression::'
    dctx = zstd.ZstdDecompressor(dict_data=dict_data)
    for compressed_data in input_data:
        buffer = io.BytesIO()
-       with dctx.write_to(buffer) as decompressor:
+       with dctx.stream_writer(buffer) as decompressor:
            decompressor.write(compressed_data)
        # Do something with raw data in ``buffer``.
 
@@ -843,56 +924,69 b' a ``ZstdCompressionDict`` later) via ``a'
    dict_data = zstd.train_dictionary(size, samples)
    raw_data = dict_data.as_bytes()
 
-The following named arguments to ``train_dictionary`` can also be used
-to further control dictionary generation.
+By default, when a ``ZstdCompressionDict`` is *attached* to a
+``ZstdCompressor``, each ``ZstdCompressor`` performs work to prepare the
+dictionary for use. This is fine if only 1 compression operation is being
+performed or if the ``ZstdCompressor`` is being reused for multiple operations.
+But if multiple ``ZstdCompressor`` instances are being used with the dictionary,
+this can add overhead.
 
-selectivity
-   Integer selectivity level. Default is 9. Larger values yield more data in
-   dictionary.
-level
-   Integer compression level. Default is 6.
-dict_id
-   Integer dictionary ID for the produced dictionary. Default is 0, which
-   means to use a random value.
-notifications
-   Controls writing of informational messages to ``stderr``. ``0`` (the
-   default) means to write nothing. ``1`` writes errors. ``2`` writes
-   progression info. ``3`` writes more details. And ``4`` writes all info.
+It is possible to *precompute* the dictionary so it can readily be consumed
+by multiple ``ZstdCompressor`` instances::
+
+    d = zstd.ZstdCompressionDict(data)
 
-Cover Dictionaries
-^^^^^^^^^^^^^^^^^^
+    # Precompute for compression level 3.
+    d.precompute_compress(level=3)
 
-An alternate dictionary training mechanism named *cover* is also available.
-More details about this training mechanism are available in the paper
-*Effective Construction of Relative Lempel-Ziv Dictionaries* (authors:
-Liao, Petri, Moffat, Wirth).
-
-To use this mechanism, use ``zstd.train_cover_dictionary()`` instead of
-``zstd.train_dictionary()``. The function behaves nearly the same except
-its arguments are different and the returned dictionary will contain ``k``
-and ``d`` attributes reflecting the parameters to the cover algorithm.
+    # Precompute with specific compression parameters.
+    params = zstd.ZstdCompressionParameters(...)
+    d.precompute_compress(compression_params=params)
 
 .. note::
 
-   The ``k`` and ``d`` attributes are only populated on dictionary
-   instances created by this function. If a ``ZstdCompressionDict`` is
-   constructed from raw bytes data, the ``k`` and ``d`` attributes will
-   be ``0``.
+   When a dictionary is precomputed, the compression parameters used to
+   precompute the dictionary overwrite some of the compression parameters
+   specified to ``ZstdCompressor.__init__``.
+
+Training Dictionaries
+^^^^^^^^^^^^^^^^^^^^^
+
+Unless using *prefix* dictionaries, dictionary data is produced by *training*
+on existing data::
+
+   dict_data = zstd.train_dictionary(size, samples)
+
+This takes a target dictionary size and list of bytes instances and creates and
+returns a ``ZstdCompressionDict``.
+
+The dictionary training mechanism is known as *cover*. More details about it are
+available in the paper *Effective Construction of Relative Lempel-Ziv
+Dictionaries* (authors: Liao, Petri, Moffat, Wirth).
+
+The cover algorithm takes parameters ``k` and ``d``. These are the
+*segment size* and *dmer size*, respectively. The returned dictionary
+instance created by this function has ``k`` and ``d`` attributes
+containing the values for these parameters. If a ``ZstdCompressionDict``
+is constructed from raw bytes data (a content-only dictionary), the
+``k`` and ``d`` attributes will be ``0``.
 
 The segment and dmer size parameters to the cover algorithm can either be
-specified manually or you can ask ``train_cover_dictionary()`` to try
-multiple values and pick the best one, where *best* means the smallest
-compressed data size.
-
-In manual mode, the ``k`` and ``d`` arguments must be specified or a
-``ZstdError`` will be raised.
+specified manually or ``train_dictionary()`` can try multiple values
+and pick the best one, where *best* means the smallest compressed data size.
+This later mode is called *optimization* mode.
 
-In automatic mode (triggered by specifying ``optimize=True``), ``k``
-and ``d`` are optional. If a value isn't specified, then default values for
-both are tested.  The ``steps`` argument can control the number of steps
-through ``k`` values. The ``level`` argument defines the compression level
-that will be used when testing the compressed size. And ``threads`` can
-specify the number of threads to use for concurrent operation.
+If none of ``k``, ``d``, ``steps``, ``threads``, ``level``, ``notifications``,
+or ``dict_id`` (basically anything from the underlying ``ZDICT_cover_params_t``
+struct) are defined, *optimization* mode is used with default parameter
+values.
+
+If ``steps`` or ``threads`` are defined, then *optimization* mode is engaged
+with explicit control over those parameters. Specifying ``threads=0`` or
+``threads=1`` can be used to engage *optimization* mode if other parameters
+are not defined.
+
+Otherwise, non-*optimization* mode is used with the parameters specified.
 
 This function takes the following arguments:
 
@@ -909,64 +1003,92 b' d'
 dict_id
    Integer dictionary ID for the produced dictionary. Default is 0, which uses
    a random value.
-optimize
-   When true, test dictionary generation with multiple parameters.
+steps
+   Number of steps through ``k`` values to perform when trying parameter
+   variations.
+threads
+   Number of threads to use when trying parameter variations. Default is 0,
+   which means to use a single thread. A negative value can be specified to
+   use as many threads as there are detected logical CPUs.
 level
-   Integer target compression level when testing compression with
-   ``optimize=True``. Default is 1.
-steps
-   Number of steps through ``k`` values to perform when ``optimize=True``.
-   Default is 32.
-threads
-   Number of threads to use when ``optimize=True``. Default is 0, which means
-   to use a single thread. A negative value can be specified to use as many
-   threads as there are detected logical CPUs.
+   Integer target compression level when trying parameter variations.
 notifications
-   Controls writing of informational messages to ``stderr``. See the
-   documentation for ``train_dictionary()`` for more.
+   Controls writing of informational messages to ``stderr``. ``0`` (the
+   default) means to write nothing. ``1`` writes errors. ``2`` writes
+   progression info. ``3`` writes more details. And ``4`` writes all info.
 
 Explicit Compression Parameters
 -------------------------------
 
-Zstandard's integer compression levels along with the input size and dictionary
-size are converted into a data structure defining multiple parameters to tune
-behavior of the compression algorithm. It is possible to use define this
-data structure explicitly to have lower-level control over compression behavior.
+Zstandard offers a high-level *compression level* that maps to lower-level
+compression parameters. For many consumers, this numeric level is the only
+compression setting you'll need to touch.
+
+But for advanced use cases, it might be desirable to tweak these lower-level
+settings.
 
-The ``zstd.CompressionParameters`` type represents this data structure.
-You can see how Zstandard converts compression levels to this data structure
-by calling ``zstd.get_compression_parameters()``. e.g.::
+The ``ZstdCompressionParameters`` type represents these low-level compression
+settings.
 
-    params = zstd.get_compression_parameters(5)
+Instances of this type can be constructed from a myriad of keyword arguments
+(defined below) for complete low-level control over each adjustable
+compression setting.
+
+From a higher level, one can construct a ``ZstdCompressionParameters`` instance
+given a desired compression level and target input and dictionary size
+using ``ZstdCompressionParameters.from_level()``. e.g.::
 
-This function also accepts the uncompressed data size and dictionary size
-to adjust parameters::
+    # Derive compression settings for compression level 7.
+    params = zstd.ZstdCompressionParameters.from_level(7)
 
-    params = zstd.get_compression_parameters(3, source_size=len(data), dict_size=len(dict_data))
+    # With an input size of 1MB
+    params = zstd.ZstdCompressionParameters.from_level(7, source_size=1048576)
+
+Using ``from_level()``, it is also possible to override individual compression
+parameters or to define additional settings that aren't automatically derived.
+e.g.::
 
-You can also construct compression parameters from their low-level components::
+    params = zstd.ZstdCompressionParameters.from_level(4, window_log=10)
+    params = zstd.ZstdCompressionParameters.from_level(5, threads=4)
+
+Or you can define low-level compression settings directly::
 
-    params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST)
+    params = zstd.ZstdCompressionParameters(window_log=12, enable_ldm=True)
 
-You can then configure a compressor to use the custom parameters::
+Once a ``ZstdCompressionParameters`` instance is obtained, it can be used to
+configure a compressor::
 
     cctx = zstd.ZstdCompressor(compression_params=params)
 
-The members/attributes of ``CompressionParameters`` instances are as follows::
+The named arguments and attributes of ``ZstdCompressionParameters`` are as
+follows:
 
+* format
+* compression_level
 * window_log
+* hash_log
 * chain_log
-* hash_log
 * search_log
-* search_length
+* min_match
 * target_length
-* strategy
+* compression_strategy
+* write_content_size
+* write_checksum
+* write_dict_id
+* job_size
+* overlap_size_log
+* compress_literals
+* force_max_window
+* enable_ldm
+* ldm_hash_log
+* ldm_min_match
+* ldm_bucket_size_log
+* ldm_hash_every_log
+* threads
 
-This is the order the arguments are passed to the constructor if not using
-named arguments.
-
-You'll need to read the Zstandard documentation for what these parameters
-do.
+Some of these are very low-level settings. It may help to consult the official
+zstandard documentation for their behavior. Look for the ``ZSTD_p_*`` constants
+in ``zstd.h`` (https://github.com/facebook/zstd/blob/dev/lib/zstd.h).
 
 Frame Inspection
 ----------------
@@ -1003,15 +1125,17 b' has_checksum'
    Bool indicating whether a 4 byte content checksum is stored at the end
    of the frame.
 
+``zstd.frame_header_size(data)`` returns the size of the zstandard frame
+header.
+
+``zstd.frame_content_size(data)`` returns the content size as parsed from
+the frame header. ``-1`` means the content size is unknown. ``0`` means
+an empty frame. The content size is usually correct. However, it may not
+be accurate.
+
 Misc Functionality
 ------------------
 
-estimate_compression_context_size(CompressionParameters)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Given a ``CompressionParameters`` struct, estimate the memory size required
-to perform compression.
-
 estimate_decompression_context_size()
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -1041,6 +1165,11 b' FRAME_HEADER'
 MAGIC_NUMBER
     Frame header as an integer
 
+CONTENTSIZE_UNKNOWN
+    Value for content size when the content size is unknown.
+CONTENTSIZE_ERROR
+    Value for content size when content size couldn't be determined.
+
 WINDOWLOG_MIN
     Minimum value for compression parameter
 WINDOWLOG_MAX
@@ -1063,8 +1192,6 b' SEARCHLENGTH_MAX'
     Maximum value for compression parameter
 TARGETLENGTH_MIN
     Minimum value for compression parameter
-TARGETLENGTH_MAX
-    Maximum value for compression parameter
 STRATEGY_FAST
     Compression strategy
 STRATEGY_DFAST
@@ -1079,6 +1206,13 b' STRATEGY_BTLAZY2'
     Compression strategy
 STRATEGY_BTOPT
     Compression strategy
+STRATEGY_BTULTRA
+    Compression strategy
+
+FORMAT_ZSTD1
+    Zstandard frame format
+FORMAT_ZSTD1_MAGICLESS
+    Zstandard frame format without magic header
 
 Performance Considerations
 --------------------------
@@ -1090,7 +1224,7 b' instantiating a new ``ZstdCompressor`` o'
 operation. The differences are magnified as the size of data decreases. For
 example, the difference between *context* reuse and non-reuse for 100,000
 100 byte inputs will be significant (possiby over 10x faster to reuse contexts)
-whereas 10 1,000,000 byte inputs will be more similar in speed (because the
+whereas 10 100,000,000 byte inputs will be more similar in speed (because the
 time spent doing compression dwarfs time spent creating new *contexts*).
 
 Buffer Types
@@ -1187,9 +1321,8 b' There are multiple APIs for performing c'
 because different applications have different needs and the library wants to
 facilitate optimal use in as many use cases as possible.
 
-From a high-level, APIs are divided into *one-shot* and *streaming*. See
-the ``Concepts`` section for a description of how these are different at
-the C layer.
+From a high-level, APIs are divided into *one-shot* and *streaming*: either you
+are operating on all data at once or you operate on it piecemeal.
 
 The *one-shot* APIs are useful for small data, where the input or output
 size is known. (The size can come from a buffer length, file size, or
@@ -1222,145 +1355,39 b' There is potential for long pauses as da'
 underlying stream (say from interacting with a filesystem or network). This
 could add considerable overhead.
 
-Concepts
-========
-
-It is important to have a basic understanding of how Zstandard works in order
-to optimally use this library. In addition, there are some low-level Python
-concepts that are worth explaining to aid understanding. This section aims to
-provide that knowledge.
-
-Zstandard Frames and Compression Format
----------------------------------------
-
-Compressed zstandard data almost always exists within a container called a
-*frame*. (For the technically curious, see the
-`specification <https://github.com/facebook/zstd/blob/3bee41a70eaf343fbcae3637b3f6edbe52f35ed8/doc/zstd_compression_format.md>_.)
-
-The frame contains a header and optional trailer. The header contains a
-magic number to self-identify as a zstd frame and a description of the
-compressed data that follows.
-
-Among other things, the frame *optionally* contains the size of the
-decompressed data the frame represents, a 32-bit checksum of the
-decompressed data (to facilitate verification during decompression),
-and the ID of the dictionary used to compress the data.
-
-Storing the original content size in the frame (``write_content_size=True``
-to ``ZstdCompressor``) is important for performance in some scenarios. Having
-the decompressed size stored there (or storing it elsewhere) allows
-decompression to perform a single memory allocation that is exactly sized to
-the output. This is faster than continuously growing a memory buffer to hold
-output.
+Thread Safety
+=============
 
-Compression and Decompression Contexts
---------------------------------------
-
-In order to perform a compression or decompression operation with the zstd
-C API, you need what's called a *context*. A context essentially holds
-configuration and state for a compression or decompression operation. For
-example, a compression context holds the configured compression level.
-
-Contexts can be reused for multiple operations. Since creating and
-destroying contexts is not free, there are performance advantages to
-reusing contexts.
-
-The ``ZstdCompressor`` and ``ZstdDecompressor`` types are essentially
-wrappers around these contexts in the zstd C API.
+``ZstdCompressor`` and ``ZstdDecompressor`` instances have no guarantees
+about thread safety. Do not operate on the same ``ZstdCompressor`` and
+``ZstdDecompressor`` instance simultaneously from different threads. It is
+fine to have different threads call into a single instance, just not at the
+same time.
 
-One-shot And Streaming Operations
----------------------------------
-
-A compression or decompression operation can either be performed as a
-single *one-shot* operation or as a continuous *streaming* operation.
-
-In one-shot mode (the *simple* APIs provided by the Python interface),
-**all** input is handed to the compressor or decompressor as a single buffer
-and **all** output is returned as a single buffer.
-
-In streaming mode, input is delivered to the compressor or decompressor as
-a series of chunks via multiple function calls. Likewise, output is
-obtained in chunks as well.
-
-Streaming operations require an additional *stream* object to be created
-to track the operation. These are logical extensions of *context*
-instances.
+Some operations require multiple function calls to complete. e.g. streaming
+operations. A single ``ZstdCompressor`` or ``ZstdDecompressor`` cannot be used
+for simultaneously active operations. e.g. you must not start a streaming
+operation when another streaming operation is already active.
 
-There are advantages and disadvantages to each mode of operation. There
-are scenarios where certain modes can't be used. See the
-``Choosing an API`` section for more.
-
-Dictionaries
-------------
-
-A compression *dictionary* is essentially data used to seed the compressor
-state so it can achieve better compression. The idea is that if you are
-compressing a lot of similar pieces of data (e.g. JSON documents or anything
-sharing similar structure), then you can find common patterns across multiple
-objects then leverage those common patterns during compression and
-decompression operations to achieve better compression ratios.
-
-Dictionary compression is generally only useful for small inputs - data no
-larger than a few kilobytes. The upper bound on this range is highly dependent
-on the input data and the dictionary.
-
-Python Buffer Protocol
-----------------------
-
-Many functions in the library operate on objects that implement Python's
-`buffer protocol <https://docs.python.org/3.6/c-api/buffer.html>`_.
-
-The *buffer protocol* is an internal implementation detail of a Python
-type that allows instances of that type (objects) to be exposed as a raw
-pointer (or buffer) in the C API. In other words, it allows objects to be
-exposed as an array of bytes.
+The C extension releases the GIL during non-trivial calls into the zstd C
+API. Non-trivial calls are notably compression and decompression. Trivial
+calls are things like parsing frame parameters. Where the GIL is released
+is considered an implementation detail and can change in any release.
 
-From the perspective of the C API, objects implementing the *buffer protocol*
-all look the same: they are just a pointer to a memory address of a defined
-length. This allows the C API to be largely type agnostic when accessing their
-data. This allows custom types to be passed in without first converting them
-to a specific type.
-
-Many Python types implement the buffer protocol. These include ``bytes``
-(``str`` on Python 2), ``bytearray``, ``array.array``, ``io.BytesIO``,
-``mmap.mmap``, and ``memoryview``.
-
-``python-zstandard`` APIs that accept objects conforming to the buffer
-protocol require that the buffer is *C contiguous* and has a single
-dimension (``ndim==1``). This is usually the case. An example of where it
-is not is a Numpy matrix type.
-
-Requiring Output Sizes for Non-Streaming Decompression APIs
------------------------------------------------------------
-
-Non-streaming decompression APIs require that either the output size is
-explicitly defined (either in the zstd frame header or passed into the
-function) or that a max output size is specified. This restriction is for
-your safety.
-
-The *one-shot* decompression APIs store the decompressed result in a
-single buffer. This means that a buffer needs to be pre-allocated to hold
-the result. If the decompressed size is not known, then there is no universal
-good default size to use. Any default will fail or will be highly sub-optimal
-in some scenarios (it will either be too small or will put stress on the
-memory allocator to allocate a too large block).
-
-A *helpful* API may retry decompression with buffers of increasing size.
-While useful, there are obvious performance disadvantages, namely redoing
-decompression N times until it works. In addition, there is a security
-concern. Say the input came from highly compressible data, like 1 GB of the
-same byte value. The output size could be several magnitudes larger than the
-input size. An input of <100KB could decompress to >1GB. Without a bounds
-restriction on the decompressed size, certain inputs could exhaust all system
-memory. That's not good and is why the maximum output size is limited.
+APIs that accept bytes-like objects don't enforce that the underlying object
+is read-only. However, it is assumed that the passed object is read-only for
+the duration of the function call. It is possible to pass a mutable object
+(like a ``bytearray``) to e.g. ``ZstdCompressor.compress()``, have the GIL
+released, and mutate the object from another thread. Such a race condition
+is a bug in the consumer of python-zstandard. Most Python data types are
+immutable, so unless you are doing something fancy, you don't need to
+worry about this.
 
 Note on Zstandard's *Experimental* API
 ======================================
 
 Many of the Zstandard APIs used by this module are marked as *experimental*
-within the Zstandard project. This includes a large number of useful
-features, such as compression and frame parameters and parts of dictionary
-compression.
+within the Zstandard project.
 
 It is unclear how Zstandard's C API will evolve over time, especially with
 regards to this *experimental* functionality. We will try to maintain
@@ -1371,7 +1398,7 b' Since a copy of the Zstandard source cod'
 module and since we compile against it, the behavior of a specific
 version of this module should be constant for all of time. So if you
 pin the version of this module used in your projects (which is a Python
-best practice), you should be buffered from unwanted future changes.
+best practice), you should be shielded from unwanted future changes.
 
 Donate
 ======

contrib/python-zstandard/c-ext/bufferutil.c

@@ -83,7 +83,7 b' static int BufferWithSegments_init(ZstdB'
 	}
 
 	if (segments.len % sizeof(BufferSegment)) {
-		PyErr_Format(PyExc_ValueError, "segments array size is not a multiple of %lu",
+		PyErr_Format(PyExc_ValueError, "segments array size is not a multiple of %zu",
 			sizeof(BufferSegment));
 		goto except;
 	}
@@ -123,7 +123,7 b' except:'
 	PyBuffer_Release(&self->parent);
 	PyBuffer_Release(&segments);
 	return -1;
-};
+}
 
 /**
  * Construct a BufferWithSegments from existing memory and offsets.
@@ -188,6 +188,12 b' static ZstdBufferSegment* BufferWithSegm'
 		return NULL;
 	}
 
+	if (self->segments[i].length > PY_SSIZE_T_MAX) {
+		PyErr_Format(PyExc_ValueError,
+			"item at offset %zd is too large for this platform", i);
+		return NULL;
+	}
+
 	result = (ZstdBufferSegment*)PyObject_CallObject((PyObject*)&ZstdBufferSegmentType, NULL);
 	if (NULL == result) {
 		return NULL;
@@ -197,7 +203,7 b' static ZstdBufferSegment* BufferWithSegm'
 	Py_INCREF(self);
 
 	result->data = (char*)self->data + self->segments[i].offset;
-	result->dataSize = self->segments[i].length;
+	result->dataSize = (Py_ssize_t)self->segments[i].length;
 	result->offset = self->segments[i].offset;
 
 	return result;
@@ -205,7 +211,13 b' static ZstdBufferSegment* BufferWithSegm'
 
 #if PY_MAJOR_VERSION >= 3
 static int BufferWithSegments_getbuffer(ZstdBufferWithSegments* self, Py_buffer* view, int flags) {
-	return PyBuffer_FillInfo(view, (PyObject*)self, self->data, self->dataSize, 1, flags);
+	if (self->dataSize > PY_SSIZE_T_MAX) {
+		view->obj = NULL;
+		PyErr_SetString(PyExc_BufferError, "buffer is too large for this platform");
+		return -1;
+	}
+
+	return PyBuffer_FillInfo(view, (PyObject*)self, self->data, (Py_ssize_t)self->dataSize, 1, flags);
 }
 #else
 static Py_ssize_t BufferWithSegments_getreadbuffer(ZstdBufferWithSegments* self, Py_ssize_t segment, void **ptrptr) {
@@ -214,8 +226,13 b' static Py_ssize_t BufferWithSegments_get'
 		return -1;
 	}
 
+	if (self->dataSize > PY_SSIZE_T_MAX) {
+		PyErr_SetString(PyExc_ValueError, "buffer is too large for this platform");
+		return -1;
+	}
+
 	*ptrptr = self->data;
-	return self->dataSize;
+	return (Py_ssize_t)self->dataSize;
 }
 
 static Py_ssize_t BufferWithSegments_getsegcount(ZstdBufferWithSegments* self, Py_ssize_t* len) {
@@ -232,7 +249,12 b' PyDoc_STRVAR(BufferWithSegments_tobytes_'
 );
 
 static PyObject* BufferWithSegments_tobytes(ZstdBufferWithSegments* self) {
-	return PyBytes_FromStringAndSize(self->data, self->dataSize);
+	if (self->dataSize > PY_SSIZE_T_MAX) {
+		PyErr_SetString(PyExc_ValueError, "buffer is too large for this platform");
+		return NULL;
+	}
+
+	return PyBytes_FromStringAndSize(self->data, (Py_ssize_t)self->dataSize);
 }
 
 PyDoc_STRVAR(BufferWithSegments_segments__doc__,

contrib/python-zstandard/c-ext/compressiondict.c

@@ -14,125 +14,11 b' ZstdCompressionDict* train_dictionary(Py'
 	static char* kwlist[] = {
 		"dict_size",
 		"samples",
-		"selectivity",
-		"level",
-		"notifications",
-		"dict_id",
-		NULL
-	};
-	size_t capacity;
-	PyObject* samples;
-	Py_ssize_t samplesLen;
-	unsigned  selectivity = 0;
-	int level = 0;
-	unsigned notifications = 0;
-	unsigned dictID = 0;
-	ZDICT_params_t zparams;
-	Py_ssize_t sampleIndex;
-	Py_ssize_t sampleSize;
-	PyObject* sampleItem;
-	size_t zresult;
-	void* sampleBuffer = NULL;
-	void* sampleOffset;
-	size_t samplesSize = 0;
-	size_t* sampleSizes = NULL;
-	void* dict = NULL;
-	ZstdCompressionDict* result = NULL;
-
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|IiII:train_dictionary",
-		kwlist,
-		&capacity,
-		&PyList_Type, &samples,
-		&selectivity, &level, &notifications, &dictID)) {
-		return NULL;
-	}
-
-	memset(&zparams, 0, sizeof(zparams));
-
-	zparams.selectivityLevel = selectivity;
-	zparams.compressionLevel = level;
-	zparams.notificationLevel = notifications;
-	zparams.dictID = dictID;
-
-	/* Figure out the size of the raw samples */
-	samplesLen = PyList_Size(samples);
-	for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) {
-		sampleItem = PyList_GetItem(samples, sampleIndex);
-		if (!PyBytes_Check(sampleItem)) {
-			PyErr_SetString(PyExc_ValueError, "samples must be bytes");
-			return NULL;
-		}
-		samplesSize += PyBytes_GET_SIZE(sampleItem);
-	}
-
-	/* Now that we know the total size of the raw simples, we can allocate
-	a buffer for the raw data */
-	sampleBuffer = PyMem_Malloc(samplesSize);
-	if (!sampleBuffer) {
-		PyErr_NoMemory();
-		goto finally;
-	}
-	sampleSizes = PyMem_Malloc(samplesLen * sizeof(size_t));
-	if (!sampleSizes) {
-		PyErr_NoMemory();
-		goto finally;
-	}
-
-	sampleOffset = sampleBuffer;
-	/* Now iterate again and assemble the samples in the buffer */
-	for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) {
-		sampleItem = PyList_GetItem(samples, sampleIndex);
-		sampleSize = PyBytes_GET_SIZE(sampleItem);
-		sampleSizes[sampleIndex] = sampleSize;
-		memcpy(sampleOffset, PyBytes_AS_STRING(sampleItem), sampleSize);
-		sampleOffset = (char*)sampleOffset + sampleSize;
-	}
-
-	dict = PyMem_Malloc(capacity);
-	if (!dict) {
-		PyErr_NoMemory();
-		goto finally;
-	}
-
-	/* TODO consider using dup2() to redirect zstd's stderr writing to a buffer */
-	Py_BEGIN_ALLOW_THREADS
-	zresult = ZDICT_trainFromBuffer_advanced(dict, capacity,
-		sampleBuffer, sampleSizes, (unsigned int)samplesLen,
-		zparams);
-	Py_END_ALLOW_THREADS
-	if (ZDICT_isError(zresult)) {
-		PyErr_Format(ZstdError, "Cannot train dict: %s", ZDICT_getErrorName(zresult));
-		PyMem_Free(dict);
-		goto finally;
-	}
-
-	result = PyObject_New(ZstdCompressionDict, &ZstdCompressionDictType);
-	if (!result) {
-		goto finally;
-	}
-
-	result->dictData = dict;
-	result->dictSize = zresult;
-	result->d = 0;
-	result->k = 0;
-
-finally:
-	PyMem_Free(sampleBuffer);
-	PyMem_Free(sampleSizes);
-
-	return result;
-}
-
-ZstdCompressionDict* train_cover_dictionary(PyObject* self, PyObject* args, PyObject* kwargs) {
-	static char* kwlist[] = {
-		"dict_size",
-		"samples",
 		"k",
 		"d",
 		"notifications",
 		"dict_id",
 		"level",
-		"optimize",
 		"steps",
 		"threads",
 		NULL
@@ -145,10 +31,9 b' ZstdCompressionDict* train_cover_diction'
 	unsigned notifications = 0;
 	unsigned dictID = 0;
 	int level = 0;
-	PyObject* optimize = NULL;
 	unsigned steps = 0;
 	int threads = 0;
-	COVER_params_t params;
+	ZDICT_cover_params_t params;
 	Py_ssize_t samplesLen;
 	Py_ssize_t i;
 	size_t samplesSize = 0;
@@ -160,9 +45,9 b' ZstdCompressionDict* train_cover_diction'
 	size_t zresult;
 	ZstdCompressionDict* result = NULL;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|IIIIiOIi:train_cover_dictionary",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|IIIIiIi:train_dictionary",
 		kwlist, &capacity, &PyList_Type, &samples,
-		&k, &d, &notifications, &dictID, &level, &optimize, &steps, &threads)) {
+		&k, &d, &notifications, &dictID, &level, &steps, &threads)) {
 		return NULL;
 	}
 
@@ -175,9 +60,9 b' ZstdCompressionDict* train_cover_diction'
 	params.d = d;
 	params.steps = steps;
 	params.nbThreads = threads;
-	params.notificationLevel = notifications;
-	params.dictID = dictID;
-	params.compressionLevel = level;
+	params.zParams.notificationLevel = notifications;
+	params.zParams.dictID = dictID;
+	params.zParams.compressionLevel = level;
 
 	/* Figure out total size of input samples. */
 	samplesLen = PyList_Size(samples);
@@ -219,12 +104,21 b' ZstdCompressionDict* train_cover_diction'
 	}
 
 	Py_BEGIN_ALLOW_THREADS
-	if (optimize && PyObject_IsTrue(optimize)) {
-		zresult = COVER_optimizeTrainFromBuffer(dict, capacity,
+	/* No parameters uses the default function, which will use default params
+	   and call ZDICT_optimizeTrainFromBuffer_cover under the hood. */
+	if (!params.k && !params.d && !params.zParams.compressionLevel
+		&& !params.zParams.notificationLevel && !params.zParams.dictID) {
+		zresult = ZDICT_trainFromBuffer(dict, capacity, sampleBuffer,
+			sampleSizes, (unsigned)samplesLen);
+	}
+	/* Use optimize mode if user controlled steps or threads explicitly. */
+	else if (params.steps || params.nbThreads) {
+		zresult = ZDICT_optimizeTrainFromBuffer_cover(dict, capacity,
 			sampleBuffer, sampleSizes, (unsigned)samplesLen, &params);
 	}
+	/* Non-optimize mode with explicit control. */
 	else {
-		zresult = COVER_trainFromBuffer(dict, capacity,
+		zresult = ZDICT_trainFromBuffer_cover(dict, capacity,
 			sampleBuffer, sampleSizes, (unsigned)samplesLen, params);
 	}
 	Py_END_ALLOW_THREADS
@@ -243,8 +137,11 b' ZstdCompressionDict* train_cover_diction'
 
 	result->dictData = dict;
 	result->dictSize = zresult;
+	result->dictType = ZSTD_dct_fullDict;
 	result->d = params.d;
 	result->k = params.k;
+	result->cdict = NULL;
+	result->ddict = NULL;
 
 finally:
 	PyMem_Free(sampleBuffer);
@@ -253,43 +150,99 b' finally:'
 	return result;
 }
 
+int ensure_ddict(ZstdCompressionDict* dict) {
+	if (dict->ddict) {
+		return 0;
+	}
+
+	Py_BEGIN_ALLOW_THREADS
+	dict->ddict = ZSTD_createDDict_advanced(dict->dictData, dict->dictSize,
+		ZSTD_dlm_byRef, dict->dictType, ZSTD_defaultCMem);
+	Py_END_ALLOW_THREADS
+	if (!dict->ddict) {
+		PyErr_SetString(ZstdError, "could not create decompression dict");
+		return 1;
+	}
+
+	return 0;
+}
+
 PyDoc_STRVAR(ZstdCompressionDict__doc__,
 "ZstdCompressionDict(data) - Represents a computed compression dictionary\n"
 "\n"
 "This type holds the results of a computed Zstandard compression dictionary.\n"
-"Instances are obtained by calling ``train_dictionary()`` or by passing bytes\n"
-"obtained from another source into the constructor.\n"
+"Instances are obtained by calling ``train_dictionary()`` or by passing\n"
+"bytes obtained from another source into the constructor.\n"
 );
 
-static int ZstdCompressionDict_init(ZstdCompressionDict* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
+static int ZstdCompressionDict_init(ZstdCompressionDict* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		"dict_type",
+		NULL
+	};
+
+	int result = -1;
+	Py_buffer source;
+	unsigned dictType = ZSTD_dct_auto;
 
 	self->dictData = NULL;
 	self->dictSize = 0;
+	self->cdict = NULL;
+	self->ddict = NULL;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:ZstdCompressionDict",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|I:ZstdCompressionDict",
 #else
-	if (!PyArg_ParseTuple(args, "s#:ZstdCompressionDict",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|I:ZstdCompressionDict",
 #endif
-		&source, &sourceSize)) {
+		kwlist, &source, &dictType)) {
 		return -1;
 	}
 
-	self->dictData = PyMem_Malloc(sourceSize);
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	if (dictType != ZSTD_dct_auto && dictType != ZSTD_dct_rawContent
+		&& dictType != ZSTD_dct_fullDict) {
+		PyErr_Format(PyExc_ValueError,
+			"invalid dictionary load mode: %d; must use DICT_TYPE_* constants",
+			dictType);
+		goto finally;
+	}
+
+	self->dictType = dictType;
+
+	self->dictData = PyMem_Malloc(source.len);
 	if (!self->dictData) {
 		PyErr_NoMemory();
-		return -1;
+		goto finally;
 	}
 
-	memcpy(self->dictData, source, sourceSize);
-	self->dictSize = sourceSize;
+	memcpy(self->dictData, source.buf, source.len);
+	self->dictSize = source.len;
+
+	result = 0;
 
-	return 0;
+finally:
+	PyBuffer_Release(&source);
+	return result;
+}
+
+static void ZstdCompressionDict_dealloc(ZstdCompressionDict* self) {
+	if (self->cdict) {
+		ZSTD_freeCDict(self->cdict);
+		self->cdict = NULL;
 	}
 
-static void ZstdCompressionDict_dealloc(ZstdCompressionDict* self) {
+	if (self->ddict) {
+		ZSTD_freeDDict(self->ddict);
+		self->ddict = NULL;
+	}
+
 	if (self->dictData) {
 		PyMem_Free(self->dictData);
 		self->dictData = NULL;
@@ -298,6 +251,74 b' static void ZstdCompressionDict_dealloc('
 	PyObject_Del(self);
 }
 
+PyDoc_STRVAR(ZstdCompressionDict_precompute_compress__doc__,
+"Precompute a dictionary so it can be used by multiple compressors.\n"
+);
+
+static PyObject* ZstdCompressionDict_precompute_compress(ZstdCompressionDict* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"level",
+		"compression_params",
+		NULL
+	};
+
+	int level = 0;
+	ZstdCompressionParametersObject* compressionParams = NULL;
+	ZSTD_compressionParameters cParams;
+	size_t zresult;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!:precompute_compress", kwlist,
+		&level, &ZstdCompressionParametersType, &compressionParams)) {
+		return NULL;
+	}
+
+	if (level && compressionParams) {
+		PyErr_SetString(PyExc_ValueError,
+			"must only specify one of level or compression_params");
+		return NULL;
+	}
+
+	if (!level && !compressionParams) {
+		PyErr_SetString(PyExc_ValueError,
+			"must specify one of level or compression_params");
+		return NULL;
+	}
+
+	if (self->cdict) {
+		zresult = ZSTD_freeCDict(self->cdict);
+		self->cdict = NULL;
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "unable to free CDict: %s",
+				ZSTD_getErrorName(zresult));
+			return NULL;
+		}
+	}
+
+	if (level) {
+		cParams = ZSTD_getCParams(level, 0, self->dictSize);
+	}
+	else {
+		cParams.chainLog = compressionParams->chainLog;
+		cParams.hashLog = compressionParams->hashLog;
+		cParams.searchLength = compressionParams->minMatch;
+		cParams.searchLog = compressionParams->searchLog;
+		cParams.strategy = compressionParams->compressionStrategy;
+		cParams.targetLength = compressionParams->targetLength;
+		cParams.windowLog = compressionParams->windowLog;
+	}
+
+	assert(!self->cdict);
+	self->cdict = ZSTD_createCDict_advanced(self->dictData, self->dictSize,
+		ZSTD_dlm_byRef, self->dictType, cParams, ZSTD_defaultCMem);
+
+	if (!self->cdict) {
+		PyErr_SetString(ZstdError, "unable to precompute dictionary");
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
 static PyObject* ZstdCompressionDict_dict_id(ZstdCompressionDict* self) {
 	unsigned dictID = ZDICT_getDictID(self->dictData, self->dictSize);
 
@@ -313,6 +334,8 b' static PyMethodDef ZstdCompressionDict_m'
 	PyDoc_STR("dict_id() -- obtain the numeric dictionary ID") },
 	{ "as_bytes", (PyCFunction)ZstdCompressionDict_as_bytes, METH_NOARGS,
 	PyDoc_STR("as_bytes() -- obtain the raw bytes constituting the dictionary data") },
+	{ "precompute_compress", (PyCFunction)ZstdCompressionDict_precompute_compress,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressionDict_precompute_compress__doc__ },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/compressionparams.c

@@ -8,204 +8,448 b''
 
 #include "python-zstandard.h"
 
-void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams) {
-	zparams->windowLog = params->windowLog;
-	zparams->chainLog = params->chainLog;
-	zparams->hashLog = params->hashLog;
-	zparams->searchLog = params->searchLog;
-	zparams->searchLength = params->searchLength;
-	zparams->targetLength = params->targetLength;
-	zparams->strategy = params->strategy;
-}
+extern PyObject* ZstdError;
 
-CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args) {
-	int compressionLevel;
-	unsigned PY_LONG_LONG sourceSize = 0;
-	Py_ssize_t dictSize = 0;
-	ZSTD_compressionParameters params;
-	CompressionParametersObject* result;
-
-	if (!PyArg_ParseTuple(args, "i|Kn:get_compression_parameters",
-		&compressionLevel, &sourceSize, &dictSize)) {
-		return NULL;
+int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value) {
+	size_t zresult = ZSTD_CCtxParam_setParameter(params, param, value);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "unable to set compression context parameter: %s",
+			ZSTD_getErrorName(zresult));
+		return 1;
 	}
 
-	params = ZSTD_getCParams(compressionLevel, sourceSize, dictSize);
+	return 0;
+}
+
+#define TRY_SET_PARAMETER(params, param, value) if (set_parameter(params, param, value)) return -1;
 
-	result = PyObject_New(CompressionParametersObject, &CompressionParametersType);
-	if (!result) {
-		return NULL;
+int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj) {
+	TRY_SET_PARAMETER(params, ZSTD_p_format, obj->format);
+	TRY_SET_PARAMETER(params, ZSTD_p_compressionLevel, (unsigned)obj->compressionLevel);
+	TRY_SET_PARAMETER(params, ZSTD_p_windowLog, obj->windowLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_hashLog, obj->hashLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_chainLog, obj->chainLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_searchLog, obj->searchLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_minMatch, obj->minMatch);
+	TRY_SET_PARAMETER(params, ZSTD_p_targetLength, obj->targetLength);
+	TRY_SET_PARAMETER(params, ZSTD_p_compressionStrategy, obj->compressionStrategy);
+	TRY_SET_PARAMETER(params, ZSTD_p_contentSizeFlag, obj->contentSizeFlag);
+	TRY_SET_PARAMETER(params, ZSTD_p_checksumFlag, obj->checksumFlag);
+	TRY_SET_PARAMETER(params, ZSTD_p_dictIDFlag, obj->dictIDFlag);
+	TRY_SET_PARAMETER(params, ZSTD_p_nbWorkers, obj->threads);
+	TRY_SET_PARAMETER(params, ZSTD_p_jobSize, obj->jobSize);
+	TRY_SET_PARAMETER(params, ZSTD_p_overlapSizeLog, obj->overlapSizeLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_compressLiterals, obj->compressLiterals);
+	TRY_SET_PARAMETER(params, ZSTD_p_forceMaxWindow, obj->forceMaxWindow);
+	TRY_SET_PARAMETER(params, ZSTD_p_enableLongDistanceMatching, obj->enableLongDistanceMatching);
+	TRY_SET_PARAMETER(params, ZSTD_p_ldmHashLog, obj->ldmHashLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_ldmMinMatch, obj->ldmMinMatch);
+	TRY_SET_PARAMETER(params, ZSTD_p_ldmBucketSizeLog, obj->ldmBucketSizeLog);
+	TRY_SET_PARAMETER(params, ZSTD_p_ldmHashEveryLog, obj->ldmHashEveryLog);
+
+	return 0;
+}
+
+int reset_params(ZstdCompressionParametersObject* params) {
+	if (params->params) {
+		ZSTD_CCtxParams_reset(params->params);
+	}
+	else {
+		params->params = ZSTD_createCCtxParams();
+		if (!params->params) {
+			PyErr_NoMemory();
+			return 1;
+		}
 	}
 
-	result->windowLog = params.windowLog;
-	result->chainLog = params.chainLog;
-	result->hashLog = params.hashLog;
-	result->searchLog = params.searchLog;
-	result->searchLength = params.searchLength;
-	result->targetLength = params.targetLength;
-	result->strategy = params.strategy;
-
-	return result;
+	return set_parameters(params->params, params);
 }
 
-static int CompressionParameters_init(CompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
+static int ZstdCompressionParameters_init(ZstdCompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
+		"format",
+		"compression_level",
 		"window_log",
-		"chain_log",
 		"hash_log",
+		"chain_log",
 		"search_log",
-		"search_length",
+		"min_match",
 		"target_length",
-		"strategy",
+		"compression_strategy",
+		"write_content_size",
+		"write_checksum",
+		"write_dict_id",
+		"job_size",
+		"overlap_size_log",
+		"force_max_window",
+		"enable_ldm",
+		"ldm_hash_log",
+		"ldm_min_match",
+		"ldm_bucket_size_log",
+		"ldm_hash_every_log",
+		"threads",
+		"compress_literals",
 		NULL
 	};
 
-	unsigned windowLog;
-	unsigned chainLog;
-	unsigned hashLog;
-	unsigned searchLog;
-	unsigned searchLength;
-	unsigned targetLength;
-	unsigned strategy;
-	ZSTD_compressionParameters params;
-	size_t zresult;
+	unsigned format = 0;
+	int compressionLevel = 0;
+	unsigned windowLog = 0;
+	unsigned hashLog = 0;
+	unsigned chainLog = 0;
+	unsigned searchLog = 0;
+	unsigned minMatch = 0;
+	unsigned targetLength = 0;
+	unsigned compressionStrategy = 0;
+	unsigned contentSizeFlag = 1;
+	unsigned checksumFlag = 0;
+	unsigned dictIDFlag = 0;
+	unsigned jobSize = 0;
+	unsigned overlapSizeLog = 0;
+	unsigned forceMaxWindow = 0;
+	unsigned enableLDM = 0;
+	unsigned ldmHashLog = 0;
+	unsigned ldmMinMatch = 0;
+	unsigned ldmBucketSizeLog = 0;
+	unsigned ldmHashEveryLog = 0;
+	int threads = 0;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "IIIIIII:CompressionParameters",
-		kwlist, &windowLog, &chainLog, &hashLog, &searchLog, &searchLength,
-		&targetLength, &strategy)) {
-		return -1;
-	}
+	/* Setting value 0 has the effect of disabling. So we use -1 as a default
+	 * to detect whether to set. Then we automatically derive the expected value
+	 * based on the level, just like zstandard does itself. */
+	int compressLiterals = -1;
 
-	if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid window log value");
-		return -1;
-	}
-
-	if (chainLog < ZSTD_CHAINLOG_MIN || chainLog > ZSTD_CHAINLOG_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid chain log value");
-		return -1;
-	}
-
-	if (hashLog < ZSTD_HASHLOG_MIN || hashLog > ZSTD_HASHLOG_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid hash log value");
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs,
+		"|IiIIIIIIIIIIIIIIIIIIii:CompressionParameters",
+		kwlist, &format, &compressionLevel, &windowLog, &hashLog, &chainLog,
+		&searchLog, &minMatch, &targetLength, &compressionStrategy,
+		&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapSizeLog,
+		&forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch, &ldmBucketSizeLog,
+		&ldmHashEveryLog, &threads, &compressLiterals)) {
 		return -1;
 	}
 
-	if (searchLog < ZSTD_SEARCHLOG_MIN || searchLog > ZSTD_SEARCHLOG_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid search log value");
-		return -1;
+	if (threads < 0) {
+		threads = cpu_count();
 	}
 
-	if (searchLength < ZSTD_SEARCHLENGTH_MIN || searchLength > ZSTD_SEARCHLENGTH_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid search length value");
-		return -1;
-	}
-
-	if (targetLength < ZSTD_TARGETLENGTH_MIN || targetLength > ZSTD_TARGETLENGTH_MAX) {
-		PyErr_SetString(PyExc_ValueError, "invalid target length value");
-		return -1;
+	if (compressLiterals < 0) {
+		compressLiterals = compressionLevel >= 0;
 	}
 
-	if (strategy < ZSTD_fast || strategy > ZSTD_btopt) {
-		PyErr_SetString(PyExc_ValueError, "invalid strategy value");
-		return -1;
-	}
-
+	self->format = format;
+	self->compressionLevel = compressionLevel;
 	self->windowLog = windowLog;
+	self->hashLog = hashLog;
 	self->chainLog = chainLog;
-	self->hashLog = hashLog;
 	self->searchLog = searchLog;
-	self->searchLength = searchLength;
+	self->minMatch = minMatch;
 	self->targetLength = targetLength;
-	self->strategy = strategy;
+	self->compressionStrategy = compressionStrategy;
+	self->contentSizeFlag = contentSizeFlag;
+	self->checksumFlag = checksumFlag;
+	self->dictIDFlag = dictIDFlag;
+	self->threads = threads;
+	self->jobSize = jobSize;
+	self->overlapSizeLog = overlapSizeLog;
+	self->compressLiterals = compressLiterals;
+	self->forceMaxWindow = forceMaxWindow;
+	self->enableLongDistanceMatching = enableLDM;
+	self->ldmHashLog = ldmHashLog;
+	self->ldmMinMatch = ldmMinMatch;
+	self->ldmBucketSizeLog = ldmBucketSizeLog;
+	self->ldmHashEveryLog = ldmHashEveryLog;
 
-	ztopy_compression_parameters(self, &params);
-	zresult = ZSTD_checkCParams(params);
-
-	if (ZSTD_isError(zresult)) {
-		PyErr_Format(PyExc_ValueError, "invalid compression parameters: %s",
-			ZSTD_getErrorName(zresult));
+	if (reset_params(self)) {
 		return -1;
 	}
 
 	return 0;
 }
 
-PyDoc_STRVAR(CompressionParameters_estimated_compression_context_size__doc__,
+PyDoc_STRVAR(ZstdCompressionParameters_from_level__doc__,
+"Create a CompressionParameters from a compression level and target sizes\n"
+);
+
+ZstdCompressionParametersObject* CompressionParameters_from_level(PyObject* undef, PyObject* args, PyObject* kwargs) {
+	int managedKwargs = 0;
+	int level;
+	PyObject* sourceSize = NULL;
+	PyObject* dictSize = NULL;
+	unsigned PY_LONG_LONG iSourceSize = 0;
+	Py_ssize_t iDictSize = 0;
+	PyObject* val;
+	ZSTD_compressionParameters params;
+	ZstdCompressionParametersObject* result = NULL;
+	int res;
+
+	if (!PyArg_ParseTuple(args, "i:from_level",
+		&level)) {
+		return NULL;
+	}
+
+	if (!kwargs) {
+		kwargs = PyDict_New();
+		if (!kwargs) {
+			return NULL;
+		}
+		managedKwargs = 1;
+	}
+
+	sourceSize = PyDict_GetItemString(kwargs, "source_size");
+	if (sourceSize) {
+#if PY_MAJOR_VERSION >= 3
+		iSourceSize = PyLong_AsUnsignedLongLong(sourceSize);
+		if (iSourceSize == (unsigned PY_LONG_LONG)(-1)) {
+			goto cleanup;
+		}
+#else
+		iSourceSize = PyInt_AsUnsignedLongLongMask(sourceSize);
+#endif
+
+		PyDict_DelItemString(kwargs, "source_size");
+	}
+
+	dictSize = PyDict_GetItemString(kwargs, "dict_size");
+	if (dictSize) {
+#if PY_MAJOR_VERSION >= 3
+		iDictSize = PyLong_AsSsize_t(dictSize);
+#else
+		iDictSize = PyInt_AsSsize_t(dictSize);
+#endif
+		if (iDictSize == -1) {
+			goto cleanup;
+		}
+
+		PyDict_DelItemString(kwargs, "dict_size");
+	}
+
+
+	params = ZSTD_getCParams(level, iSourceSize, iDictSize);
+
+	/* Values derived from the input level and sizes are passed along to the
+	   constructor. But only if a value doesn't already exist. */
+	val = PyDict_GetItemString(kwargs, "window_log");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.windowLog);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "window_log", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "chain_log");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.chainLog);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "chain_log", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "hash_log");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.hashLog);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "hash_log", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "search_log");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.searchLog);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "search_log", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "min_match");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.searchLength);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "min_match", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "target_length");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.targetLength);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "target_length", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "compression_strategy");
+	if (!val) {
+		val = PyLong_FromUnsignedLong(params.strategy);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "compression_strategy", val);
+		Py_DECREF(val);
+	}
+
+	val = PyDict_GetItemString(kwargs, "compress_literals");
+	if (!val) {
+		val = PyLong_FromLong(level >= 0 ? 1 : 0);
+		if (!val) {
+			goto cleanup;
+		}
+		PyDict_SetItemString(kwargs, "compress_literals", val);
+		Py_DECREF(val);
+	}
+
+	result = PyObject_New(ZstdCompressionParametersObject, &ZstdCompressionParametersType);
+	if (!result) {
+		goto cleanup;
+	}
+
+	result->params = NULL;
+
+	val = PyTuple_New(0);
+	if (!val) {
+		Py_CLEAR(result);
+		goto cleanup;
+	}
+
+	res = ZstdCompressionParameters_init(result, val, kwargs);
+	Py_DECREF(val);
+
+	if (res) {
+		Py_CLEAR(result);
+		goto cleanup;
+	}
+
+cleanup:
+	if (managedKwargs) {
+		Py_DECREF(kwargs);
+	}
+
+	return result;
+}
+
+PyDoc_STRVAR(ZstdCompressionParameters_estimated_compression_context_size__doc__,
 "Estimate the size in bytes of a compression context for compression parameters\n"
 );
 
-PyObject* CompressionParameters_estimated_compression_context_size(CompressionParametersObject* self) {
-	ZSTD_compressionParameters params;
-
-	ztopy_compression_parameters(self, &params);
-
-	return PyLong_FromSize_t(ZSTD_estimateCCtxSize(params));
+PyObject* ZstdCompressionParameters_estimated_compression_context_size(ZstdCompressionParametersObject* self) {
+	return PyLong_FromSize_t(ZSTD_estimateCCtxSize_usingCCtxParams(self->params));
 }
 
-PyObject* estimate_compression_context_size(PyObject* self, PyObject* args) {
-	CompressionParametersObject* params;
-	ZSTD_compressionParameters zparams;
-	PyObject* result;
+PyDoc_STRVAR(ZstdCompressionParameters__doc__,
+"ZstdCompressionParameters: low-level control over zstd compression");
 
-	if (!PyArg_ParseTuple(args, "O!:estimate_compression_context_size",
-		&CompressionParametersType, &params)) {
-		return NULL;
+static void ZstdCompressionParameters_dealloc(ZstdCompressionParametersObject* self) {
+	if (self->params) {
+		ZSTD_freeCCtxParams(self->params);
+		self->params = NULL;
 	}
 
-	ztopy_compression_parameters(params, &zparams);
-	result = PyLong_FromSize_t(ZSTD_estimateCCtxSize(zparams));
-	return result;
-}
-
-PyDoc_STRVAR(CompressionParameters__doc__,
-"CompressionParameters: low-level control over zstd compression");
-
-static void CompressionParameters_dealloc(PyObject* self) {
 	PyObject_Del(self);
 }
 
-static PyMethodDef CompressionParameters_methods[] = {
+static PyMethodDef ZstdCompressionParameters_methods[] = {
+	{
+		"from_level",
+		(PyCFunction)CompressionParameters_from_level,
+		METH_VARARGS | METH_KEYWORDS | METH_STATIC,
+		ZstdCompressionParameters_from_level__doc__
+	},
 	{
 		"estimated_compression_context_size",
-		(PyCFunction)CompressionParameters_estimated_compression_context_size,
+		(PyCFunction)ZstdCompressionParameters_estimated_compression_context_size,
 		METH_NOARGS,
-		CompressionParameters_estimated_compression_context_size__doc__
+		ZstdCompressionParameters_estimated_compression_context_size__doc__
 	},
 	{ NULL, NULL }
 };
 
-static PyMemberDef CompressionParameters_members[] = {
+static PyMemberDef ZstdCompressionParameters_members[] = {
+	{ "format", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, format), READONLY,
+	  "compression format" },
+	{ "compression_level", T_INT,
+	  offsetof(ZstdCompressionParametersObject, compressionLevel), READONLY,
+	  "compression level" },
 	{ "window_log", T_UINT,
-	  offsetof(CompressionParametersObject, windowLog), READONLY,
+	  offsetof(ZstdCompressionParametersObject, windowLog), READONLY,
 	  "window log" },
-	{ "chain_log", T_UINT,
-	  offsetof(CompressionParametersObject, chainLog), READONLY,
-	  "chain log" },
 	{ "hash_log", T_UINT,
-	  offsetof(CompressionParametersObject, hashLog), READONLY,
+	  offsetof(ZstdCompressionParametersObject, hashLog), READONLY,
 	  "hash log" },
+	{ "chain_log", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, chainLog), READONLY,
+	  "chain log" },
 	{ "search_log", T_UINT,
-	  offsetof(CompressionParametersObject, searchLog), READONLY,
+	  offsetof(ZstdCompressionParametersObject, searchLog), READONLY,
 	  "search log" },
-	{ "search_length", T_UINT,
-	  offsetof(CompressionParametersObject, searchLength), READONLY,
+	{ "min_match", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, minMatch), READONLY,
 	  "search length" },
 	{ "target_length", T_UINT,
-	  offsetof(CompressionParametersObject, targetLength), READONLY,
+	  offsetof(ZstdCompressionParametersObject, targetLength), READONLY,
 	  "target length" },
-	{ "strategy", T_INT,
-	  offsetof(CompressionParametersObject, strategy), READONLY,
-	  "strategy" },
+	{ "compression_strategy", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, compressionStrategy), READONLY,
+	  "compression strategy" },
+	{ "write_content_size", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, contentSizeFlag), READONLY,
+	  "whether to write content size in frames" },
+	{ "write_checksum", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, checksumFlag), READONLY,
+	  "whether to write checksum in frames" },
+	{ "write_dict_id", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, dictIDFlag), READONLY,
+	  "whether to write dictionary ID in frames" },
+	{ "threads", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, threads), READONLY,
+	  "number of threads to use" },
+	{ "job_size", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, jobSize), READONLY,
+	  "size of compression job when using multiple threads" },
+	{ "overlap_size_log", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, overlapSizeLog), READONLY,
+	  "Size of previous input reloaded at the beginning of each job" },
+	{ "compress_literals", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, compressLiterals), READONLY,
+	  "whether Huffman compression of literals is in use" },
+	{ "force_max_window", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, forceMaxWindow), READONLY,
+	  "force back references to remain smaller than window size" },
+	{ "enable_ldm", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, enableLongDistanceMatching), READONLY,
+	  "whether to enable long distance matching" },
+	{ "ldm_hash_log", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, ldmHashLog), READONLY,
+	  "Size of the table for long distance matching, as a power of 2" },
+	{ "ldm_min_match", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, ldmMinMatch), READONLY,
+	  "minimum size of searched matches for long distance matcher" },
+	{ "ldm_bucket_size_log", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, ldmBucketSizeLog), READONLY,
+	  "log size of each bucket in the LDM hash table for collision resolution" },
+	{ "ldm_hash_every_log", T_UINT,
+	  offsetof(ZstdCompressionParametersObject, ldmHashEveryLog), READONLY,
+	  "frequency of inserting/looking up entries in the LDM hash table" },
 	{ NULL }
 };
 
-PyTypeObject CompressionParametersType = {
+PyTypeObject ZstdCompressionParametersType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
-	"CompressionParameters", /* tp_name */
-	sizeof(CompressionParametersObject), /* tp_basicsize */
+	"ZstdCompressionParameters", /* tp_name */
+	sizeof(ZstdCompressionParametersObject), /* tp_basicsize */
 	0,                         /* tp_itemsize */
-	(destructor)CompressionParameters_dealloc, /* tp_dealloc */
+	(destructor)ZstdCompressionParameters_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
@@ -221,33 +465,38 b' PyTypeObject CompressionParametersType ='
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
-	CompressionParameters__doc__, /* tp_doc */
+	ZstdCompressionParameters__doc__, /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
-	CompressionParameters_methods, /* tp_methods */
-	CompressionParameters_members, /* tp_members */
+	ZstdCompressionParameters_methods, /* tp_methods */
+	ZstdCompressionParameters_members, /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
-	(initproc)CompressionParameters_init, /* tp_init */
+	(initproc)ZstdCompressionParameters_init, /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 void compressionparams_module_init(PyObject* mod) {
-	Py_TYPE(&CompressionParametersType) = &PyType_Type;
-	if (PyType_Ready(&CompressionParametersType) < 0) {
+	Py_TYPE(&ZstdCompressionParametersType) = &PyType_Type;
+	if (PyType_Ready(&ZstdCompressionParametersType) < 0) {
 		return;
 	}
 
-	Py_INCREF(&CompressionParametersType);
+	Py_INCREF(&ZstdCompressionParametersType);
+	PyModule_AddObject(mod, "ZstdCompressionParameters",
+		(PyObject*)&ZstdCompressionParametersType);
+
+	/* TODO remove deprecated alias. */
+	Py_INCREF(&ZstdCompressionParametersType);
 	PyModule_AddObject(mod, "CompressionParameters",
-		(PyObject*)&CompressionParametersType);
+		(PyObject*)&ZstdCompressionParametersType);
 }

contrib/python-zstandard/c-ext/compressionwriter.c

@@ -22,20 +22,18 b' static void ZstdCompressionWriter_deallo'
 }
 
 static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) {
+	size_t zresult;
+
 	if (self->entered) {
 		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
-	if (self->compressor->mtcctx) {
-		if (init_mtcstream(self->compressor, self->sourceSize)) {
-			return NULL;
-		}
-	}
-	else {
-		if (0 != init_cstream(self->compressor, self->sourceSize)) {
-			return NULL;
-		}
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->compressor->cctx, self->sourceSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
 	}
 
 	self->entered = 1;
@@ -59,8 +57,12 b' static PyObject* ZstdCompressionWriter_e'
 
 	self->entered = 0;
 
-	if ((self->compressor->cstream || self->compressor->mtcctx) && exc_type == Py_None
-		&& exc_value == Py_None && exc_tb == Py_None) {
+	if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) {
+		ZSTD_inBuffer inBuffer;
+
+		inBuffer.src = NULL;
+		inBuffer.size = 0;
+		inBuffer.pos = 0;
 
 		output.dst = PyMem_Malloc(self->outSize);
 		if (!output.dst) {
@@ -70,12 +72,7 b' static PyObject* ZstdCompressionWriter_e'
 		output.pos = 0;
 
 		while (1) {
-			if (self->compressor->mtcctx) {
-				zresult = ZSTDMT_endStream(self->compressor->mtcctx, &output);
-			}
-			else {
-				zresult = ZSTD_endStream(self->compressor->cstream, &output);
-			}
+			zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &inBuffer, ZSTD_e_end);
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "error ending compression stream: %s",
 					ZSTD_getErrorName(zresult));
@@ -107,18 +104,17 b' static PyObject* ZstdCompressionWriter_e'
 }
 
 static PyObject* ZstdCompressionWriter_memory_size(ZstdCompressionWriter* self) {
-	if (!self->compressor->cstream) {
-		PyErr_SetString(ZstdError, "cannot determine size of an inactive compressor; "
-			"call when a context manager is active");
-		return NULL;
-	}
-
-	return PyLong_FromSize_t(ZSTD_sizeof_CStream(self->compressor->cstream));
+	return PyLong_FromSize_t(ZSTD_sizeof_CCtx(self->compressor->cctx));
 }
 
-static PyObject* ZstdCompressionWriter_write(ZstdCompressionWriter* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
+static PyObject* ZstdCompressionWriter_write(ZstdCompressionWriter* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		NULL
+	};
+
+	PyObject* result = NULL;
+	Py_buffer source;
 	size_t zresult;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
@@ -126,44 +122,46 b' static PyObject* ZstdCompressionWriter_w'
 	Py_ssize_t totalWrite = 0;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:write", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write",
 #else
-	if (!PyArg_ParseTuple(args, "s#:write", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:write",
 #endif
+		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!self->entered) {
 		PyErr_SetString(ZstdError, "compress must be called from an active context manager");
-		return NULL;
+		goto finally;
+	}
+
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
 	}
 
 	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
-		return PyErr_NoMemory();
+		PyErr_NoMemory();
+		goto finally;
 	}
 	output.size = self->outSize;
 	output.pos = 0;
 
-	input.src = source;
-	input.size = sourceSize;
+	input.src = source.buf;
+	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < sourceSize) {
+	while ((ssize_t)input.pos < source.len) {
 		Py_BEGIN_ALLOW_THREADS
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_compressStream(self->compressor->mtcctx,
-				&output, &input);
-		}
-		else {
-			zresult = ZSTD_compressStream(self->compressor->cstream, &output, &input);
-		}
+		zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyMem_Free(output.dst);
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
-			return NULL;
+			goto finally;
 		}
 
 		/* Copy data from output buffer to writer. */
@@ -176,18 +174,24 b' static PyObject* ZstdCompressionWriter_w'
 				output.dst, output.pos);
 			Py_XDECREF(res);
 			totalWrite += output.pos;
+			self->bytesCompressed += output.pos;
 		}
 		output.pos = 0;
 	}
 
 	PyMem_Free(output.dst);
 
-	return PyLong_FromSsize_t(totalWrite);
+	result = PyLong_FromSsize_t(totalWrite);
+
+finally:
+	PyBuffer_Release(&source);
+	return result;
 }
 
 static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args) {
 	size_t zresult;
 	ZSTD_outBuffer output;
+	ZSTD_inBuffer input;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
 
@@ -196,6 +200,10 b' static PyObject* ZstdCompressionWriter_f'
 		return NULL;
 	}
 
+	input.src = NULL;
+	input.size = 0;
+	input.pos = 0;
+
 	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
 		return PyErr_NoMemory();
@@ -205,12 +213,7 b' static PyObject* ZstdCompressionWriter_f'
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_flushStream(self->compressor->mtcctx, &output);
-		}
-		else {
-			zresult = ZSTD_flushStream(self->compressor->cstream, &output);
-		}
+		zresult = ZSTD_compress_generic(self->compressor->cctx, &output, &input, ZSTD_e_flush);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
@@ -233,6 +236,7 b' static PyObject* ZstdCompressionWriter_f'
 				output.dst, output.pos);
 			Py_XDECREF(res);
 			totalWrite += output.pos;
+			self->bytesCompressed += output.pos;
 		}
 		output.pos = 0;
 	}
@@ -242,6 +246,10 b' static PyObject* ZstdCompressionWriter_f'
 	return PyLong_FromSsize_t(totalWrite);
 }
 
+static PyObject* ZstdCompressionWriter_tell(ZstdCompressionWriter* self) {
+	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
+}
+
 static PyMethodDef ZstdCompressionWriter_methods[] = {
 	{ "__enter__", (PyCFunction)ZstdCompressionWriter_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context.") },
@@ -249,10 +257,12 b' static PyMethodDef ZstdCompressionWriter'
 	PyDoc_STR("Exit a compression context.") },
 	{ "memory_size", (PyCFunction)ZstdCompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size of the underlying compressor") },
-	{ "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS,
+	{ "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
 	{ "flush", (PyCFunction)ZstdCompressionWriter_flush, METH_NOARGS,
 	PyDoc_STR("Flush data and finish a zstd frame") },
+	{ "tell", (PyCFunction)ZstdCompressionWriter_tell, METH_NOARGS,
+	PyDoc_STR("Returns current number of bytes compressed") },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/compressobj.c

@@ -23,9 +23,13 b' static void ZstdCompressionObj_dealloc(Z'
 	PyObject_Del(self);
 }
 
-static PyObject* ZstdCompressionObj_compress(ZstdCompressionObj* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
+static PyObject* ZstdCompressionObj_compress(ZstdCompressionObj* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		NULL
+	};
+
+	Py_buffer source;
 	ZSTD_inBuffer input;
 	size_t zresult;
 	PyObject* result = NULL;
@@ -37,38 +41,43 b' static PyObject* ZstdCompressionObj_comp'
 	}
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:compress", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:compress",
 #else
-	if (!PyArg_ParseTuple(args, "s#:compress", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:compress",
 #endif
+		kwlist, &source)) {
 		return NULL;
 	}
 
-	input.src = source;
-	input.size = sourceSize;
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	input.src = source.buf;
+	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < sourceSize) {
+	while ((ssize_t)input.pos < source.len) {
 		Py_BEGIN_ALLOW_THREADS
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_compressStream(self->compressor->mtcctx,
-				&self->output, &input);
-		}
-		else {
-			zresult = ZSTD_compressStream(self->compressor->cstream, &self->output, &input);
-		}
+			zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+				&input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
-			return NULL;
+			Py_CLEAR(result);
+			goto finally;
 		}
 
 		if (self->output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
-				if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) {
-					return NULL;
+
+				if (safe_pybytes_resize(&result, resultSize + self->output.pos)) {
+					Py_CLEAR(result);
+					goto finally;
 				}
 
 				memcpy(PyBytes_AS_STRING(result) + resultSize,
@@ -77,7 +86,7 b' static PyObject* ZstdCompressionObj_comp'
 			else {
 				result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 				if (!result) {
-					return NULL;
+					goto finally;
 				}
 			}
 
@@ -85,21 +94,29 b' static PyObject* ZstdCompressionObj_comp'
 		}
 	}
 
-	if (result) {
-		return result;
+	if (NULL == result) {
+		result = PyBytes_FromString("");
 	}
-	else {
-		return PyBytes_FromString("");
-	}
+
+finally:
+	PyBuffer_Release(&source);
+
+	return result;
 }
 
-static PyObject* ZstdCompressionObj_flush(ZstdCompressionObj* self, PyObject* args) {
+static PyObject* ZstdCompressionObj_flush(ZstdCompressionObj* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"flush_mode",
+		NULL
+	};
+
 	int flushMode = compressorobj_flush_finish;
 	size_t zresult;
 	PyObject* result = NULL;
 	Py_ssize_t resultSize = 0;
+	ZSTD_inBuffer input;
 
-	if (!PyArg_ParseTuple(args, "|i:flush", &flushMode)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:flush", kwlist, &flushMode)) {
 		return NULL;
 	}
 
@@ -115,16 +132,16 b' static PyObject* ZstdCompressionObj_flus'
 
 	assert(self->output.pos == 0);
 
+	input.src = NULL;
+	input.size = 0;
+	input.pos = 0;
+
 	if (flushMode == compressorobj_flush_block) {
 		/* The output buffer is of size ZSTD_CStreamOutSize(), which is 
 		   guaranteed to hold a full block. */
 		Py_BEGIN_ALLOW_THREADS
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_flushStream(self->compressor->mtcctx, &self->output);
-		}
-		else {
-			zresult = ZSTD_flushStream(self->compressor->cstream, &self->output);
-		}
+			zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+				&input, ZSTD_e_flush);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
@@ -156,12 +173,8 b' static PyObject* ZstdCompressionObj_flus'
 	self->finished = 1;
 
 	while (1) {
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_endStream(self->compressor->mtcctx, &self->output);
-		}
-		else {
-			zresult = ZSTD_endStream(self->compressor->cstream, &self->output);
-		}
+		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+			&input, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
@@ -171,7 +184,9 b' static PyObject* ZstdCompressionObj_flus'
 		if (self->output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
-				if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) {
+
+				if (safe_pybytes_resize(&result, resultSize + self->output.pos)) {
+					Py_XDECREF(result);
 					return NULL;
 				}
 
@@ -202,9 +217,9 b' static PyObject* ZstdCompressionObj_flus'
 }
 
 static PyMethodDef ZstdCompressionObj_methods[] = {
-	{ "compress", (PyCFunction)ZstdCompressionObj_compress, METH_VARARGS,
+	{ "compress", (PyCFunction)ZstdCompressionObj_compress, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("compress data") },
-	{ "flush", (PyCFunction)ZstdCompressionObj_flush, METH_VARARGS,
+	{ "flush", (PyCFunction)ZstdCompressionObj_flush, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("finish compression operation") },
 	{ NULL, NULL }
 };

contrib/python-zstandard/c-ext/compressor.c

@@ -11,118 +11,78 b''
 
 extern PyObject* ZstdError;
 
-int populate_cdict(ZstdCompressor* compressor, ZSTD_parameters* zparams) {
-	ZSTD_customMem zmem;
+int ensure_cctx(ZstdCompressor* compressor) {
+	size_t zresult;
+
+	assert(compressor);
+	assert(compressor->cctx);
+	assert(compressor->params);
 
-	if (compressor->cdict || !compressor->dict || !compressor->dict->dictData) {
-		return 0;
+	ZSTD_CCtx_reset(compressor->cctx);
+
+	zresult = ZSTD_CCtx_setParametersUsingCCtxParams(compressor->cctx, compressor->params);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "could not set compression parameters: %s",
+			ZSTD_getErrorName(zresult));
+		return 1;
 	}
 
-	Py_BEGIN_ALLOW_THREADS
-	memset(&zmem, 0, sizeof(zmem));
-	compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData,
-		compressor->dict->dictSize, 1, *zparams, zmem);
-	Py_END_ALLOW_THREADS
-
-	if (!compressor->cdict) {
-		PyErr_SetString(ZstdError, "could not create compression dictionary");
-		return 1;
+	if (compressor->dict) {
+		if (compressor->dict->cdict) {
+			zresult = ZSTD_CCtx_refCDict(compressor->cctx, compressor->dict->cdict);
+		}
+		else {
+			zresult = ZSTD_CCtx_loadDictionary_advanced(compressor->cctx,
+				compressor->dict->dictData, compressor->dict->dictSize,
+				ZSTD_dlm_byRef, compressor->dict->dictType);
+		}
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "could not load compression dictionary: %s",
+				ZSTD_getErrorName(zresult));
+			return 1;
+		}
 	}
 
 	return 0;
 }
 
-/**
- * Ensure the ZSTD_CStream on a ZstdCompressor instance is initialized.
- *
- * Returns 0 on success. Other value on failure. Will set a Python exception
- * on failure.
- */
-int init_cstream(ZstdCompressor* compressor, unsigned long long sourceSize) {
-	ZSTD_parameters zparams;
-	void* dictData = NULL;
-	size_t dictSize = 0;
-	size_t zresult;
+static PyObject* frame_progression(ZSTD_CCtx* cctx) {
+	PyObject* result = NULL;
+	PyObject* value;
+	ZSTD_frameProgression progression;
 
-	if (compressor->cstream) {
-		zresult = ZSTD_resetCStream(compressor->cstream, sourceSize);
-		if (ZSTD_isError(zresult)) {
-			PyErr_Format(ZstdError, "could not reset CStream: %s",
-				ZSTD_getErrorName(zresult));
-			return -1;
-		}
-
-		return 0;
+	result = PyTuple_New(3);
+	if (!result) {
+		return NULL;
 	}
 
-	compressor->cstream = ZSTD_createCStream();
-	if (!compressor->cstream) {
-		PyErr_SetString(ZstdError, "could not create CStream");
-		return -1;
-	}
+	progression = ZSTD_getFrameProgression(cctx);
 
-	if (compressor->dict) {
-		dictData = compressor->dict->dictData;
-		dictSize = compressor->dict->dictSize;
-	}
-
-	memset(&zparams, 0, sizeof(zparams));
-	if (compressor->cparams) {
-		ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
-		/* Do NOT call ZSTD_adjustCParams() here because the compression params
-		come from the user. */
-	}
-	else {
-		zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize);
+	value = PyLong_FromUnsignedLongLong(progression.ingested);
+	if (!value) {
+		Py_DECREF(result);
+		return NULL;
 	}
 
-	zparams.fParams = compressor->fparams;
-
-	zresult = ZSTD_initCStream_advanced(compressor->cstream, dictData, dictSize,
-		zparams, sourceSize);
+	PyTuple_SET_ITEM(result, 0, value);
 
-	if (ZSTD_isError(zresult)) {
-		ZSTD_freeCStream(compressor->cstream);
-		compressor->cstream = NULL;
-		PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult));
-		return -1;
+	value = PyLong_FromUnsignedLongLong(progression.consumed);
+	if (!value) {
+		Py_DECREF(result);
+		return NULL;
 	}
 
-	return 0;;
-}
+	PyTuple_SET_ITEM(result, 1, value);
 
-int init_mtcstream(ZstdCompressor* compressor, Py_ssize_t sourceSize) {
-	size_t zresult;
-	void* dictData = NULL;
-	size_t dictSize = 0;
-	ZSTD_parameters zparams;
-
-	assert(compressor->mtcctx);
-
-	if (compressor->dict) {
-		dictData = compressor->dict->dictData;
-		dictSize = compressor->dict->dictSize;
+	value = PyLong_FromUnsignedLongLong(progression.produced);
+	if (!value) {
+		Py_DECREF(result);
+		return NULL;
 	}
 
-	memset(&zparams, 0, sizeof(zparams));
-	if (compressor->cparams) {
-		ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
-	}
-	else {
-		zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize);
-	}
-
-	zparams.fParams = compressor->fparams;
+	PyTuple_SET_ITEM(result, 2, value);
 
-	zresult = ZSTDMT_initCStream_advanced(compressor->mtcctx, dictData, dictSize,
-		zparams, sourceSize);
-
-	if (ZSTD_isError(zresult)) {
-		PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult));
-		return -1;
-	}
-
-	return 0;
+	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor__doc__,
@@ -147,9 +107,9 b' PyDoc_STRVAR(ZstdCompressor__doc__,'
 "   If True, a 4 byte content checksum will be written with the compressed\n"
 "   data, allowing the decompressor to perform content verification.\n"
 "write_content_size\n"
-"   If True, the decompressed content size will be included in the header of\n"
-"   the compressed data. This data will only be written if the compressor\n"
-"   knows the size of the input data.\n"
+"   If True (the default), the decompressed content size will be included in\n"
+"   the header of the compressed data. This data will only be written if the\n"
+"   compressor knows the size of the input data.\n"
 "write_dict_id\n"
 "   Determines whether the dictionary ID will be written into the compressed\n"
 "   data. Defaults to True. Only adds content to the compressed data if\n"
@@ -175,7 +135,7 b' static int ZstdCompressor_init(ZstdCompr'
 
 	int level = 3;
 	ZstdCompressionDict* dict = NULL;
-	CompressionParametersObject* params = NULL;
+	ZstdCompressionParametersObject* params = NULL;
 	PyObject* writeChecksum = NULL;
 	PyObject* writeContentSize = NULL;
 	PyObject* writeDictID = NULL;
@@ -183,16 +143,11 b' static int ZstdCompressor_init(ZstdCompr'
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOOi:ZstdCompressor",
 		kwlist,	&level, &ZstdCompressionDictType, &dict,
-		&CompressionParametersType, &params,
+		&ZstdCompressionParametersType, &params,
 		&writeChecksum, &writeContentSize, &writeDictID, &threads)) {
 		return -1;
 	}
 
-	if (level < 1) {
-		PyErr_SetString(PyExc_ValueError, "level must be greater than 0");
-		return -1;
-	}
-
 	if (level > ZSTD_maxCLevel()) {
 		PyErr_Format(PyExc_ValueError, "level must be less than %d",
 			ZSTD_maxCLevel() + 1);
@@ -203,79 +158,135 b' static int ZstdCompressor_init(ZstdCompr'
 		threads = cpu_count();
 	}
 
-	self->threads = threads;
-
 	/* We create a ZSTD_CCtx for reuse among multiple operations to reduce the
 	   overhead of each compression operation. */
-	if (threads) {
-		self->mtcctx = ZSTDMT_createCCtx(threads);
-		if (!self->mtcctx) {
-			PyErr_NoMemory();
+	self->cctx = ZSTD_createCCtx();
+	if (!self->cctx) {
+		PyErr_NoMemory();
+		return -1;
+	}
+
+	/* TODO stuff the original parameters away somewhere so we can reset later. This
+	   will allow us to do things like automatically adjust cparams based on input
+	   size (assuming zstd isn't doing that internally). */
+
+	self->params = ZSTD_createCCtxParams();
+	if (!self->params) {
+		PyErr_NoMemory();
+		return -1;
+	}
+
+	if (params && writeChecksum) {
+		PyErr_SetString(PyExc_ValueError,
+			"cannot define compression_params and write_checksum");
+		return -1;
+	}
+
+	if (params && writeContentSize) {
+		PyErr_SetString(PyExc_ValueError,
+			"cannot define compression_params and write_content_size");
+		return -1;
+	}
+
+	if (params && writeDictID) {
+		PyErr_SetString(PyExc_ValueError,
+			"cannot define compression_params and write_dict_id");
+		return -1;
+	}
+
+	if (params && threads) {
+		PyErr_SetString(PyExc_ValueError,
+			"cannot define compression_params and threads");
+		return -1;
+	}
+
+	if (params) {
+		if (set_parameters(self->params, params)) {
 			return -1;
 		}
 	}
 	else {
-		self->cctx = ZSTD_createCCtx();
-		if (!self->cctx) {
-			PyErr_NoMemory();
+		if (set_parameter(self->params, ZSTD_p_compressionLevel, level)) {
+			return -1;
+		}
+
+		if (set_parameter(self->params, ZSTD_p_contentSizeFlag,
+			writeContentSize ? PyObject_IsTrue(writeContentSize) : 1)) {
+			return -1;
+		}
+
+		if (set_parameter(self->params, ZSTD_p_checksumFlag,
+			writeChecksum ? PyObject_IsTrue(writeChecksum) : 0)) {
 			return -1;
 		}
-	}
+
+		if (set_parameter(self->params, ZSTD_p_dictIDFlag,
+			writeDictID ? PyObject_IsTrue(writeDictID) : 1)) {
+			return -1;
+		}
 
-	self->compressionLevel = level;
+		if (threads) {
+			if (set_parameter(self->params, ZSTD_p_nbWorkers, threads)) {
+				return -1;
+			}
+		}
+	}
 
 	if (dict) {
 		self->dict = dict;
 		Py_INCREF(dict);
 	}
 
-	if (params) {
-		self->cparams = params;
-		Py_INCREF(params);
-	}
-
-	memset(&self->fparams, 0, sizeof(self->fparams));
-
-	if (writeChecksum && PyObject_IsTrue(writeChecksum)) {
-		self->fparams.checksumFlag = 1;
-	}
-	if (writeContentSize && PyObject_IsTrue(writeContentSize)) {
-		self->fparams.contentSizeFlag = 1;
-	}
-	if (writeDictID && PyObject_Not(writeDictID)) {
-		self->fparams.noDictIDFlag = 1;
+	if (ensure_cctx(self)) {
+		return -1;
 	}
 
 	return 0;
 }
 
 static void ZstdCompressor_dealloc(ZstdCompressor* self) {
-	if (self->cstream) {
-		ZSTD_freeCStream(self->cstream);
-		self->cstream = NULL;
-	}
-
-	Py_XDECREF(self->cparams);
-	Py_XDECREF(self->dict);
-
-	if (self->cdict) {
-		ZSTD_freeCDict(self->cdict);
-		self->cdict = NULL;
-	}
-
 	if (self->cctx) {
 		ZSTD_freeCCtx(self->cctx);
 		self->cctx = NULL;
 	}
 
-	if (self->mtcctx) {
-		ZSTDMT_freeCCtx(self->mtcctx);
-		self->mtcctx = NULL;
+	if (self->params) {
+		ZSTD_freeCCtxParams(self->params);
+		self->params = NULL;
 	}
 
+	Py_XDECREF(self->dict);
 	PyObject_Del(self);
 }
 
+PyDoc_STRVAR(ZstdCompressor_memory_size__doc__,
+"memory_size()\n"
+"\n"
+"Obtain the memory usage of this compressor, in bytes.\n"
+);
+
+static PyObject* ZstdCompressor_memory_size(ZstdCompressor* self) {
+	if (self->cctx) {
+		return PyLong_FromSize_t(ZSTD_sizeof_CCtx(self->cctx));
+	}
+	else {
+		PyErr_SetString(ZstdError, "no compressor context found; this should never happen");
+		return NULL;
+	}
+}
+
+PyDoc_STRVAR(ZstdCompressor_frame_progression__doc__,
+"frame_progression()\n"
+"\n"
+"Return information on how much work the compressor has done.\n"
+"\n"
+"Returns a 3-tuple of (ingested, consumed, produced).\n"
+);
+
+static PyObject* ZstdCompressor_frame_progression(ZstdCompressor* self) {
+	return frame_progression(self->cctx);
+}
+
 PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__,
 "copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n"
 "compress data between streams\n"
@@ -304,7 +315,7 b' static PyObject* ZstdCompressor_copy_str'
 
 	PyObject* source;
 	PyObject* dest;
-	Py_ssize_t sourceSize = 0;
+	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t inSize = ZSTD_CStreamInSize();
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZSTD_inBuffer input;
@@ -313,14 +324,14 b' static PyObject* ZstdCompressor_copy_str'
 	Py_ssize_t totalWrite = 0;
 	char* readBuffer;
 	Py_ssize_t readSize;
-	PyObject* readResult;
+	PyObject* readResult = NULL;
 	PyObject* res = NULL;
 	size_t zresult;
 	PyObject* writeResult;
 	PyObject* totalReadPy;
 	PyObject* totalWritePy;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk:copy_stream", kwlist,
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|Kkk:copy_stream", kwlist,
 		&source, &dest, &sourceSize, &inSize, &outSize)) {
 		return NULL;
 	}
@@ -335,22 +346,18 b' static PyObject* ZstdCompressor_copy_str'
 		return NULL;
 	}
 
-	/* Prevent free on uninitialized memory in finally. */
-	output.dst = NULL;
-
-	if (self->mtcctx) {
-		if (init_mtcstream(self, sourceSize)) {
-			res = NULL;
-			goto finally;
-		}
-	}
-	else {
-		if (0 != init_cstream(self, sourceSize)) {
-			res = NULL;
-			goto finally;
-		}
+	if (ensure_cctx(self)) {
+		return NULL;
 	}
 
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
+	}
+
+	/* Prevent free on uninitialized memory in finally. */
 	output.dst = PyMem_Malloc(outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
@@ -360,6 +367,10 b' static PyObject* ZstdCompressor_copy_str'
 	output.size = outSize;
 	output.pos = 0;
 
+	input.src = NULL;
+	input.size = 0;
+	input.pos = 0;
+
 	while (1) {
 		/* Try to read from source stream. */
 		readResult = PyObject_CallMethod(source, "read", "n", inSize);
@@ -384,12 +395,7 b' static PyObject* ZstdCompressor_copy_str'
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
-			if (self->mtcctx) {
-				zresult = ZSTDMT_compressStream(self->mtcctx, &output, &input);
-			}
-			else {
-				zresult = ZSTD_compressStream(self->cstream, &output, &input);
-			}
+			zresult = ZSTD_compress_generic(self->cctx, &output, &input, ZSTD_e_continue);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
@@ -410,16 +416,18 b' static PyObject* ZstdCompressor_copy_str'
 				output.pos = 0;
 			}
 		}
+
+		Py_CLEAR(readResult);
 	}
 
 	/* We've finished reading. Now flush the compressor stream. */
+	assert(input.pos == input.size);
+
 	while (1) {
-		if (self->mtcctx) {
-			zresult = ZSTDMT_endStream(self->mtcctx, &output);
-		}
-		else {
-			zresult = ZSTD_endStream(self->cstream, &output);
-		}
+		Py_BEGIN_ALLOW_THREADS
+		zresult = ZSTD_compress_generic(self->cctx, &output, &input, ZSTD_e_end);
+		Py_END_ALLOW_THREADS
+
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
@@ -455,11 +463,81 b' finally:'
 		PyMem_Free(output.dst);
 	}
 
+	Py_XDECREF(readResult);
+
 	return res;
 }
 
+PyDoc_STRVAR(ZstdCompressor_stream_reader__doc__,
+"stream_reader(source, [size=0])\n"
+"\n"
+"Obtain an object that behaves like an I/O stream.\n"
+"\n"
+"The source object can be any object with a ``read(size)`` method\n"
+"or an object that conforms to the buffer protocol.\n"
+);
+
+static ZstdCompressionReader* ZstdCompressor_stream_reader(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"source",
+		"size",
+		"read_size",
+		NULL
+	};
+
+	PyObject* source;
+	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
+	size_t readSize = ZSTD_CStreamInSize();
+	ZstdCompressionReader* result = NULL;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kk:stream_reader", kwlist,
+		&source, &sourceSize, &readSize)) {
+		return NULL;
+	}
+
+	result = (ZstdCompressionReader*)PyObject_CallObject((PyObject*)&ZstdCompressionReaderType, NULL);
+	if (!result) {
+		return NULL;
+	}
+
+	if (PyObject_HasAttrString(source, "read")) {
+		result->reader = source;
+		Py_INCREF(source);
+		result->readSize = readSize;
+	}
+	else if (1 == PyObject_CheckBuffer(source)) {
+		if (0 != PyObject_GetBuffer(source, &result->buffer, PyBUF_CONTIG_RO)) {
+			goto except;
+		}
+
+		assert(result->buffer.len >= 0);
+
+		sourceSize = result->buffer.len;
+	}
+	else {
+		PyErr_SetString(PyExc_TypeError,
+			"must pass an object with a read() method or that conforms to the buffer protocol");
+		goto except;
+	}
+
+	if (ensure_cctx(self)) {
+		goto except;
+	}
+
+	result->compressor = self;
+	Py_INCREF(self);
+	result->sourceSize = sourceSize;
+
+	return result;
+
+except:
+	Py_CLEAR(result);
+
+	return NULL;
+}
+
 PyDoc_STRVAR(ZstdCompressor_compress__doc__,
-"compress(data, allow_empty=False)\n"
+"compress(data)\n"
 "\n"
 "Compress data in a single operation.\n"
 "\n"
@@ -473,122 +551,79 b' PyDoc_STRVAR(ZstdCompressor_compress__do'
 static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
-		"allow_empty",
 		NULL
 	};
 
-	const char* source;
-	Py_ssize_t sourceSize;
-	PyObject* allowEmpty = NULL;
+	Py_buffer source;
 	size_t destSize;
-	PyObject* output;
-	char* dest;
-	void* dictData = NULL;
-	size_t dictSize = 0;
+	PyObject* output = NULL;
 	size_t zresult;
-	ZSTD_parameters zparams;
+	ZSTD_outBuffer outBuffer;
+	ZSTD_inBuffer inBuffer;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O:compress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|O:compress",
 #else
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O:compress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|O:compress",
 #endif
-		kwlist, &source, &sourceSize, &allowEmpty)) {
-		return NULL;
-	}
-
-	if (self->threads && self->dict) {
-		PyErr_SetString(ZstdError,
-			"compress() cannot be used with both dictionaries and multi-threaded compression");
-		return NULL;
-	}
-
-	if (self->threads && self->cparams) {
-		PyErr_SetString(ZstdError,
-			"compress() cannot be used with both compression parameters and multi-threaded compression");
-		return NULL;
-	}
-
-	/* Limitation in zstd C API doesn't let decompression side distinguish
-	   between content size of 0 and unknown content size. This can make round
-	   tripping via Python difficult. Until this is fixed, require a flag
-	   to fire the footgun.
-	   https://github.com/indygreg/python-zstandard/issues/11 */
-	if (0 == sourceSize && self->fparams.contentSizeFlag
-		&& (!allowEmpty || PyObject_Not(allowEmpty))) {
-		PyErr_SetString(PyExc_ValueError, "cannot write empty inputs when writing content sizes");
-		return NULL;
-	}
-
-	destSize = ZSTD_compressBound(sourceSize);
-	output = PyBytes_FromStringAndSize(NULL, destSize);
-	if (!output) {
+		kwlist, &source)) {
 		return NULL;
 	}
 
-	dest = PyBytes_AsString(output);
-
-	if (self->dict) {
-		dictData = self->dict->dictData;
-		dictSize = self->dict->dictSize;
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
 	}
 
-	memset(&zparams, 0, sizeof(zparams));
-	if (!self->cparams) {
-		zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize);
+	if (ensure_cctx(self)) {
+		goto finally;
 	}
-	else {
-		ztopy_compression_parameters(self->cparams, &zparams.cParams);
-		/* Do NOT call ZSTD_adjustCParams() here because the compression params
-		come from the user. */
+
+	destSize = ZSTD_compressBound(source.len);
+	output = PyBytes_FromStringAndSize(NULL, destSize);
+	if (!output) {
+		goto finally;
 	}
 
-	zparams.fParams = self->fparams;
-
-	/* The raw dict data has to be processed before it can be used. Since this
-	adds overhead - especially if multiple dictionary compression operations
-	are performed on the same ZstdCompressor instance - we create a
-	ZSTD_CDict once and reuse it for all operations.
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, source.len);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		Py_CLEAR(output);
+		goto finally;
+	}
 
-	Note: the compression parameters used for the first invocation (possibly
-	derived from the source size) will be reused on all subsequent invocations.
-	https://github.com/facebook/zstd/issues/358 contains more info. We could
-	potentially add an argument somewhere to control this behavior.
-	*/
-	if (0 != populate_cdict(self, &zparams)) {
-		Py_DECREF(output);
-		return NULL;
-	}
+	inBuffer.src = source.buf;
+	inBuffer.size = source.len;
+	inBuffer.pos = 0;
+
+	outBuffer.dst = PyBytes_AsString(output);
+	outBuffer.size = destSize;
+	outBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	if (self->mtcctx) {
-		zresult = ZSTDMT_compressCCtx(self->mtcctx, dest, destSize,
-			source, sourceSize, self->compressionLevel);
-	}
-	else {
-		/* By avoiding ZSTD_compress(), we don't necessarily write out content
-		   size. This means the argument to ZstdCompressor to control frame
-		   parameters is honored. */
-		if (self->cdict) {
-			zresult = ZSTD_compress_usingCDict(self->cctx, dest, destSize,
-				source, sourceSize, self->cdict);
-		}
-		else {
-			zresult = ZSTD_compress_advanced(self->cctx, dest, destSize,
-				source, sourceSize, dictData, dictSize, zparams);
-		}
-	}
+	/* By avoiding ZSTD_compress(), we don't necessarily write out content
+		size. This means the argument to ZstdCompressor to control frame
+		parameters is honored. */
+	zresult = ZSTD_compress_generic(self->cctx, &outBuffer, &inBuffer, ZSTD_e_end);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult));
 		Py_CLEAR(output);
-		return NULL;
+		goto finally;
 	}
-	else {
-		Py_SIZE(output) = zresult;
+	else if (zresult) {
+		PyErr_SetString(ZstdError, "unexpected partial frame flush");
+		Py_CLEAR(output);
+		goto finally;
 	}
 
+	Py_SIZE(output) = outBuffer.pos;
+
+finally:
+	PyBuffer_Release(&source);
 	return output;
 }
 
@@ -608,11 +643,23 b' static ZstdCompressionObj* ZstdCompresso'
 		NULL
 	};
 
-	Py_ssize_t inSize = 0;
+	unsigned long long inSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZstdCompressionObj* result = NULL;
+	size_t zresult;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n:compressobj", kwlist, &inSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|K:compressobj", kwlist, &inSize)) {
+		return NULL;
+	}
+
+	if (ensure_cctx(self)) {
+		return NULL;
+	}
+
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, inSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
@@ -621,19 +668,6 b' static ZstdCompressionObj* ZstdCompresso'
 		return NULL;
 	}
 
-	if (self->mtcctx) {
-		if (init_mtcstream(self, inSize)) {
-			Py_DECREF(result);
-			return NULL;
-		}
-	}
-	else {
-		if (0 != init_cstream(self, inSize)) {
-			Py_DECREF(result);
-			return NULL;
-		}
-	}
-
 	result->output.dst = PyMem_Malloc(outSize);
 	if (!result->output.dst) {
 		PyErr_NoMemory();
@@ -647,9 +681,9 b' static ZstdCompressionObj* ZstdCompresso'
 	return result;
 }
 
-PyDoc_STRVAR(ZstdCompressor_read_from__doc__,
-"read_from(reader, [size=0, read_size=default, write_size=default])\n"
-"Read uncompress data from a reader and return an iterator\n"
+PyDoc_STRVAR(ZstdCompressor_read_to_iter__doc__,
+"read_to_iter(reader, [size=0, read_size=default, write_size=default])\n"
+"Read uncompressed data from a reader and return an iterator\n"
 "\n"
 "Returns an iterator of compressed data produced from reading from ``reader``.\n"
 "\n"
@@ -667,7 +701,7 b' PyDoc_STRVAR(ZstdCompressor_read_from__d'
 "not consume from the reader unless the caller consumes from the iterator.\n"
 );
 
-static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
+static ZstdCompressorIterator* ZstdCompressor_read_to_iter(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"reader",
 		"size",
@@ -677,12 +711,13 b' static ZstdCompressorIterator* ZstdCompr'
 	};
 
 	PyObject* reader;
-	Py_ssize_t sourceSize = 0;
+	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t inSize = ZSTD_CStreamInSize();
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZstdCompressorIterator* result;
+	size_t zresult;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk:read_from", kwlist,
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kkk:read_to_iter", kwlist,
 		&reader, &sourceSize, &inSize, &outSize)) {
 		return NULL;
 	}
@@ -696,18 +731,11 b' static ZstdCompressorIterator* ZstdCompr'
 		Py_INCREF(result->reader);
 	}
 	else if (1 == PyObject_CheckBuffer(reader)) {
-		result->buffer = PyMem_Malloc(sizeof(Py_buffer));
-		if (!result->buffer) {
+		if (0 != PyObject_GetBuffer(reader, &result->buffer, PyBUF_CONTIG_RO)) {
 			goto except;
 		}
 
-		memset(result->buffer, 0, sizeof(Py_buffer));
-
-		if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) {
-			goto except;
-		}
-
-		sourceSize = result->buffer->len;
+		sourceSize = result->buffer.len;
 	}
 	else {
 		PyErr_SetString(PyExc_ValueError,
@@ -715,22 +743,20 b' static ZstdCompressorIterator* ZstdCompr'
 		goto except;
 	}
 
+	if (ensure_cctx(self)) {
+		return NULL;
+	}
+
+	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "error setting source size: %s",
+			ZSTD_getErrorName(zresult));
+		return NULL;
+	}
+
 	result->compressor = self;
 	Py_INCREF(result->compressor);
 
-	result->sourceSize = sourceSize;
-
-	if (self->mtcctx) {
-		if (init_mtcstream(self, sourceSize)) {
-			goto except;
-		}
-	}
-	else {
-		if (0 != init_cstream(self, sourceSize)) {
-			goto except;
-		}
-	}
-
 	result->inSize = inSize;
 	result->outSize = outSize;
 
@@ -744,16 +770,13 b' static ZstdCompressorIterator* ZstdCompr'
 	goto finally;
 
 except:
-	Py_XDECREF(result->compressor);
-	Py_XDECREF(result->reader);
-	Py_DECREF(result);
-	result = NULL;
+	Py_CLEAR(result);
 
 finally:
 	return result;
 }
 
-PyDoc_STRVAR(ZstdCompressor_write_to___doc__,
+PyDoc_STRVAR(ZstdCompressor_stream_writer___doc__,
 "Create a context manager to write compressed data to an object.\n"
 "\n"
 "The passed object must have a ``write()`` method.\n"
@@ -771,7 +794,7 b' PyDoc_STRVAR(ZstdCompressor_write_to___d'
 "for a compressor output stream.\n"
 );
 
-static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
+static ZstdCompressionWriter* ZstdCompressor_stream_writer(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"writer",
 		"size",
@@ -781,10 +804,10 b' static ZstdCompressionWriter* ZstdCompre'
 
 	PyObject* writer;
 	ZstdCompressionWriter* result;
-	Py_ssize_t sourceSize = 0;
+	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t outSize = ZSTD_CStreamOutSize();
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk:write_to", kwlist,
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kk:stream_writer", kwlist,
 		&writer, &sourceSize, &outSize)) {
 		return NULL;
 	}
@@ -794,6 +817,10 b' static ZstdCompressionWriter* ZstdCompre'
 		return NULL;
 	}
 
+	if (ensure_cctx(self)) {
+		return NULL;
+	}
+
 	result = (ZstdCompressionWriter*)PyObject_CallObject((PyObject*)&ZstdCompressionWriterType, NULL);
 	if (!result) {
 		return NULL;
@@ -807,6 +834,7 b' static ZstdCompressionWriter* ZstdCompre'
 
 	result->sourceSize = sourceSize;
 	result->outSize = outSize;
+	result->bytesCompressed = 0;
 
 	return result;
 }
@@ -833,6 +861,7 b' typedef enum {'
 	WorkerError_none = 0,
 	WorkerError_zstd = 1,
 	WorkerError_no_memory = 2,
+	WorkerError_nospace = 3,
 } WorkerError;
 
 /**
@@ -841,10 +870,6 b' typedef enum {'
 typedef struct {
 	/* Used for compression. */
 	ZSTD_CCtx* cctx;
-	ZSTD_CDict* cdict;
-	int cLevel;
-	CompressionParametersObject* cParams;
-	ZSTD_frameParameters fParams;
 
 	/* What to compress. */
 	DataSource* sources;
@@ -868,7 +893,6 b' static void compress_worker(WorkerState*'
 	Py_ssize_t remainingItems = state->endOffset - state->startOffset + 1;
 	Py_ssize_t currentBufferStartOffset = state->startOffset;
 	size_t zresult;
-	ZSTD_parameters zparams;
 	void* newDest;
 	size_t allocationSize;
 	size_t boundSize;
@@ -879,16 +903,10 b' static void compress_worker(WorkerState*'
 	assert(!state->destBuffers);
 	assert(0 == state->destCount);
 
-	if (state->cParams) {
-		ztopy_compression_parameters(state->cParams, &zparams.cParams);
-	}
-
-	zparams.fParams = state->fParams;
-
 	/*
 	 * The total size of the compressed data is unknown until we actually
 	 * compress data. That means we can't pre-allocate the exact size we need.
-	 * 
+	 *
 	 * There is a cost to every allocation and reallocation. So, it is in our
 	 * interest to minimize the number of allocations.
 	 *
@@ -927,7 +945,8 b' static void compress_worker(WorkerState*'
 
 	destBuffer->segmentsSize = remainingItems;
 
-	allocationSize = roundpow2(state->totalSourceSize >> 4);
+	assert(state->totalSourceSize <= SIZE_MAX);
+	allocationSize = roundpow2((size_t)state->totalSourceSize >> 4);
 
 	/* If the maximum size of the output is larger than that, round up. */
 	boundSize = ZSTD_compressBound(sources[inputOffset].sourceSize);
@@ -949,6 +968,8 b' static void compress_worker(WorkerState*'
 		size_t sourceSize = sources[inputOffset].sourceSize;
 		size_t destAvailable;
 		void* dest;
+		ZSTD_outBuffer opOutBuffer;
+		ZSTD_inBuffer opInBuffer;
 
 		destAvailable = destBuffer->destSize - destOffset;
 		boundSize = ZSTD_compressBound(sourceSize);
@@ -1004,7 +1025,8 b' static void compress_worker(WorkerState*'
 			 * We could dynamically update allocation size based on work done so far.
 			 * For now, keep is simple.
 			 */
-			allocationSize = roundpow2(state->totalSourceSize >> 4);
+			assert(state->totalSourceSize <= SIZE_MAX);
+			allocationSize = roundpow2((size_t)state->totalSourceSize >> 4);
 
 			if (boundSize > allocationSize) {
 				allocationSize = roundpow2(boundSize);
@@ -1032,19 +1054,15 b' static void compress_worker(WorkerState*'
 
 		dest = (char*)destBuffer->dest + destOffset;
 
-		if (state->cdict) {
-			zresult = ZSTD_compress_usingCDict(state->cctx, dest, destAvailable,
-				source, sourceSize, state->cdict);
-		}
-		else {
-			if (!state->cParams) {
-				zparams.cParams = ZSTD_getCParams(state->cLevel, sourceSize, 0);
-			}
+		opInBuffer.src = source;
+		opInBuffer.size = sourceSize;
+		opInBuffer.pos = 0;
 
-			zresult = ZSTD_compress_advanced(state->cctx, dest, destAvailable,
-				source, sourceSize, NULL, 0, zparams);
-		}
+		opOutBuffer.dst = dest;
+		opOutBuffer.size = destAvailable;
+		opOutBuffer.pos = 0;
 
+		zresult = ZSTD_CCtx_setPledgedSrcSize(state->cctx, sourceSize);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;
@@ -1052,10 +1070,23 b' static void compress_worker(WorkerState*'
 			break;
 		}
 
+		zresult = ZSTD_compress_generic(state->cctx, &opOutBuffer, &opInBuffer, ZSTD_e_end);
+		if (ZSTD_isError(zresult)) {
+			state->error = WorkerError_zstd;
+			state->zresult = zresult;
+			state->errorOffset = inputOffset;
+			break;
+		}
+		else if (zresult) {
+			state->error = WorkerError_nospace;
+			state->errorOffset = inputOffset;
+			break;
+		}
+
 		destBuffer->segments[inputOffset - currentBufferStartOffset].offset = destOffset;
-		destBuffer->segments[inputOffset - currentBufferStartOffset].length = zresult;
+		destBuffer->segments[inputOffset - currentBufferStartOffset].length = opOutBuffer.pos;
 
-		destOffset += zresult;
+		destOffset += opOutBuffer.pos;
 		remainingItems--;
 	}
 
@@ -1072,15 +1103,14 b' static void compress_worker(WorkerState*'
 }
 
 ZstdBufferWithSegmentsCollection* compress_from_datasources(ZstdCompressor* compressor,
-	DataSources* sources, unsigned int threadCount) {
-	ZSTD_parameters zparams;
+	DataSources* sources, Py_ssize_t threadCount) {
 	unsigned long long bytesPerWorker;
 	POOL_ctx* pool = NULL;
 	WorkerState* workerStates = NULL;
 	Py_ssize_t i;
 	unsigned long long workerBytes = 0;
 	Py_ssize_t workerStartOffset = 0;
-	size_t currentThread = 0;
+	Py_ssize_t currentThread = 0;
 	int errored = 0;
 	Py_ssize_t segmentsCount = 0;
 	Py_ssize_t segmentIndex;
@@ -1093,34 +1123,12 b' ZstdBufferWithSegmentsCollection* compre'
 	assert(threadCount >= 1);
 
 	/* More threads than inputs makes no sense. */
-	threadCount = sources->sourcesSize < threadCount ? (unsigned int)sources->sourcesSize
+	threadCount = sources->sourcesSize < threadCount ? sources->sourcesSize
 													 : threadCount;
 
 	/* TODO lower thread count when input size is too small and threads would add
 	overhead. */
 
-	/*
-	 * When dictionaries are used, parameters are derived from the size of the
-	 * first element.
-	 *
-	 * TODO come up with a better mechanism.
-	 */
-	memset(&zparams, 0, sizeof(zparams));
-	if (compressor->cparams) {
-		ztopy_compression_parameters(compressor->cparams, &zparams.cParams);
-	}
-	else {
-		zparams.cParams = ZSTD_getCParams(compressor->compressionLevel,
-			sources->sources[0].sourceSize,
-			compressor->dict ? compressor->dict->dictSize : 0);
-	}
-
-	zparams.fParams = compressor->fparams;
-
-	if (0 != populate_cdict(compressor, &zparams)) {
-		return NULL;
-	}
-
 	workerStates = PyMem_Malloc(threadCount * sizeof(WorkerState));
 	if (NULL == workerStates) {
 		PyErr_NoMemory();
@@ -1140,16 +1148,42 b' ZstdBufferWithSegmentsCollection* compre'
 	bytesPerWorker = sources->totalSourceSize / threadCount;
 
 	for (i = 0; i < threadCount; i++) {
+		size_t zresult;
+
 		workerStates[i].cctx = ZSTD_createCCtx();
 		if (!workerStates[i].cctx) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
-		workerStates[i].cdict = compressor->cdict;
-		workerStates[i].cLevel = compressor->compressionLevel;
-		workerStates[i].cParams = compressor->cparams;
-		workerStates[i].fParams = compressor->fparams;
+		zresult = ZSTD_CCtx_setParametersUsingCCtxParams(workerStates[i].cctx,
+			compressor->params);
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "could not set compression parameters: %s",
+				ZSTD_getErrorName(zresult));
+			goto finally;
+		}
+
+		if (compressor->dict) {
+			if (compressor->dict->cdict) {
+				zresult = ZSTD_CCtx_refCDict(workerStates[i].cctx, compressor->dict->cdict);
+			}
+			else {
+				zresult = ZSTD_CCtx_loadDictionary_advanced(
+					workerStates[i].cctx,
+					compressor->dict->dictData,
+					compressor->dict->dictSize,
+					ZSTD_dlm_byRef,
+					compressor->dict->dictType);
+			}
+
+			if (ZSTD_isError(zresult)) {
+				PyErr_Format(ZstdError, "could not load compression dictionary: %s",
+					ZSTD_getErrorName(zresult));
+				goto finally;
+			}
+
+		}
 
 		workerStates[i].sources = sources->sources;
 		workerStates[i].sourcesSize = sources->sourcesSize;
@@ -1221,6 +1255,13 b' ZstdBufferWithSegmentsCollection* compre'
 				workerStates[i].errorOffset, ZSTD_getErrorName(workerStates[i].zresult));
 			errored = 1;
 			break;
+
+		case WorkerError_nospace:
+			PyErr_Format(ZstdError, "error compressing item %zd: not enough space in output",
+				workerStates[i].errorOffset);
+			errored = 1;
+			break;
+
 		default:
 			;
 		}
@@ -1341,12 +1382,6 b' static ZstdBufferWithSegmentsCollection*'
 	Py_ssize_t sourceCount = 0;
 	ZstdBufferWithSegmentsCollection* result = NULL;
 
-	if (self->mtcctx) {
-		PyErr_SetString(ZstdError,
-			"function cannot be called on ZstdCompressor configured for multi-threaded compression");
-		return NULL;
-	}
-
 	memset(&sources, 0, sizeof(sources));
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:multi_compress_to_buffer", kwlist,
@@ -1372,8 +1407,14 b' static ZstdBufferWithSegmentsCollection*'
 		}
 
 		for (i = 0; i < buffer->segmentCount; i++) {
+			if (buffer->segments[i].length > SIZE_MAX) {
+				PyErr_Format(PyExc_ValueError,
+					"buffer segment %zd is too large for this platform", i);
+				goto finally;
+			}
+
 			sources.sources[i].sourceData = (char*)buffer->data + buffer->segments[i].offset;
-			sources.sources[i].sourceSize = buffer->segments[i].length;
+			sources.sources[i].sourceSize = (size_t)buffer->segments[i].length;
 			sources.totalSourceSize += buffer->segments[i].length;
 		}
 
@@ -1397,8 +1438,15 b' static ZstdBufferWithSegmentsCollection*'
 			buffer = collection->buffers[i];
 
 			for (j = 0; j < buffer->segmentCount; j++) {
+				if (buffer->segments[j].length > SIZE_MAX) {
+					PyErr_Format(PyExc_ValueError,
+						"buffer segment %zd in buffer %zd is too large for this platform",
+						j, i);
+					goto finally;
+				}
+
 				sources.sources[offset].sourceData = (char*)buffer->data + buffer->segments[j].offset;
-				sources.sources[offset].sourceSize = buffer->segments[j].length;
+				sources.sources[offset].sourceSize = (size_t)buffer->segments[j].length;
 				sources.totalSourceSize += buffer->segments[j].length;
 
 				offset++;
@@ -1416,11 +1464,6 b' static ZstdBufferWithSegmentsCollection*'
 			goto finally;
 		}
 
-		/*
-		 * It isn't clear whether the address referred to by Py_buffer.buf
-		 * is still valid after PyBuffer_Release. We we hold a reference to all
-		 * Py_buffer instances for the duration of the operation.
-		 */
 		dataBuffers = PyMem_Malloc(sourceCount * sizeof(Py_buffer));
 		if (NULL == dataBuffers) {
 			PyErr_NoMemory();
@@ -1459,6 +1502,11 b' static ZstdBufferWithSegmentsCollection*'
 		goto finally;
 	}
 
+	if (sources.totalSourceSize > SIZE_MAX) {
+		PyErr_SetString(PyExc_ValueError, "sources are too large for this platform");
+		goto finally;
+	}
+
 	result = compress_from_datasources(self, &sources, threads);
 
 finally:
@@ -1482,12 +1530,24 b' static PyMethodDef ZstdCompressor_method'
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ },
 	{ "copy_stream", (PyCFunction)ZstdCompressor_copy_stream,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ },
-	{ "read_from", (PyCFunction)ZstdCompressor_read_from,
-	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_from__doc__ },
-	{ "write_to", (PyCFunction)ZstdCompressor_write_to,
-	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_write_to___doc__ },
+	{ "stream_reader", (PyCFunction)ZstdCompressor_stream_reader,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_reader__doc__ },
+	{ "stream_writer", (PyCFunction)ZstdCompressor_stream_writer,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_writer___doc__ },
+	{ "read_to_iter", (PyCFunction)ZstdCompressor_read_to_iter,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_to_iter__doc__ },
+	/* TODO Remove deprecated API */
+	{ "read_from", (PyCFunction)ZstdCompressor_read_to_iter,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_to_iter__doc__ },
+	/* TODO remove deprecated API */
+	{ "write_to", (PyCFunction)ZstdCompressor_stream_writer,
+	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_writer___doc__ },
 	{ "multi_compress_to_buffer", (PyCFunction)ZstdCompressor_multi_compress_to_buffer,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_multi_compress_to_buffer__doc__ },
+	{ "memory_size", (PyCFunction)ZstdCompressor_memory_size,
+	METH_NOARGS, ZstdCompressor_memory_size__doc__ },
+	{ "frame_progression", (PyCFunction)ZstdCompressor_frame_progression,
+	METH_NOARGS, ZstdCompressor_frame_progression__doc__ },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/compressoriterator.c

@@ -21,10 +21,9 b' static void ZstdCompressorIterator_deall'
 	Py_XDECREF(self->compressor);
 	Py_XDECREF(self->reader);
 
-	if (self->buffer) {
-		PyBuffer_Release(self->buffer);
-		PyMem_FREE(self->buffer);
-		self->buffer = NULL;
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	if (self->output.dst) {
@@ -58,14 +57,8 b' feedcompressor:'
 	/* If we have data left in the input, consume it. */
 	if (self->input.pos < self->input.size) {
 		Py_BEGIN_ALLOW_THREADS
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_compressStream(self->compressor->mtcctx,
-				&self->output, &self->input);
-		}
-		else {
-			zresult = ZSTD_compressStream(self->compressor->cstream, &self->output,
-				&self->input);
-		}
+		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+			&self->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		/* Release the Python object holding the input buffer. */
@@ -107,14 +100,14 b' feedcompressor:'
 			PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 		}
 		else {
-			assert(self->buffer && self->buffer->buf);
+			assert(self->buffer.buf);
 
 			/* Only support contiguous C arrays. */
-			assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL);
-			assert(self->buffer->itemsize == 1);
+			assert(self->buffer.strides == NULL && self->buffer.suboffsets == NULL);
+			assert(self->buffer.itemsize == 1);
 
-			readBuffer = (char*)self->buffer->buf + self->bufferOffset;
-			bufferRemaining = self->buffer->len - self->bufferOffset;
+			readBuffer = (char*)self->buffer.buf + self->bufferOffset;
+			bufferRemaining = self->buffer.len - self->bufferOffset;
 			readSize = min(bufferRemaining, (Py_ssize_t)self->inSize);
 			self->bufferOffset += readSize;
 		}
@@ -130,12 +123,12 b' feedcompressor:'
 
 	/* EOF */
 	if (0 == readSize) {
-		if (self->compressor->mtcctx) {
-			zresult = ZSTDMT_endStream(self->compressor->mtcctx, &self->output);
-		}
-		else {
-			zresult = ZSTD_endStream(self->compressor->cstream, &self->output);
-		}
+		self->input.src = NULL;
+		self->input.size = 0;
+		self->input.pos = 0;
+
+		zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+			&self->input, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
@@ -159,13 +152,8 b' feedcompressor:'
 	self->input.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	if (self->compressor->mtcctx) {
-		zresult = ZSTDMT_compressStream(self->compressor->mtcctx, &self->output,
-			&self->input);
-	}
-	else {
-		zresult = ZSTD_compressStream(self->compressor->cstream, &self->output, &self->input);
-	}
+	zresult = ZSTD_compress_generic(self->compressor->cctx, &self->output,
+		&self->input, ZSTD_e_continue);
 	Py_END_ALLOW_THREADS
 
 	/* The input buffer currently points to memory managed by Python

contrib/python-zstandard/c-ext/constants.c

@@ -52,6 +52,11 b' void constants_module_init(PyObject* mod'
 		PyErr_Format(PyExc_ValueError, "could not create frame header object");
 	}
 
+	PyModule_AddObject(mod, "CONTENTSIZE_UNKNOWN",
+		PyLong_FromUnsignedLongLong(ZSTD_CONTENTSIZE_UNKNOWN));
+	PyModule_AddObject(mod, "CONTENTSIZE_ERROR",
+		PyLong_FromUnsignedLongLong(ZSTD_CONTENTSIZE_ERROR));
+
 	PyModule_AddIntConstant(mod, "MAX_COMPRESSION_LEVEL", ZSTD_maxCLevel());
 	PyModule_AddIntConstant(mod, "COMPRESSION_RECOMMENDED_INPUT_SIZE",
 		(long)ZSTD_CStreamInSize());
@@ -75,7 +80,9 b' void constants_module_init(PyObject* mod'
 	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MIN", ZSTD_SEARCHLENGTH_MIN);
 	PyModule_AddIntConstant(mod, "SEARCHLENGTH_MAX", ZSTD_SEARCHLENGTH_MAX);
 	PyModule_AddIntConstant(mod, "TARGETLENGTH_MIN", ZSTD_TARGETLENGTH_MIN);
-	PyModule_AddIntConstant(mod, "TARGETLENGTH_MAX", ZSTD_TARGETLENGTH_MAX);
+	PyModule_AddIntConstant(mod, "LDM_MINMATCH_MIN", ZSTD_LDM_MINMATCH_MIN);
+	PyModule_AddIntConstant(mod, "LDM_MINMATCH_MAX", ZSTD_LDM_MINMATCH_MAX);
+	PyModule_AddIntConstant(mod, "LDM_BUCKETSIZELOG_MAX", ZSTD_LDM_BUCKETSIZELOG_MAX);
 
 	PyModule_AddIntConstant(mod, "STRATEGY_FAST", ZSTD_fast);
 	PyModule_AddIntConstant(mod, "STRATEGY_DFAST", ZSTD_dfast);
@@ -84,4 +91,12 b' void constants_module_init(PyObject* mod'
 	PyModule_AddIntConstant(mod, "STRATEGY_LAZY2", ZSTD_lazy2);
 	PyModule_AddIntConstant(mod, "STRATEGY_BTLAZY2", ZSTD_btlazy2);
 	PyModule_AddIntConstant(mod, "STRATEGY_BTOPT", ZSTD_btopt);
+	PyModule_AddIntConstant(mod, "STRATEGY_BTULTRA", ZSTD_btultra);
+
+	PyModule_AddIntConstant(mod, "DICT_TYPE_AUTO", ZSTD_dct_auto);
+	PyModule_AddIntConstant(mod, "DICT_TYPE_RAWCONTENT", ZSTD_dct_rawContent);
+	PyModule_AddIntConstant(mod, "DICT_TYPE_FULLDICT", ZSTD_dct_fullDict);
+
+	PyModule_AddIntConstant(mod, "FORMAT_ZSTD1", ZSTD_f_zstd1);
+	PyModule_AddIntConstant(mod, "FORMAT_ZSTD1_MAGICLESS", ZSTD_f_zstd1_magicless);
 }

contrib/python-zstandard/c-ext/decompressionwriter.c

@@ -27,7 +27,7 b' static PyObject* ZstdDecompressionWriter'
 		return NULL;
 	}
 
-	if (0 != init_dstream(self->decompressor)) {
+	if (ensure_dctx(self->decompressor, 1)) {
 		return NULL;
 	}
 
@@ -44,18 +44,17 b' static PyObject* ZstdDecompressionWriter'
 }
 
 static PyObject* ZstdDecompressionWriter_memory_size(ZstdDecompressionWriter* self) {
-	if (!self->decompressor->dstream) {
-		PyErr_SetString(ZstdError, "cannot determine size of inactive decompressor; "
-			"call when context manager is active");
-		return NULL;
-	}
-
-	return PyLong_FromSize_t(ZSTD_sizeof_DStream(self->decompressor->dstream));
+	return PyLong_FromSize_t(ZSTD_sizeof_DCtx(self->decompressor->dctx));
 }
 
-static PyObject* ZstdDecompressionWriter_write(ZstdDecompressionWriter* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
+static PyObject* ZstdDecompressionWriter_write(ZstdDecompressionWriter* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		NULL
+	};
+
+	PyObject* result = NULL;
+	Py_buffer source;
 	size_t zresult = 0;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
@@ -63,41 +62,47 b' static PyObject* ZstdDecompressionWriter'
 	Py_ssize_t totalWrite = 0;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:write", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write",
 #else
-	if (!PyArg_ParseTuple(args, "s#:write", &source, &sourceSize)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:write",
 #endif
+		kwlist, &source)) {
 		return NULL;
 	}
 
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
 	if (!self->entered) {
 		PyErr_SetString(ZstdError, "write must be called from an active context manager");
-		return NULL;
+		goto finally;
 	}
 
-	assert(self->decompressor->dstream);
-
 	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
-		return PyErr_NoMemory();
+		PyErr_NoMemory();
+		goto finally;
 	}
 	output.size = self->outSize;
 	output.pos = 0;
 
-	input.src = source;
-	input.size = sourceSize;
+	input.src = source.buf;
+	input.size = source.len;
 	input.pos = 0;
 
-	while ((ssize_t)input.pos < sourceSize) {
+	while ((ssize_t)input.pos < source.len) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_decompressStream(self->decompressor->dstream, &output, &input);
+		zresult = ZSTD_decompress_generic(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyMem_Free(output.dst);
 			PyErr_Format(ZstdError, "zstd decompress error: %s",
 				ZSTD_getErrorName(zresult));
-			return NULL;
+			goto finally;
 		}
 
 		if (output.pos) {
@@ -115,7 +120,11 b' static PyObject* ZstdDecompressionWriter'
 
 	PyMem_Free(output.dst);
 
-	return PyLong_FromSsize_t(totalWrite);
+	result = PyLong_FromSsize_t(totalWrite);
+
+finally:
+	PyBuffer_Release(&source);
+	return result;
 }
 
 static PyMethodDef ZstdDecompressionWriter_methods[] = {
@@ -125,7 +134,7 b' static PyMethodDef ZstdDecompressionWrit'
 	PyDoc_STR("Exit a decompression context.") },
 	{ "memory_size", (PyCFunction)ZstdDecompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size in bytes of the underlying decompressor.") },
-	{ "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS,
+	{ "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
 	{ NULL, NULL }
 };

contrib/python-zstandard/c-ext/decompressobj.c

@@ -20,56 +20,61 b' static void DecompressionObj_dealloc(Zst'
 	PyObject_Del(self);
 }
 
-static PyObject* DecompressionObj_decompress(ZstdDecompressionObj* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
+static PyObject* DecompressionObj_decompress(ZstdDecompressionObj* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		NULL
+	};
+
+	Py_buffer source;
 	size_t zresult;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
-	size_t outSize = ZSTD_DStreamOutSize();
 	PyObject* result = NULL;
 	Py_ssize_t resultSize = 0;
 
-	/* Constructor should ensure stream is populated. */
-	assert(self->decompressor->dstream);
-
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot use a decompressobj multiple times");
 		return NULL;
 	}
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:decompress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:decompress",
 #else
-	if (!PyArg_ParseTuple(args, "s#:decompress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:decompress",
 #endif
-		&source, &sourceSize)) {
+		kwlist, &source)) {
 		return NULL;
 	}
 
-	input.src = source;
-	input.size = sourceSize;
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	input.src = source.buf;
+	input.size = source.len;
 	input.pos = 0;
 
-	output.dst = PyMem_Malloc(outSize);
+	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
-		return NULL;
+		goto except;
 	}
-	output.size = outSize;
+	output.size = self->outSize;
 	output.pos = 0;
 
 	/* Read input until exhausted. */
 	while (input.pos < input.size) {
 		Py_BEGIN_ALLOW_THREADS
-		zresult = ZSTD_decompressStream(self->decompressor->dstream, &output, &input);
+		zresult = ZSTD_decompress_generic(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd decompressor error: %s",
 				ZSTD_getErrorName(zresult));
-			result = NULL;
-			goto finally;
+			goto except;
 		}
 
 		if (0 == zresult) {
@@ -79,7 +84,8 b' static PyObject* DecompressionObj_decomp'
 		if (output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
-				if (-1 == _PyBytes_Resize(&result, resultSize + output.pos)) {
+				if (-1 == safe_pybytes_resize(&result, resultSize + output.pos)) {
+					Py_XDECREF(result);
 					goto except;
 				}
 
@@ -108,13 +114,14 b' except:'
 
 finally:
 	PyMem_Free(output.dst);
+	PyBuffer_Release(&source);
 
 	return result;
 }
 
 static PyMethodDef DecompressionObj_methods[] = {
 	{ "decompress", (PyCFunction)DecompressionObj_decompress,
-	  METH_VARARGS, PyDoc_STR("decompress data") },
+	  METH_VARARGS | METH_KEYWORDS, PyDoc_STR("decompress data") },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/decompressor.c

@@ -12,54 +12,40 b''
 extern PyObject* ZstdError;
 
 /**
-  * Ensure the ZSTD_DStream on a ZstdDecompressor is initialized and reset.
-  *
-  * This should be called before starting a decompression operation with a
-  * ZSTD_DStream on a ZstdDecompressor.
-  */
-int init_dstream(ZstdDecompressor* decompressor) {
-	void* dictData = NULL;
-	size_t dictSize = 0;
+ * Ensure the ZSTD_DCtx on a decompressor is initiated and ready for a new operation.
+ */
+int ensure_dctx(ZstdDecompressor* decompressor, int loadDict) {
 	size_t zresult;
 
-	/* Simple case of dstream already exists. Just reset it. */
-	if (decompressor->dstream) {
-		zresult = ZSTD_resetDStream(decompressor->dstream);
+	ZSTD_DCtx_reset(decompressor->dctx);
+
+	if (decompressor->maxWindowSize) {
+		zresult = ZSTD_DCtx_setMaxWindowSize(decompressor->dctx, decompressor->maxWindowSize);
 		if (ZSTD_isError(zresult)) {
-			PyErr_Format(ZstdError, "could not reset DStream: %s",
+			PyErr_Format(ZstdError, "unable to set max window size: %s",
 				ZSTD_getErrorName(zresult));
-			return -1;
+			return 1;
 		}
-
-		return 0;
 	}
 
-	decompressor->dstream = ZSTD_createDStream();
-	if (!decompressor->dstream) {
-		PyErr_SetString(ZstdError, "could not create DStream");
-		return -1;
-	}
-
-	if (decompressor->dict) {
-		dictData = decompressor->dict->dictData;
-		dictSize = decompressor->dict->dictSize;
+	zresult = ZSTD_DCtx_setFormat(decompressor->dctx, decompressor->format);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "unable to set decoding format: %s",
+			ZSTD_getErrorName(zresult));
+		return 1;
 	}
 
-	if (dictData) {
-		zresult = ZSTD_initDStream_usingDict(decompressor->dstream, dictData, dictSize);
-	}
-	else {
-		zresult = ZSTD_initDStream(decompressor->dstream);
-	}
+	if (loadDict && decompressor->dict) {
+		if (ensure_ddict(decompressor->dict)) {
+			return 1;
+		}
 
-	if (ZSTD_isError(zresult)) {
-		/* Don't leave a reference to an invalid object. */
-		ZSTD_freeDStream(decompressor->dstream);
-		decompressor->dstream = NULL;
-
-		PyErr_Format(ZstdError, "could not initialize DStream: %s",
-			ZSTD_getErrorName(zresult));
-		return -1;
+		zresult = ZSTD_DCtx_refDDict(decompressor->dctx, decompressor->dict->ddict);
+		if (ZSTD_isError(zresult)) {
+			PyErr_Format(ZstdError, "unable to reference prepared dictionary: %s",
+				ZSTD_getErrorName(zresult));
+			return 1;
+		}
 	}
 
 	return 0;
@@ -76,36 +62,46 b' PyDoc_STRVAR(Decompressor__doc__,'
 static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"dict_data",
+		"max_window_size",
+		"format",
 		NULL
 	};
 
 	ZstdCompressionDict* dict = NULL;
+	size_t maxWindowSize = 0;
+	ZSTD_format_e format = ZSTD_f_zstd1;
 
 	self->dctx = NULL;
 	self->dict = NULL;
-	self->ddict = NULL;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!:ZstdDecompressor", kwlist,
-		&ZstdCompressionDictType, &dict)) {
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!II:ZstdDecompressor", kwlist,
+		&ZstdCompressionDictType, &dict, &maxWindowSize, &format)) {
 		return -1;
 	}
 
-	/* TODO lazily initialize the reference ZSTD_DCtx on first use since
-	   not instances of ZstdDecompressor will use a ZSTD_DCtx. */
 	self->dctx = ZSTD_createDCtx();
 	if (!self->dctx) {
 		PyErr_NoMemory();
 		goto except;
 	}
 
+	self->maxWindowSize = maxWindowSize;
+	self->format = format;
+
 	if (dict) {
 		self->dict = dict;
 		Py_INCREF(dict);
 	}
 
+	if (ensure_dctx(self, 1)) {
+		goto except;
+	}
+
 	return 0;
 
 except:
+	Py_CLEAR(self->dict);
+
 	if (self->dctx) {
 		ZSTD_freeDCtx(self->dctx);
 		self->dctx = NULL;
@@ -117,16 +113,6 b' except:'
 static void Decompressor_dealloc(ZstdDecompressor* self) {
 	Py_CLEAR(self->dict);
 
-	if (self->ddict) {
-		ZSTD_freeDDict(self->ddict);
-		self->ddict = NULL;
-	}
-
-	if (self->dstream) {
-		ZSTD_freeDStream(self->dstream);
-		self->dstream = NULL;
-	}
-
 	if (self->dctx) {
 		ZSTD_freeDCtx(self->dctx);
 		self->dctx = NULL;
@@ -135,6 +121,20 b' static void Decompressor_dealloc(ZstdDec'
 	PyObject_Del(self);
 }
 
+PyDoc_STRVAR(Decompressor_memory_size__doc__,
+"memory_size() -- Size of decompression context, in bytes\n"
+);
+
+static PyObject* Decompressor_memory_size(ZstdDecompressor* self) {
+	if (self->dctx) {
+		return PyLong_FromSize_t(ZSTD_sizeof_DCtx(self->dctx));
+	}
+	else {
+		PyErr_SetString(ZstdError, "no decompressor context found; this should never happen");
+		return NULL;
+	}
+}
+
 PyDoc_STRVAR(Decompressor_copy_stream__doc__,
 	"copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n"
 	"\n"
@@ -166,7 +166,7 b' static PyObject* Decompressor_copy_strea'
 	Py_ssize_t totalWrite = 0;
 	char* readBuffer;
 	Py_ssize_t readSize;
-	PyObject* readResult;
+	PyObject* readResult = NULL;
 	PyObject* res = NULL;
 	size_t zresult = 0;
 	PyObject* writeResult;
@@ -191,7 +191,7 b' static PyObject* Decompressor_copy_strea'
 	/* Prevent free on uninitialized memory in finally. */
 	output.dst = NULL;
 
-	if (0 != init_dstream(self)) {
+	if (ensure_dctx(self, 1)) {
 		res = NULL;
 		goto finally;
 	}
@@ -229,7 +229,7 b' static PyObject* Decompressor_copy_strea'
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompressStream(self->dstream, &output, &input);
+			zresult = ZSTD_decompress_generic(self->dctx, &output, &input);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
@@ -252,6 +252,8 b' static PyObject* Decompressor_copy_strea'
 				output.pos = 0;
 			}
 		}
+
+		Py_CLEAR(readResult);
 	}
 
 	/* Source stream is exhausted. Finish up. */
@@ -267,6 +269,8 b' finally:'
 		PyMem_Free(output.dst);
 	}
 
+	Py_XDECREF(readResult);
+
 	return res;
 }
 
@@ -300,98 +304,114 b' PyObject* Decompressor_decompress(ZstdDe'
 		NULL
 	};
 
-	const char* source;
-	Py_ssize_t sourceSize;
+	Py_buffer source;
 	Py_ssize_t maxOutputSize = 0;
 	unsigned long long decompressedSize;
 	size_t destCapacity;
 	PyObject* result = NULL;
-	void* dictData = NULL;
-	size_t dictSize = 0;
 	size_t zresult;
+	ZSTD_outBuffer outBuffer;
+	ZSTD_inBuffer inBuffer;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n:decompress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|n:decompress",
 #else
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n:decompress",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|n:decompress",
 #endif
-		kwlist, &source, &sourceSize, &maxOutputSize)) {
+		kwlist, &source, &maxOutputSize)) {
 		return NULL;
 	}
 
-	if (self->dict) {
-		dictData = self->dict->dictData;
-		dictSize = self->dict->dictSize;
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
 	}
 
-	if (dictData && !self->ddict) {
-		Py_BEGIN_ALLOW_THREADS
-		self->ddict = ZSTD_createDDict_byReference(dictData, dictSize);
-		Py_END_ALLOW_THREADS
-
-		if (!self->ddict) {
-			PyErr_SetString(ZstdError, "could not create decompression dict");
-			return NULL;
-		}
+	if (ensure_dctx(self, 1)) {
+		goto finally;
 	}
 
-	decompressedSize = ZSTD_getDecompressedSize(source, sourceSize);
-	/* 0 returned if content size not in the zstd frame header */
-	if (0 == decompressedSize) {
+	decompressedSize = ZSTD_getFrameContentSize(source.buf, source.len);
+
+	if (ZSTD_CONTENTSIZE_ERROR == decompressedSize) {
+		PyErr_SetString(ZstdError, "error determining content size from frame header");
+		goto finally;
+	}
+	/* Special case of empty frame. */
+	else if (0 == decompressedSize) {
+		result = PyBytes_FromStringAndSize("", 0);
+		goto finally;
+	}
+	/* Missing content size in frame header. */
+	if (ZSTD_CONTENTSIZE_UNKNOWN == decompressedSize) {
 		if (0 == maxOutputSize) {
-			PyErr_SetString(ZstdError, "input data invalid or missing content size "
-				"in frame header");
-			return NULL;
+			PyErr_SetString(ZstdError, "could not determine content size in frame header");
+			goto finally;
 		}
-		else {
-			result = PyBytes_FromStringAndSize(NULL, maxOutputSize);
-			destCapacity = maxOutputSize;
+
+		result = PyBytes_FromStringAndSize(NULL, maxOutputSize);
+		destCapacity = maxOutputSize;
+		decompressedSize = 0;
+	}
+	/* Size is recorded in frame header. */
+	else {
+		assert(SIZE_MAX >= PY_SSIZE_T_MAX);
+		if (decompressedSize > PY_SSIZE_T_MAX) {
+			PyErr_SetString(ZstdError, "frame is too large to decompress on this platform");
+			goto finally;
 		}
-	}
-	else {
-		result = PyBytes_FromStringAndSize(NULL, decompressedSize);
-		destCapacity = decompressedSize;
+
+		result = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)decompressedSize);
+		destCapacity = (size_t)decompressedSize;
 	}
 
 	if (!result) {
-		return NULL;
+		goto finally;
 	}
 
+	outBuffer.dst = PyBytes_AsString(result);
+	outBuffer.size = destCapacity;
+	outBuffer.pos = 0;
+
+	inBuffer.src = source.buf;
+	inBuffer.size = source.len;
+	inBuffer.pos = 0;
+
 	Py_BEGIN_ALLOW_THREADS
-	if (self->ddict) {
-		zresult = ZSTD_decompress_usingDDict(self->dctx,
-			PyBytes_AsString(result), destCapacity,
-			source, sourceSize, self->ddict);
-	}
-	else {
-		zresult = ZSTD_decompressDCtx(self->dctx,
-			PyBytes_AsString(result), destCapacity, source, sourceSize);
-	}
+	zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult));
-		Py_DECREF(result);
-		return NULL;
+		Py_CLEAR(result);
+		goto finally;
 	}
-	else if (decompressedSize && zresult != decompressedSize) {
+	else if (zresult) {
+		PyErr_Format(ZstdError, "decompression error: did not decompress full frame");
+		Py_CLEAR(result);
+		goto finally;
+	}
+	else if (decompressedSize && outBuffer.pos != decompressedSize) {
 		PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu",
 			zresult, decompressedSize);
-		Py_DECREF(result);
-		return NULL;
+		Py_CLEAR(result);
+		goto finally;
 	}
-	else if (zresult < destCapacity) {
-		if (_PyBytes_Resize(&result, zresult)) {
-			Py_DECREF(result);
-			return NULL;
+	else if (outBuffer.pos < destCapacity) {
+		if (safe_pybytes_resize(&result, outBuffer.pos)) {
+			Py_CLEAR(result);
+			goto finally;
 		}
 	}
 
+finally:
+	PyBuffer_Release(&source);
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_decompressobj__doc__,
-"decompressobj()\n"
+"decompressobj([write_size=default])\n"
 "\n"
 "Incrementally feed data into a decompressor.\n"
 "\n"
@@ -400,25 +420,43 b' PyDoc_STRVAR(Decompressor_decompressobj_'
 "callers can swap in the zstd decompressor while using the same API.\n"
 );
 
-static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self) {
-	ZstdDecompressionObj* result = (ZstdDecompressionObj*)PyObject_CallObject((PyObject*)&ZstdDecompressionObjType, NULL);
+static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"write_size",
+		NULL
+	};
+
+	ZstdDecompressionObj* result = NULL;
+	size_t outSize = ZSTD_DStreamOutSize();
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|k:decompressobj", kwlist, &outSize)) {
+		return NULL;
+	}
+
+	if (!outSize) {
+		PyErr_SetString(PyExc_ValueError, "write_size must be positive");
+		return NULL;
+	}
+
+	result = (ZstdDecompressionObj*)PyObject_CallObject((PyObject*)&ZstdDecompressionObjType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
-	if (0 != init_dstream(self)) {
+	if (ensure_dctx(self, 1)) {
 		Py_DECREF(result);
 		return NULL;
 	}
 
 	result->decompressor = self;
 	Py_INCREF(result->decompressor);
+	result->outSize = outSize;
 
 	return result;
 }
 
-PyDoc_STRVAR(Decompressor_read_from__doc__,
-"read_from(reader[, read_size=default, write_size=default, skip_bytes=0])\n"
+PyDoc_STRVAR(Decompressor_read_to_iter__doc__,
+"read_to_iter(reader[, read_size=default, write_size=default, skip_bytes=0])\n"
 "Read compressed data and return an iterator\n"
 "\n"
 "Returns an iterator of decompressed data chunks produced from reading from\n"
@@ -437,7 +475,7 b' PyDoc_STRVAR(Decompressor_read_from__doc'
 "the source.\n"
 );
 
-static ZstdDecompressorIterator* Decompressor_read_from(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
+static ZstdDecompressorIterator* Decompressor_read_to_iter(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"reader",
 		"read_size",
@@ -452,7 +490,7 b' static ZstdDecompressorIterator* Decompr'
 	ZstdDecompressorIterator* result;
 	size_t skipBytes = 0;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk:read_from", kwlist,
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk:read_to_iter", kwlist,
 		&reader, &inSize, &outSize, &skipBytes)) {
 		return NULL;
 	}
@@ -474,14 +512,7 b' static ZstdDecompressorIterator* Decompr'
 	}
 	else if (1 == PyObject_CheckBuffer(reader)) {
 		/* Object claims it is a buffer. Try to get a handle to it. */
-		result->buffer = PyMem_Malloc(sizeof(Py_buffer));
-		if (!result->buffer) {
-			goto except;
-		}
-
-		memset(result->buffer, 0, sizeof(Py_buffer));
-
-		if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) {
+		if (0 != PyObject_GetBuffer(reader, &result->buffer, PyBUF_CONTIG_RO)) {
 			goto except;
 		}
 	}
@@ -498,7 +529,7 b' static ZstdDecompressorIterator* Decompr'
 	result->outSize = outSize;
 	result->skipBytes = skipBytes;
 
-	if (0 != init_dstream(self)) {
+	if (ensure_dctx(self, 1)) {
 		goto except;
 	}
 
@@ -511,13 +542,6 b' static ZstdDecompressorIterator* Decompr'
 	goto finally;
 
 except:
-	Py_CLEAR(result->reader);
-
-	if (result->buffer) {
-		PyBuffer_Release(result->buffer);
-		Py_CLEAR(result->buffer);
-	}
-
 	Py_CLEAR(result);
 
 finally:
@@ -525,7 +549,62 b' finally:'
 	return result;
 }
 
-PyDoc_STRVAR(Decompressor_write_to__doc__,
+PyDoc_STRVAR(Decompressor_stream_reader__doc__,
+"stream_reader(source, [read_size=default])\n"
+"\n"
+"Obtain an object that behaves like an I/O stream that can be used for\n"
+"reading decompressed output from an object.\n"
+"\n"
+"The source object can be any object with a ``read(size)`` method or that\n"
+"conforms to the buffer protocol.\n"
+);
+
+static ZstdDecompressionReader* Decompressor_stream_reader(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"source",
+		"read_size",
+		NULL
+	};
+
+	PyObject* source;
+	size_t readSize = ZSTD_DStreamInSize();
+	ZstdDecompressionReader* result;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:stream_reader", kwlist,
+		&source, &readSize)) {
+		return NULL;
+	}
+
+	result = (ZstdDecompressionReader*)PyObject_CallObject((PyObject*)&ZstdDecompressionReaderType, NULL);
+	if (NULL == result) {
+		return NULL;
+	}
+
+	if (PyObject_HasAttrString(source, "read")) {
+		result->reader = source;
+		Py_INCREF(source);
+		result->readSize = readSize;
+	}
+	else if (1 == PyObject_CheckBuffer(source)) {
+		if (0 != PyObject_GetBuffer(source, &result->buffer, PyBUF_CONTIG_RO)) {
+			Py_CLEAR(result);
+			return NULL;
+		}
+	}
+	else {
+		PyErr_SetString(PyExc_TypeError,
+			"must pass an object with a read() method or that conforms to the buffer protocol");
+		Py_CLEAR(result);
+		return NULL;
+	}
+
+	result->decompressor = self;
+	Py_INCREF(self);
+
+	return result;
+}
+
+PyDoc_STRVAR(Decompressor_stream_writer__doc__,
 "Create a context manager to write decompressed data to an object.\n"
 "\n"
 "The passed object must have a ``write()`` method.\n"
@@ -538,7 +617,7 b' PyDoc_STRVAR(Decompressor_write_to__doc_'
 "streaming decompressor.\n"
 );
 
-static ZstdDecompressionWriter* Decompressor_write_to(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
+static ZstdDecompressionWriter* Decompressor_stream_writer(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"writer",
 		"write_size",
@@ -549,7 +628,7 b' static ZstdDecompressionWriter* Decompre'
 	size_t outSize = ZSTD_DStreamOutSize();
 	ZstdDecompressionWriter* result;
 
-	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:write_to", kwlist,
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k:stream_writer", kwlist,
 		&writer, &outSize)) {
 		return NULL;
 	}
@@ -579,7 +658,7 b' PyDoc_STRVAR(Decompressor_decompress_con'
 "Decompress a series of chunks using the content dictionary chaining technique\n"
 );
 
-static PyObject* Decompressor_decompress_content_dict_chain(PyObject* self, PyObject* args, PyObject* kwargs) {
+static PyObject* Decompressor_decompress_content_dict_chain(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"frames",
 		NULL
@@ -592,9 +671,8 b' static PyObject* Decompressor_decompress'
 	PyObject* chunk;
 	char* chunkData;
 	Py_ssize_t chunkSize;
-	ZSTD_DCtx* dctx = NULL;
 	size_t zresult;
-	ZSTD_frameParams frameParams;
+	ZSTD_frameHeader frameHeader;
 	void* buffer1 = NULL;
 	size_t buffer1Size = 0;
 	size_t buffer1ContentSize = 0;
@@ -603,6 +681,8 b' static PyObject* Decompressor_decompress'
 	size_t buffer2ContentSize = 0;
 	void* destBuffer = NULL;
 	PyObject* result = NULL;
+	ZSTD_outBuffer outBuffer;
+	ZSTD_inBuffer inBuffer;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!:decompress_content_dict_chain",
 		kwlist, &PyList_Type, &chunks)) {
@@ -624,7 +704,7 b' static PyObject* Decompressor_decompress'
 
 	/* We require that all chunks be zstd frames and that they have content size set. */
 	PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
-	zresult = ZSTD_getFrameParams(&frameParams, (void*)chunkData, chunkSize);
+	zresult = ZSTD_getFrameHeader(&frameHeader, (void*)chunkData, chunkSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 is not a valid zstd frame");
 		return NULL;
@@ -634,32 +714,56 b' static PyObject* Decompressor_decompress'
 		return NULL;
 	}
 
-	if (0 == frameParams.frameContentSize) {
+	if (ZSTD_CONTENTSIZE_UNKNOWN == frameHeader.frameContentSize) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 missing content size in frame");
 		return NULL;
 	}
 
-	dctx = ZSTD_createDCtx();
-	if (!dctx) {
-		PyErr_NoMemory();
+	assert(ZSTD_CONTENTSIZE_ERROR != frameHeader.frameContentSize);
+
+	/* We check against PY_SSIZE_T_MAX here because we ultimately cast the
+	 * result to a Python object and it's length can be no greater than
+	 * Py_ssize_t. In theory, we could have an intermediate frame that is 
+	 * larger. But a) why would this API be used for frames that large b)
+	 * it isn't worth the complexity to support. */
+	assert(SIZE_MAX >= PY_SSIZE_T_MAX);
+	if (frameHeader.frameContentSize > PY_SSIZE_T_MAX) {
+		PyErr_SetString(PyExc_ValueError,
+			"chunk 0 is too large to decompress on this platform");
+		return NULL;
+	}
+
+	if (ensure_dctx(self, 0)) {
 		goto finally;
 	}
 
-	buffer1Size = frameParams.frameContentSize;
+	buffer1Size = (size_t)frameHeader.frameContentSize;
 	buffer1 = PyMem_Malloc(buffer1Size);
 	if (!buffer1) {
 		goto finally;
 	}
 
+	outBuffer.dst = buffer1;
+	outBuffer.size = buffer1Size;
+	outBuffer.pos = 0;
+
+	inBuffer.src = chunkData;
+	inBuffer.size = chunkSize;
+	inBuffer.pos = 0;
+
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_decompressDCtx(dctx, buffer1, buffer1Size, chunkData, chunkSize);
+	zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "could not decompress chunk 0: %s", ZSTD_getErrorName(zresult));
 		goto finally;
 	}
+	else if (zresult) {
+		PyErr_Format(ZstdError, "chunk 0 did not decompress full frame");
+		goto finally;
+	}
 
-	buffer1ContentSize = zresult;
+	buffer1ContentSize = outBuffer.pos;
 
 	/* Special case of a simple chain. */
 	if (1 == chunksLen) {
@@ -668,7 +772,7 b' static PyObject* Decompressor_decompress'
 	}
 
 	/* This should ideally look at next chunk. But this is slightly simpler. */
-	buffer2Size = frameParams.frameContentSize;
+	buffer2Size = (size_t)frameHeader.frameContentSize;
 	buffer2 = PyMem_Malloc(buffer2Size);
 	if (!buffer2) {
 		goto finally;
@@ -688,7 +792,7 b' static PyObject* Decompressor_decompress'
 		}
 
 		PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
-		zresult = ZSTD_getFrameParams(&frameParams, (void*)chunkData, chunkSize);
+		zresult = ZSTD_getFrameHeader(&frameHeader, (void*)chunkData, chunkSize);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd is not a valid zstd frame", chunkIndex);
 			goto finally;
@@ -698,18 +802,30 b' static PyObject* Decompressor_decompress'
 			goto finally;
 		}
 
-		if (0 == frameParams.frameContentSize) {
+		if (ZSTD_CONTENTSIZE_UNKNOWN == frameHeader.frameContentSize) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd missing content size in frame", chunkIndex);
 			goto finally;
 		}
 
+		assert(ZSTD_CONTENTSIZE_ERROR != frameHeader.frameContentSize);
+
+		if (frameHeader.frameContentSize > PY_SSIZE_T_MAX) {
+			PyErr_Format(PyExc_ValueError,
+				"chunk %zd is too large to decompress on this platform", chunkIndex);
+			goto finally;
+		}
+
+		inBuffer.src = chunkData;
+		inBuffer.size = chunkSize;
+		inBuffer.pos = 0;
+
 		parity = chunkIndex % 2;
 
 		/* This could definitely be abstracted to reduce code duplication. */
 		if (parity) {
 			/* Resize destination buffer to hold larger content. */
-			if (buffer2Size < frameParams.frameContentSize) {
-				buffer2Size = frameParams.frameContentSize;
+			if (buffer2Size < frameHeader.frameContentSize) {
+				buffer2Size = (size_t)frameHeader.frameContentSize;
 				destBuffer = PyMem_Realloc(buffer2, buffer2Size);
 				if (!destBuffer) {
 					goto finally;
@@ -718,19 +834,38 b' static PyObject* Decompressor_decompress'
 			}
 
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompress_usingDict(dctx, buffer2, buffer2Size,
-				chunkData, chunkSize, buffer1, buffer1ContentSize);
+			zresult = ZSTD_DCtx_refPrefix_advanced(self->dctx,
+				buffer1, buffer1ContentSize, ZSTD_dct_rawContent);
+			Py_END_ALLOW_THREADS
+			if (ZSTD_isError(zresult)) {
+				PyErr_Format(ZstdError,
+					"failed to load prefix dictionary at chunk %zd", chunkIndex);
+				goto finally;
+			}
+
+			outBuffer.dst = buffer2;
+			outBuffer.size = buffer2Size;
+			outBuffer.pos = 0;
+
+			Py_BEGIN_ALLOW_THREADS
+			zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
 					chunkIndex, ZSTD_getErrorName(zresult));
 				goto finally;
 			}
-			buffer2ContentSize = zresult;
+			else if (zresult) {
+				PyErr_Format(ZstdError, "chunk %zd did not decompress full frame",
+					chunkIndex);
+				goto finally;
+			}
+
+			buffer2ContentSize = outBuffer.pos;
 		}
 		else {
-			if (buffer1Size < frameParams.frameContentSize) {
-				buffer1Size = frameParams.frameContentSize;
+			if (buffer1Size < frameHeader.frameContentSize) {
+				buffer1Size = (size_t)frameHeader.frameContentSize;
 				destBuffer = PyMem_Realloc(buffer1, buffer1Size);
 				if (!destBuffer) {
 					goto finally;
@@ -739,15 +874,34 b' static PyObject* Decompressor_decompress'
 			}
 
 			Py_BEGIN_ALLOW_THREADS
-			zresult = ZSTD_decompress_usingDict(dctx, buffer1, buffer1Size,
-				chunkData, chunkSize, buffer2, buffer2ContentSize);
+			zresult = ZSTD_DCtx_refPrefix_advanced(self->dctx,
+				buffer2, buffer2ContentSize, ZSTD_dct_rawContent);
+			Py_END_ALLOW_THREADS
+			if (ZSTD_isError(zresult)) {
+				PyErr_Format(ZstdError,
+					"failed to load prefix dictionary at chunk %zd", chunkIndex);
+				goto finally;
+			}
+
+			outBuffer.dst = buffer1;
+			outBuffer.size = buffer1Size;
+			outBuffer.pos = 0;
+
+			Py_BEGIN_ALLOW_THREADS
+			zresult = ZSTD_decompress_generic(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
 					chunkIndex, ZSTD_getErrorName(zresult));
 				goto finally;
 			}
-			buffer1ContentSize = zresult;
+			else if (zresult) {
+				PyErr_Format(ZstdError, "chunk %zd did not decompress full frame",
+					chunkIndex);
+				goto finally;
+			}
+
+			buffer1ContentSize = outBuffer.pos;
 		}
 	}
 
@@ -762,17 +916,13 b' finally:'
 		PyMem_Free(buffer1);
 	}
 
-	if (dctx) {
-		ZSTD_freeDCtx(dctx);
-	}
-
 	return result;
 }
 
 typedef struct {
 	void* sourceData;
 	size_t sourceSize;
-	unsigned long long destSize;
+	size_t destSize;
 } FramePointer;
 
 typedef struct {
@@ -806,7 +956,6 b' typedef struct {'
 
 	/* Compression state and settings. */
 	ZSTD_DCtx* dctx;
-	ZSTD_DDict* ddict;
 	int requireOutputSizes;
 
 	/* Output storage. */
@@ -838,6 +987,14 b' static void decompress_worker(WorkerStat'
 	assert(0 == state->destCount);
 	assert(state->endOffset - state->startOffset >= 0);
 
+	/* We could get here due to the way work is allocated. Ideally we wouldn't
+	   get here. But that would require a bit of a refactor in the caller. */
+	if (state->totalSourceSize > SIZE_MAX) {
+		state->error = WorkerError_memory;
+		state->errorOffset = 0;
+		return;
+	}
+
 	/*
 	 * We need to allocate a buffer to hold decompressed data. How we do this
 	 * depends on what we know about the output. The following scenarios are
@@ -853,14 +1010,34 b' static void decompress_worker(WorkerStat'
 	/* Resolve ouput segments. */
 	for (frameIndex = state->startOffset; frameIndex <= state->endOffset; frameIndex++) {
 		FramePointer* fp = &framePointers[frameIndex];
+		unsigned long long decompressedSize;
 
 		if (0 == fp->destSize) {
-			fp->destSize = ZSTD_getDecompressedSize(fp->sourceData, fp->sourceSize);
-			if (0 == fp->destSize && state->requireOutputSizes) {
+			decompressedSize = ZSTD_getFrameContentSize(fp->sourceData, fp->sourceSize);
+
+			if (ZSTD_CONTENTSIZE_ERROR == decompressedSize) {
 				state->error = WorkerError_unknownSize;
 				state->errorOffset = frameIndex;
 				return;
 			}
+			else if (ZSTD_CONTENTSIZE_UNKNOWN == decompressedSize) {
+				if (state->requireOutputSizes) {
+					state->error = WorkerError_unknownSize;
+					state->errorOffset = frameIndex;
+					return;
+				}
+
+				/* This will fail the assert for .destSize > 0 below. */
+				decompressedSize = 0;
+			}
+
+			if (decompressedSize > SIZE_MAX) {
+				state->error = WorkerError_memory;
+				state->errorOffset = frameIndex;
+				return;
+			}
+
+			fp->destSize = (size_t)decompressedSize;
 		}
 
 		totalOutputSize += fp->destSize;
@@ -878,7 +1055,7 b' static void decompress_worker(WorkerStat'
 
 	assert(framePointers[state->startOffset].destSize > 0); /* For now. */
 
-	allocationSize = roundpow2(state->totalSourceSize);
+	allocationSize = roundpow2((size_t)state->totalSourceSize);
 
 	if (framePointers[state->startOffset].destSize > allocationSize) {
 		allocationSize = roundpow2(framePointers[state->startOffset].destSize);
@@ -902,6 +1079,8 b' static void decompress_worker(WorkerStat'
 	destBuffer->segmentsSize = remainingItems;
 
 	for (frameIndex = state->startOffset; frameIndex <= state->endOffset; frameIndex++) {
+		ZSTD_outBuffer outBuffer;
+		ZSTD_inBuffer inBuffer;
 		const void* source = framePointers[frameIndex].sourceData;
 		const size_t sourceSize = framePointers[frameIndex].sourceSize;
 		void* dest;
@@ -956,7 +1135,7 b' static void decompress_worker(WorkerStat'
 			/* Don't take any chances will non-NULL pointers. */
 			memset(destBuffer, 0, sizeof(DestBuffer));
 
-			allocationSize = roundpow2(state->totalSourceSize);
+			allocationSize = roundpow2((size_t)state->totalSourceSize);
 
 			if (decompressedSize > allocationSize) {
 				allocationSize = roundpow2(decompressedSize);
@@ -985,31 +1164,31 b' static void decompress_worker(WorkerStat'
 
 		dest = (char*)destBuffer->dest + destOffset;
 
-		if (state->ddict) {
-			zresult = ZSTD_decompress_usingDDict(state->dctx, dest, decompressedSize,
-				source, sourceSize, state->ddict);
-		}
-		else {
-			zresult = ZSTD_decompressDCtx(state->dctx, dest, decompressedSize,
-				source, sourceSize);
-		}
+		outBuffer.dst = dest;
+		outBuffer.size = decompressedSize;
+		outBuffer.pos = 0;
 
+		inBuffer.src = source;
+		inBuffer.size = sourceSize;
+		inBuffer.pos = 0;
+
+		zresult = ZSTD_decompress_generic(state->dctx, &outBuffer, &inBuffer);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;
 			state->errorOffset = frameIndex;
 			return;
 		}
-		else if (zresult != decompressedSize) {
+		else if (zresult || outBuffer.pos != decompressedSize) {
 			state->error = WorkerError_sizeMismatch;
-			state->zresult = zresult;
+			state->zresult = outBuffer.pos;
 			state->errorOffset = frameIndex;
 			return;
 		}
 
 		destBuffer->segments[localOffset].offset = destOffset;
-		destBuffer->segments[localOffset].length = decompressedSize;
-		destOffset += zresult;
+		destBuffer->segments[localOffset].length = outBuffer.pos;
+		destOffset += outBuffer.pos;
 		localOffset++;
 		remainingItems--;
 	}
@@ -1027,9 +1206,7 b' static void decompress_worker(WorkerStat'
 }
 
 ZstdBufferWithSegmentsCollection* decompress_from_framesources(ZstdDecompressor* decompressor, FrameSources* frames,
-	unsigned int threadCount) {
-	void* dictData = NULL;
-	size_t dictSize = 0;
+	Py_ssize_t threadCount) {
 	Py_ssize_t i = 0;
 	int errored = 0;
 	Py_ssize_t segmentsCount;
@@ -1039,7 +1216,7 b' ZstdBufferWithSegmentsCollection* decomp'
 	ZstdBufferWithSegmentsCollection* result = NULL;
 	FramePointer* framePointers = frames->frames;
 	unsigned long long workerBytes = 0;
-	int currentThread = 0;
+	Py_ssize_t currentThread = 0;
 	Py_ssize_t workerStartOffset = 0;
 	POOL_ctx* pool = NULL;
 	WorkerState* workerStates = NULL;
@@ -1049,24 +1226,14 b' ZstdBufferWithSegmentsCollection* decomp'
 	assert(threadCount >= 1);
 
 	/* More threads than inputs makes no sense under any conditions. */
-	threadCount = frames->framesSize < threadCount ? (unsigned int)frames->framesSize
+	threadCount = frames->framesSize < threadCount ? frames->framesSize
 												   : threadCount;
 
 	/* TODO lower thread count if input size is too small and threads would just
 	   add overhead. */
 
 	if (decompressor->dict) {
-		dictData = decompressor->dict->dictData;
-		dictSize = decompressor->dict->dictSize;
-	}
-
-	if (dictData && !decompressor->ddict) {
-		Py_BEGIN_ALLOW_THREADS
-		decompressor->ddict = ZSTD_createDDict_byReference(dictData, dictSize);
-		Py_END_ALLOW_THREADS
-
-		if (!decompressor->ddict) {
-			PyErr_SetString(ZstdError, "could not create decompression dict");
+		if (ensure_ddict(decompressor->dict)) {
 			return NULL;
 		}
 	}
@@ -1091,7 +1258,14 b' ZstdBufferWithSegmentsCollection* decomp'
 
 	bytesPerWorker = frames->compressedSize / threadCount;
 
+	if (bytesPerWorker > SIZE_MAX) {
+		PyErr_SetString(ZstdError, "too much data per worker for this platform");
+		goto finally;
+	}
+
 	for (i = 0; i < threadCount; i++) {
+		size_t zresult;
+
 		workerStates[i].dctx = ZSTD_createDCtx();
 		if (NULL == workerStates[i].dctx) {
 			PyErr_NoMemory();
@@ -1100,7 +1274,15 b' ZstdBufferWithSegmentsCollection* decomp'
 
 		ZSTD_copyDCtx(workerStates[i].dctx, decompressor->dctx);
 
-		workerStates[i].ddict = decompressor->ddict;
+		if (decompressor->dict) {
+			zresult = ZSTD_DCtx_refDDict(workerStates[i].dctx, decompressor->dict->ddict);
+			if (zresult) {
+				PyErr_Format(ZstdError, "unable to reference prepared dictionary: %s",
+					ZSTD_getErrorName(zresult));
+				goto finally;
+			}
+		}
+
 		workerStates[i].framePointers = framePointers;
 		workerStates[i].requireOutputSizes = 1;
 	}
@@ -1178,7 +1360,7 b' ZstdBufferWithSegmentsCollection* decomp'
 			break;
 
 		case WorkerError_sizeMismatch:
-			PyErr_Format(ZstdError, "error decompressing item %zd: decompressed %zu bytes; expected %llu",
+			PyErr_Format(ZstdError, "error decompressing item %zd: decompressed %zu bytes; expected %zu",
 				workerStates[i].errorOffset, workerStates[i].zresult,
 				framePointers[workerStates[i].errorOffset].destSize);
 			errored = 1;
@@ -1388,9 +1570,21 b' static ZstdBufferWithSegmentsCollection*'
 				decompressedSize = frameSizesP[i];
 			}
 
+			if (sourceSize > SIZE_MAX) {
+				PyErr_Format(PyExc_ValueError,
+					"item %zd is too large for this platform", i);
+				goto finally;
+			}
+
+			if (decompressedSize > SIZE_MAX) {
+				PyErr_Format(PyExc_ValueError,
+					"decompressed size of item %zd is too large for this platform", i);
+				goto finally;
+			}
+
 			framePointers[i].sourceData = sourceData;
-			framePointers[i].sourceSize = sourceSize;
-			framePointers[i].destSize = decompressedSize;
+			framePointers[i].sourceSize = (size_t)sourceSize;
+			framePointers[i].destSize = (size_t)decompressedSize;
 		}
 	}
 	else if (PyObject_TypeCheck(frames, &ZstdBufferWithSegmentsCollectionType)) {
@@ -1419,17 +1613,33 b' static ZstdBufferWithSegmentsCollection*'
 			buffer = collection->buffers[i];
 
 			for (segmentIndex = 0; segmentIndex < buffer->segmentCount; segmentIndex++) {
+				unsigned long long decompressedSize = frameSizesP ? frameSizesP[offset] : 0;
+
 				if (buffer->segments[segmentIndex].offset + buffer->segments[segmentIndex].length > buffer->dataSize) {
 					PyErr_Format(PyExc_ValueError, "item %zd has offset outside memory area",
 						offset);
 					goto finally;
 				}
 
+				if (buffer->segments[segmentIndex].length > SIZE_MAX) {
+					PyErr_Format(PyExc_ValueError,
+						"item %zd in buffer %zd is too large for this platform",
+						segmentIndex, i);
+					goto finally;
+				}
+
+				if (decompressedSize > SIZE_MAX) {
+					PyErr_Format(PyExc_ValueError,
+						"decompressed size of item %zd in buffer %zd is too large for this platform",
+						segmentIndex, i);
+					goto finally;
+				}
+
 				totalInputSize += buffer->segments[segmentIndex].length;
 
 				framePointers[offset].sourceData = (char*)buffer->data + buffer->segments[segmentIndex].offset;
-				framePointers[offset].sourceSize = buffer->segments[segmentIndex].length;
-				framePointers[offset].destSize = frameSizesP ? frameSizesP[offset] : 0;
+				framePointers[offset].sourceSize = (size_t)buffer->segments[segmentIndex].length;
+				framePointers[offset].destSize = (size_t)decompressedSize;
 
 				offset++;
 			}
@@ -1450,11 +1660,6 b' static ZstdBufferWithSegmentsCollection*'
 			goto finally;
 		}
 
-		/*
-		 * It is not clear whether Py_buffer.buf is still valid after
-		 * PyBuffer_Release. So, we hold a reference to all Py_buffer instances
-		 * for the duration of the operation.
-		 */
 		frameBuffers = PyMem_Malloc(frameCount * sizeof(Py_buffer));
 		if (NULL == frameBuffers) {
 			PyErr_NoMemory();
@@ -1465,6 +1670,8 b' static ZstdBufferWithSegmentsCollection*'
 
 		/* Do a pass to assemble info about our input buffers and output sizes. */
 		for (i = 0; i < frameCount; i++) {
+			unsigned long long decompressedSize = frameSizesP ? frameSizesP[i] : 0;
+
 			if (0 != PyObject_GetBuffer(PyList_GET_ITEM(frames, i),
 				&frameBuffers[i], PyBUF_CONTIG_RO)) {
 				PyErr_Clear();
@@ -1472,11 +1679,17 b' static ZstdBufferWithSegmentsCollection*'
 				goto finally;
 			}
 
+			if (decompressedSize > SIZE_MAX) {
+				PyErr_Format(PyExc_ValueError,
+					"decompressed size of item %zd is too large for this platform", i);
+				goto finally;
+			}
+
 			totalInputSize += frameBuffers[i].len;
 
 			framePointers[i].sourceData = frameBuffers[i].buf;
 			framePointers[i].sourceSize = frameBuffers[i].len;
-			framePointers[i].destSize = frameSizesP ? frameSizesP[i] : 0;
+			framePointers[i].destSize = (size_t)decompressedSize;
 		}
 	}
 	else {
@@ -1514,16 +1727,26 b' static PyMethodDef Decompressor_methods['
 	Decompressor_copy_stream__doc__ },
 	{ "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_decompress__doc__ },
-	{ "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_NOARGS,
+	{ "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_decompressobj__doc__ },
-	{ "read_from", (PyCFunction)Decompressor_read_from, METH_VARARGS | METH_KEYWORDS,
-	Decompressor_read_from__doc__ },
-	{ "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS,
-	Decompressor_write_to__doc__ },
+	{ "read_to_iter", (PyCFunction)Decompressor_read_to_iter, METH_VARARGS | METH_KEYWORDS,
+	Decompressor_read_to_iter__doc__ },
+	/* TODO Remove deprecated API */
+	{ "read_from", (PyCFunction)Decompressor_read_to_iter, METH_VARARGS | METH_KEYWORDS,
+	Decompressor_read_to_iter__doc__ },
+	{ "stream_reader", (PyCFunction)Decompressor_stream_reader,
+	METH_VARARGS | METH_KEYWORDS, Decompressor_stream_reader__doc__ },
+	{ "stream_writer", (PyCFunction)Decompressor_stream_writer, METH_VARARGS | METH_KEYWORDS,
+	Decompressor_stream_writer__doc__ },
+	/* TODO remove deprecated API */
+	{ "write_to", (PyCFunction)Decompressor_stream_writer, METH_VARARGS | METH_KEYWORDS,
+	Decompressor_stream_writer__doc__ },
 	{ "decompress_content_dict_chain", (PyCFunction)Decompressor_decompress_content_dict_chain,
 	  METH_VARARGS | METH_KEYWORDS, Decompressor_decompress_content_dict_chain__doc__ },
 	{ "multi_decompress_to_buffer", (PyCFunction)Decompressor_multi_decompress_to_buffer,
 	  METH_VARARGS | METH_KEYWORDS, Decompressor_multi_decompress_to_buffer__doc__ },
+	{ "memory_size", (PyCFunction)Decompressor_memory_size, METH_NOARGS,
+	Decompressor_memory_size__doc__ },
 	{ NULL, NULL }
 };
 

contrib/python-zstandard/c-ext/decompressoriterator.c

@@ -20,10 +20,9 b' static void ZstdDecompressorIterator_dea'
 	Py_XDECREF(self->decompressor);
 	Py_XDECREF(self->reader);
 
-	if (self->buffer) {
-		PyBuffer_Release(self->buffer);
-		PyMem_FREE(self->buffer);
-		self->buffer = NULL;
+	if (self->buffer.buf) {
+		PyBuffer_Release(&self->buffer);
+		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	if (self->input.src) {
@@ -45,8 +44,6 b' static DecompressorIteratorResult read_d'
 	DecompressorIteratorResult result;
 	size_t oldInputPos = self->input.pos;
 
-	assert(self->decompressor->dstream);
-
 	result.chunk = NULL;
 
 	chunk = PyBytes_FromStringAndSize(NULL, self->outSize);
@@ -60,7 +57,7 b' static DecompressorIteratorResult read_d'
 	self->output.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
-	zresult = ZSTD_decompressStream(self->decompressor->dstream, &self->output, &self->input);
+	zresult = ZSTD_decompress_generic(self->decompressor->dctx, &self->output, &self->input);
 	Py_END_ALLOW_THREADS
 
 	/* We're done with the pointer. Nullify to prevent anyone from getting a
@@ -86,7 +83,8 b' static DecompressorIteratorResult read_d'
 	/* If it produced output data, return it. */
 	if (self->output.pos) {
 		if (self->output.pos < self->outSize) {
-			if (_PyBytes_Resize(&chunk, self->output.pos)) {
+			if (safe_pybytes_resize(&chunk, self->output.pos)) {
+				Py_XDECREF(chunk);
 				result.errored = 1;
 				return result;
 			}
@@ -137,15 +135,15 b' read_from_source:'
 			PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 		}
 		else {
-			assert(self->buffer && self->buffer->buf);
+			assert(self->buffer.buf);
 
 			/* Only support contiguous C arrays for now */
-			assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL);
-			assert(self->buffer->itemsize == 1);
+			assert(self->buffer.strides == NULL && self->buffer.suboffsets == NULL);
+			assert(self->buffer.itemsize == 1);
 
 			/* TODO avoid memcpy() below */
-			readBuffer = (char *)self->buffer->buf + self->bufferOffset;
-			bufferRemaining = self->buffer->len - self->bufferOffset;
+			readBuffer = (char *)self->buffer.buf + self->bufferOffset;
+			bufferRemaining = self->buffer.len - self->bufferOffset;
 			readSize = min(bufferRemaining, (Py_ssize_t)self->inSize);
 			self->bufferOffset += readSize;
 		}

contrib/python-zstandard/c-ext/frameparams.c

@@ -13,50 +13,56 b' extern PyObject* ZstdError;'
 PyDoc_STRVAR(FrameParameters__doc__,
 	"FrameParameters: information about a zstd frame");
 
-FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args) {
-	const char* source;
-	Py_ssize_t sourceSize;
-	ZSTD_frameParams params;
+FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"data",
+		NULL
+	};
+
+	Py_buffer source;
+	ZSTD_frameHeader header;
 	FrameParametersObject* result = NULL;
 	size_t zresult;
 
 #if PY_MAJOR_VERSION >= 3
-	if (!PyArg_ParseTuple(args, "y#:get_frame_parameters",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:get_frame_parameters",
 #else
-	if (!PyArg_ParseTuple(args, "s#:get_frame_parameters",
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:get_frame_parameters",
 #endif
-		&source, &sourceSize)) {
+		kwlist, &source)) {
 		return NULL;
 	}
 
-	/* Needed for Python 2 to reject unicode */
-	if (!PyBytes_Check(PyTuple_GET_ITEM(args, 0))) {
-		PyErr_SetString(PyExc_TypeError, "argument must be bytes");
-		return NULL;
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
 	}
 
-	zresult = ZSTD_getFrameParams(&params, (void*)source, sourceSize);
+	zresult = ZSTD_getFrameHeader(&header, source.buf, source.len);
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "cannot get frame parameters: %s", ZSTD_getErrorName(zresult));
-		return NULL;
+		goto finally;
 	}
 
 	if (zresult) {
 		PyErr_Format(ZstdError, "not enough data for frame parameters; need %zu bytes", zresult);
-		return NULL;
+		goto finally;
 	}
 
 	result = PyObject_New(FrameParametersObject, &FrameParametersType);
 	if (!result) {
-		return NULL;
+		goto finally;
 	}
 
-	result->frameContentSize = params.frameContentSize;
-	result->windowSize = params.windowSize;
-	result->dictID = params.dictID;
-	result->checksumFlag = params.checksumFlag ? 1 : 0;
+	result->frameContentSize = header.frameContentSize;
+	result->windowSize = header.windowSize;
+	result->dictID = header.dictID;
+	result->checksumFlag = header.checksumFlag ? 1 : 0;
 
+finally:
+	PyBuffer_Release(&source);
 	return result;
 }
 
@@ -68,7 +74,7 b' static PyMemberDef FrameParameters_membe'
 	{ "content_size", T_ULONGLONG,
 	  offsetof(FrameParametersObject, frameContentSize), READONLY,
 	  "frame content size" },
-	{ "window_size", T_UINT,
+	{ "window_size", T_ULONGLONG,
 	  offsetof(FrameParametersObject, windowSize), READONLY,
 	  "window size" },
 	{ "dict_id", T_UINT,

contrib/python-zstandard/c-ext/python-zstandard.h

@@ -12,12 +12,10 b''
 
 #define ZSTD_STATIC_LINKING_ONLY
 #define ZDICT_STATIC_LINKING_ONLY
-#include "mem.h"
-#include "zstd.h"
-#include "zdict.h"
-#include "zstdmt_compress.h"
+#include <zstd.h>
+#include <zdict.h>
 
-#define PYTHON_ZSTANDARD_VERSION "0.8.1"
+#define PYTHON_ZSTANDARD_VERSION "0.9.0"
 
 typedef enum {
 	compressorobj_flush_finish,
@@ -25,22 +23,38 b' typedef enum {'
 } CompressorObj_Flush;
 
 /*
-   Represents a CompressionParameters type.
+   Represents a ZstdCompressionParameters type.
 
-   This type is basically a wrapper around ZSTD_compressionParameters.
+   This type holds all the low-level compression parameters that can be set.
 */
 typedef struct {
 	PyObject_HEAD
+	ZSTD_CCtx_params* params;
+	unsigned format;
+	int compressionLevel;
 	unsigned windowLog;
+	unsigned hashLog;
 	unsigned chainLog;
-	unsigned hashLog;
 	unsigned searchLog;
-	unsigned searchLength;
+	unsigned minMatch;
 	unsigned targetLength;
-	ZSTD_strategy strategy;
-} CompressionParametersObject;
+	unsigned compressionStrategy;
+	unsigned contentSizeFlag;
+	unsigned checksumFlag;
+	unsigned dictIDFlag;
+	unsigned threads;
+	unsigned jobSize;
+	unsigned overlapSizeLog;
+	unsigned compressLiterals;
+	unsigned forceMaxWindow;
+	unsigned enableLongDistanceMatching;
+	unsigned ldmHashLog;
+	unsigned ldmMinMatch;
+	unsigned ldmBucketSizeLog;
+	unsigned ldmHashEveryLog;
+} ZstdCompressionParametersObject;
 
-extern PyTypeObject CompressionParametersType;
+extern PyTypeObject ZstdCompressionParametersType;
 
 /*
    Represents a FrameParameters type.
@@ -50,7 +64,7 b' extern PyTypeObject CompressionParameter'
 typedef struct {
 	PyObject_HEAD
 	unsigned long long frameContentSize;
-	unsigned windowSize;
+	unsigned long long windowSize;
 	unsigned dictID;
 	char checksumFlag;
 } FrameParametersObject;
@@ -69,10 +83,14 b' typedef struct {'
 	void* dictData;
 	/* Size of dictionary data. */
 	size_t dictSize;
+	ZSTD_dictContentType_e dictType;
 	/* k parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned k;
 	/* d parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned d;
+	/* Digested dictionary, suitable for reuse. */
+	ZSTD_CDict* cdict;
+	ZSTD_DDict* ddict;
 } ZstdCompressionDict;
 
 extern PyTypeObject ZstdCompressionDictType;
@@ -83,29 +101,15 b' extern PyTypeObject ZstdCompressionDictT'
 typedef struct {
 	PyObject_HEAD
 
-	/* Configured compression level. Should be always set. */
-	int compressionLevel;
 	/* Number of threads to use for operations. */
 	unsigned int threads;
 	/* Pointer to compression dictionary to use. NULL if not using dictionary
 	   compression. */
 	ZstdCompressionDict* dict;
-	/* Compression context to use. Populated during object construction. NULL
-	   if using multi-threaded compression. */
+	/* Compression context to use. Populated during object construction. */
 	ZSTD_CCtx* cctx;
-	/* Multi-threaded compression context to use. Populated during object
-	   construction. NULL if not using multi-threaded compression. */
-	ZSTDMT_CCtx* mtcctx;
-	/* Digest compression dictionary. NULL initially. Populated on first use. */
-	ZSTD_CDict* cdict;
-	/* Low-level compression parameter control. NULL unless passed to
-	   constructor. Takes precedence over `compressionLevel` if defined. */
-	CompressionParametersObject* cparams;
-	/* Controls zstd frame options. */
-	ZSTD_frameParameters fparams;
-	/* Holds state for streaming compression. Shared across all invocation.
-	   Populated on first use. */
-	ZSTD_CStream* cstream;
+	/* Compression parameters in use. */
+	ZSTD_CCtx_params* params;
 } ZstdCompressor;
 
 extern PyTypeObject ZstdCompressorType;
@@ -125,9 +129,10 b' typedef struct {'
 
 	ZstdCompressor* compressor;
 	PyObject* writer;
-	Py_ssize_t sourceSize;
+	unsigned long long sourceSize;
 	size_t outSize;
 	int entered;
+	unsigned long long bytesCompressed;
 } ZstdCompressionWriter;
 
 extern PyTypeObject ZstdCompressionWriterType;
@@ -137,9 +142,8 b' typedef struct {'
 
 	ZstdCompressor* compressor;
 	PyObject* reader;
-	Py_buffer* buffer;
+	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
-	Py_ssize_t sourceSize;
 	size_t inSize;
 	size_t outSize;
 
@@ -155,11 +159,32 b' extern PyTypeObject ZstdCompressorIterat'
 typedef struct {
 	PyObject_HEAD
 
+	ZstdCompressor* compressor;
+	PyObject* reader;
+	Py_buffer buffer;
+	unsigned long long sourceSize;
+	size_t readSize;
+
+	int entered;
+	int closed;
+	unsigned long long bytesCompressed;
+
+	ZSTD_inBuffer input;
+	ZSTD_outBuffer output;
+	int finishedInput;
+	int finishedOutput;
+	PyObject* readResult;
+} ZstdCompressionReader;
+
+extern PyTypeObject ZstdCompressionReaderType;
+
+typedef struct {
+	PyObject_HEAD
+
 	ZSTD_DCtx* dctx;
-
 	ZstdCompressionDict* dict;
-	ZSTD_DDict* ddict;
-	ZSTD_DStream* dstream;
+	size_t maxWindowSize;
+	ZSTD_format_e format;
 } ZstdDecompressor;
 
 extern PyTypeObject ZstdDecompressorType;
@@ -168,6 +193,7 b' typedef struct {'
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
+	size_t outSize;
 	int finished;
 } ZstdDecompressionObj;
 
@@ -176,6 +202,40 b' extern PyTypeObject ZstdDecompressionObj'
 typedef struct {
 	PyObject_HEAD
 
+	/* Parent decompressor to which this object is associated. */
+	ZstdDecompressor* decompressor;
+	/* Object to read() from (if reading from a stream). */
+	PyObject* reader;
+	/* Size for read() operations on reader. */
+	size_t readSize;
+	/* Buffer to read from (if reading from a buffer). */
+	Py_buffer buffer;
+
+	/* Whether the context manager is active. */
+	int entered;
+	/* Whether we've closed the stream. */
+	int closed;
+
+	/* Number of bytes decompressed and returned to user. */
+	unsigned long long bytesDecompressed;
+
+	/* Tracks data going into decompressor. */
+	ZSTD_inBuffer input;
+
+	/* Holds output from read() operation on reader. */
+	PyObject* readResult;
+
+	/* Whether all input has been sent to the decompressor. */
+	int finishedInput;
+	/* Whether all output has been flushed from the decompressor. */
+	int finishedOutput;
+} ZstdDecompressionReader;
+
+extern PyTypeObject ZstdDecompressionReaderType;
+
+typedef struct {
+	PyObject_HEAD
+
 	ZstdDecompressor* decompressor;
 	PyObject* writer;
 	size_t outSize;
@@ -189,7 +249,7 b' typedef struct {'
 
 	ZstdDecompressor* decompressor;
 	PyObject* reader;
-	Py_buffer* buffer;
+	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
 	size_t inSize;
 	size_t outSize;
@@ -209,6 +269,9 b' typedef struct {'
 } DecompressorIteratorResult;
 
 typedef struct {
+	/* The public API is that these are 64-bit unsigned integers. So these can't
+	 * be size_t, even though values larger than SIZE_MAX or PY_SSIZE_T_MAX may 
+	 * be nonsensical for this platform. */
 	unsigned long long offset;
 	unsigned long long length;
 } BufferSegment;
@@ -270,16 +333,14 b' typedef struct {'
 
 extern PyTypeObject ZstdBufferWithSegmentsCollectionType;
 
-void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams);
-CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args);
-FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args);
-PyObject* estimate_compression_context_size(PyObject* self, PyObject* args);
-int init_cstream(ZstdCompressor* compressor, unsigned long long sourceSize);
-int init_mtcstream(ZstdCompressor* compressor, Py_ssize_t sourceSize);
-int init_dstream(ZstdDecompressor* decompressor);
+int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
+int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj);
+FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs);
+int ensure_ddict(ZstdCompressionDict* dict);
+int ensure_dctx(ZstdDecompressor* decompressor, int loadDict);
 ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs);
-ZstdCompressionDict* train_cover_dictionary(PyObject* self, PyObject* args, PyObject* kwargs);
 ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, BufferSegment* segments, Py_ssize_t segmentsSize);
 Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection*);
 int cpu_count(void);
 size_t roundpow2(size_t);
+int safe_pybytes_resize(PyObject** obj, Py_ssize_t size);

contrib/python-zstandard/make_cffi.py

@@ -27,6 +27,11 b" SOURCES = ['zstd/%s' % p for p in ("
     'compress/fse_compress.c',
     'compress/huf_compress.c',
     'compress/zstd_compress.c',
+    'compress/zstd_double_fast.c',
+    'compress/zstd_fast.c',
+    'compress/zstd_lazy.c',
+    'compress/zstd_ldm.c',
+    'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
     'decompress/zstd_decompress.c',
@@ -38,7 +43,6 b" SOURCES = ['zstd/%s' % p for p in ("
 # Headers whose preprocessed output will be fed into cdef().
 HEADERS = [os.path.join(HERE, 'zstd', *p) for p in (
     ('zstd.h',),
-    ('compress', 'zstdmt_compress.h'),
     ('dictBuilder', 'zdict.h'),
 )]
 
@@ -80,7 +84,9 b' else:'
 def preprocess(path):
     with open(path, 'rb') as fh:
         lines = []
-        for l in fh:
+        it = iter(fh)
+
+        for l in it:
             # zstd.h includes <stddef.h>, which is also included by cffi's
             # boilerplate. This can lead to duplicate declarations. So we strip
             # this include from the preprocessor invocation.
@@ -137,18 +143,21 b' def normalize_output(output):'
 
 
 ffi = cffi.FFI()
+# zstd.h uses a possible undefined MIN(). Define it until
+# https://github.com/facebook/zstd/issues/976 is fixed.
 # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
 # when cffi uses the function. Since we statically link against zstd, even
 # if we use the deprecated functions it shouldn't be a huge problem.
 ffi.set_source('_zstd_cffi', '''
-#include "mem.h"
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define ZSTD_STATIC_LINKING_ONLY
-#include "zstd.h"
+#include <zstd.h>
 #define ZDICT_STATIC_LINKING_ONLY
 #define ZDICT_DISABLE_DEPRECATE_WARNINGS
-#include "zdict.h"
-#include "zstdmt_compress.h"
-''', sources=SOURCES, include_dirs=INCLUDE_DIRS)
+#include <zdict.h>
+''', sources=SOURCES,
+     include_dirs=INCLUDE_DIRS,
+     extra_compile_args=['-DZSTD_MULTITHREAD'])
 
 DEFINE = re.compile(b'^\\#define ([a-zA-Z0-9_]+) ')
 

contrib/python-zstandard/setup.py

@@ -5,6 +5,7 b''
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
+import os
 import sys
 from setuptools import setup
 
@@ -16,14 +17,32 b' except ImportError:'
 import setup_zstd
 
 SUPPORT_LEGACY = False
+SYSTEM_ZSTD = False
+WARNINGS_AS_ERRORS = False
 
-if "--legacy" in sys.argv:
+if os.environ.get('ZSTD_WARNINGS_AS_ERRORS', ''):
+    WARNINGS_AS_ERRORS = True
+
+if '--legacy' in sys.argv:
     SUPPORT_LEGACY = True
-    sys.argv.remove("--legacy")
+    sys.argv.remove('--legacy')
+
+if '--system-zstd' in sys.argv:
+    SYSTEM_ZSTD = True
+    sys.argv.remove('--system-zstd')
+
+if '--warnings-as-errors' in sys.argv:
+    WARNINGS_AS_ERRORS = True
+    sys.argv.remote('--warning-as-errors')
 
 # Code for obtaining the Extension instance is in its own module to
 # facilitate reuse in other projects.
-extensions = [setup_zstd.get_c_extension(SUPPORT_LEGACY, 'zstd')]
+extensions = [
+    setup_zstd.get_c_extension(name='zstd',
+                               support_legacy=SUPPORT_LEGACY,
+                               system_zstd=SYSTEM_ZSTD,
+                               warnings_as_errors=WARNINGS_AS_ERRORS),
+]
 
 install_requires = []
 
@@ -31,8 +50,11 b' if cffi:'
     import make_cffi
     extensions.append(make_cffi.ffi.distutils_extension())
 
-    # Need change in 1.8 for ffi.from_buffer() behavior.
-    install_requires.append('cffi>=1.8')
+    # Need change in 1.10 for ffi.from_buffer() to handle all buffer types
+    # (like memoryview).
+    # Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
+    # garbage collection pitfalls.
+    install_requires.append('cffi>=1.11')
 
 version = None
 
@@ -62,14 +84,13 b' setup('
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Programming Language :: C',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
     ],
     keywords='zstandard zstd compression',
+    packages=['zstandard'],
     ext_modules=extensions,
     test_suite='tests',
     install_requires=install_requires,

contrib/python-zstandard/setup_zstd.py

@@ -4,7 +4,10 b''
 # This software may be modified and distributed under the terms
 # of the BSD license. See the LICENSE file for details.
 
+import distutils.ccompiler
 import os
+import sys
+
 from distutils.extension import Extension
 
 
@@ -19,6 +22,11 b" zstd_sources = ['zstd/%s' % p for p in ("
     'compress/fse_compress.c',
     'compress/huf_compress.c',
     'compress/zstd_compress.c',
+    'compress/zstd_double_fast.c',
+    'compress/zstd_fast.c',
+    'compress/zstd_lazy.c',
+    'compress/zstd_ldm.c',
+    'compress/zstd_opt.c',
     'compress/zstdmt_compress.c',
     'decompress/huf_decompress.c',
     'decompress/zstd_decompress.c',
@@ -41,7 +49,6 b" zstd_sources_legacy = ['zstd/%s' % p for"
 )]
 
 zstd_includes = [
-    'c-ext',
     'zstd',
     'zstd/common',
     'zstd/compress',
@@ -54,7 +61,14 b' zstd_includes_legacy = ['
     'zstd/legacy',
 ]
 
+ext_includes = [
+    'c-ext',
+    'zstd/common',
+]
+
 ext_sources = [
+    'zstd/common/pool.c',
+    'zstd/common/threading.c',
     'zstd.c',
     'c-ext/bufferutil.c',
     'c-ext/compressiondict.c',
@@ -62,11 +76,13 b' ext_sources = ['
     'c-ext/compressor.c',
     'c-ext/compressoriterator.c',
     'c-ext/compressionparams.c',
+    'c-ext/compressionreader.c',
     'c-ext/compressionwriter.c',
     'c-ext/constants.c',
     'c-ext/decompressobj.c',
     'c-ext/decompressor.c',
     'c-ext/decompressoriterator.c',
+    'c-ext/decompressionreader.c',
     'c-ext/decompressionwriter.c',
     'c-ext/frameparams.c',
 ]
@@ -76,27 +92,67 b' zstd_depends = ['
 ]
 
 
-def get_c_extension(support_legacy=False, name='zstd'):
+def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
+                    warnings_as_errors=False):
     """Obtain a distutils.extension.Extension for the C extension."""
     root = os.path.abspath(os.path.dirname(__file__))
 
-    sources = [os.path.join(root, p) for p in zstd_sources + ext_sources]
-    if support_legacy:
-        sources.extend([os.path.join(root, p) for p in zstd_sources_legacy])
+    sources = set([os.path.join(root, p) for p in ext_sources])
+    if not system_zstd:
+        sources.update([os.path.join(root, p) for p in zstd_sources])
+        if support_legacy:
+            sources.update([os.path.join(root, p) for p in zstd_sources_legacy])
+    sources = list(sources)
 
-    include_dirs = [os.path.join(root, d) for d in zstd_includes]
-    if support_legacy:
-        include_dirs.extend([os.path.join(root, d) for d in zstd_includes_legacy])
+    include_dirs = set([os.path.join(root, d) for d in ext_includes])
+    if not system_zstd:
+        include_dirs.update([os.path.join(root, d) for d in zstd_includes])
+        if support_legacy:
+            include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy])
+    include_dirs = list(include_dirs)
 
     depends = [os.path.join(root, p) for p in zstd_depends]
 
+    compiler = distutils.ccompiler.new_compiler()
+
+    # Needed for MSVC.
+    if hasattr(compiler, 'initialize'):
+        compiler.initialize()
+
+    if compiler.compiler_type == 'unix':
+        compiler_type = 'unix'
+    elif compiler.compiler_type == 'msvc':
+        compiler_type = 'msvc'
+    else:
+        raise Exception('unhandled compiler type: %s' %
+                        compiler.compiler_type)
+
     extra_args = ['-DZSTD_MULTITHREAD']
 
-    if support_legacy:
+    if not system_zstd:
+        extra_args.append('-DZSTDLIB_VISIBILITY=')
+        extra_args.append('-DZDICTLIB_VISIBILITY=')
+        extra_args.append('-DZSTDERRORLIB_VISIBILITY=')
+
+        if compiler_type == 'unix':
+            extra_args.append('-fvisibility=hidden')
+
+    if not system_zstd and support_legacy:
         extra_args.append('-DZSTD_LEGACY_SUPPORT=1')
 
+    if warnings_as_errors:
+        if compiler_type == 'unix':
+            extra_args.append('-Werror')
+        elif compiler_type == 'msvc':
+            extra_args.append('/WX')
+        else:
+            assert False
+
+    libraries = ['zstd'] if system_zstd else []
+
     # TODO compile with optimizations.
     return Extension(name, sources,
                      include_dirs=include_dirs,
                      depends=depends,
-                     extra_compile_args=extra_args)
+                     extra_compile_args=extra_args,
+                     libraries=libraries)

contrib/python-zstandard/tests/common.py

@@ -1,16 +1,48 b''
+import imp
 import inspect
 import io
 import os
 import types
 
+try:
+    import hypothesis
+except ImportError:
+    hypothesis = None
+
 
 def make_cffi(cls):
     """Decorator to add CFFI versions of each test method."""
 
+    # The module containing this class definition should
+    # `import zstandard as zstd`. Otherwise things may blow up.
+    mod = inspect.getmodule(cls)
+    if not hasattr(mod, 'zstd'):
+        raise Exception('test module does not contain "zstd" symbol')
+
+    if not hasattr(mod.zstd, 'backend'):
+        raise Exception('zstd symbol does not have "backend" attribute; did '
+                        'you `import zstandard as zstd`?')
+
+    # If `import zstandard` already chose the cffi backend, there is nothing
+    # for us to do: we only add the cffi variation if the default backend
+    # is the C extension.
+    if mod.zstd.backend == 'cffi':
+        return cls
+
+    old_env = dict(os.environ)
+    os.environ['PYTHON_ZSTANDARD_IMPORT_POLICY'] = 'cffi'
     try:
-        import zstd_cffi
-    except ImportError:
-        return cls
+        try:
+            mod_info = imp.find_module('zstandard')
+            mod = imp.load_module('zstandard_cffi', *mod_info)
+        except ImportError:
+            return cls
+    finally:
+        os.environ.clear()
+        os.environ.update(old_env)
+
+    if mod.backend != 'cffi':
+        raise Exception('got the zstandard %s backend instead of cffi' % mod.backend)
 
     # If CFFI version is available, dynamically construct test methods
     # that use it.
@@ -29,13 +61,13 b' def make_cffi(cls):'
         # the function object and install it in a new attribute.
         if isinstance(fn, types.FunctionType):
             globs = dict(fn.__globals__)
-            globs['zstd'] = zstd_cffi
+            globs['zstd'] = mod
             new_fn = types.FunctionType(fn.__code__, globs, name,
                                         fn.__defaults__, fn.__closure__)
             new_method = new_fn
         else:
             globs = dict(fn.__func__.func_globals)
-            globs['zstd'] = zstd_cffi
+            globs['zstd'] = mod
             new_fn = types.FunctionType(fn.__func__.func_code, globs, name,
                                         fn.__func__.func_defaults,
                                         fn.__func__.func_closure)
@@ -86,3 +118,34 b' def random_input_data():'
                 pass
 
     return _source_files
+
+
+def generate_samples():
+    inputs = [
+        b'foo',
+        b'bar',
+        b'abcdef',
+        b'sometext',
+        b'baz',
+    ]
+
+    samples = []
+
+    for i in range(128):
+        samples.append(inputs[i % 5])
+        samples.append(inputs[i % 5] * (i + 3))
+        samples.append(inputs[-(i % 5)] * (i + 2))
+
+    return samples
+
+
+if hypothesis:
+    default_settings = hypothesis.settings()
+    hypothesis.settings.register_profile('default', default_settings)
+
+    ci_settings = hypothesis.settings(max_examples=2500,
+                                      max_iterations=2500)
+    hypothesis.settings.register_profile('ci', ci_settings)
+
+    hypothesis.settings.load_profile(
+        os.environ.get('HYPOTHESIS_PROFILE', 'default'))

contrib/python-zstandard/tests/test_buffer_util.py

@@ -1,11 +1,7 b''
 import struct
+import unittest
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-import zstd
+import zstandard as zstd
 
 ss = struct.Struct('=QQ')
 

contrib/python-zstandard/tests/test_compressor.py

@@ -2,13 +2,10 b' import hashlib'
 import io
 import struct
 import sys
+import tarfile
+import unittest
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-import zstd
+import zstandard as zstd
 
 from .common import (
     make_cffi,
@@ -23,7 +20,8 b' else:'
 
 
 def multithreaded_chunk_size(level, source_size=0):
-    params = zstd.get_compression_parameters(level, source_size)
+    params = zstd.ZstdCompressionParameters.from_level(level,
+                                                       source_size=source_size)
 
     return 1 << (params.window_log + 2)
 
@@ -32,67 +30,82 b' def multithreaded_chunk_size(level, sour'
 class TestCompressor(unittest.TestCase):
     def test_level_bounds(self):
         with self.assertRaises(ValueError):
-            zstd.ZstdCompressor(level=0)
+            zstd.ZstdCompressor(level=23)
 
-        with self.assertRaises(ValueError):
-            zstd.ZstdCompressor(level=23)
+    def test_memory_size(self):
+        cctx = zstd.ZstdCompressor(level=1)
+        self.assertGreater(cctx.memory_size(), 100)
 
 
 @make_cffi
 class TestCompressor_compress(unittest.TestCase):
-    def test_multithreaded_unsupported(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'bar' * 64)
-
-        d = zstd.train_dictionary(8192, samples)
-
-        cctx = zstd.ZstdCompressor(dict_data=d, threads=2)
-
-        with self.assertRaisesRegexp(zstd.ZstdError, 'compress\(\) cannot be used with both dictionaries and multi-threaded compression'):
-            cctx.compress(b'foo')
-
-        params = zstd.get_compression_parameters(3)
-        cctx = zstd.ZstdCompressor(compression_params=params, threads=2)
-        with self.assertRaisesRegexp(zstd.ZstdError, 'compress\(\) cannot be used with both compression parameters and multi-threaded compression'):
-            cctx.compress(b'foo')
-
     def test_compress_empty(self):
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         result = cctx.compress(b'')
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
         params = zstd.get_frame_parameters(result)
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum, 0)
 
-        # TODO should be temporary until https://github.com/facebook/zstd/issues/506
-        # is fixed.
-        cctx = zstd.ZstdCompressor(write_content_size=True)
-        with self.assertRaises(ValueError):
-            cctx.compress(b'')
+        cctx = zstd.ZstdCompressor()
+        result = cctx.compress(b'')
+        self.assertEqual(result, b'\x28\xb5\x2f\xfd\x20\x00\x01\x00\x00')
+        params = zstd.get_frame_parameters(result)
+        self.assertEqual(params.content_size, 0)
+
+    def test_input_types(self):
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
+        expected = b'\x28\xb5\x2f\xfd\x00\x00\x19\x00\x00\x66\x6f\x6f'
 
-        cctx.compress(b'', allow_empty=True)
+        mutable_array = bytearray(3)
+        mutable_array[:] = b'foo'
+
+        sources = [
+            memoryview(b'foo'),
+            bytearray(b'foo'),
+            mutable_array,
+        ]
+
+        for source in sources:
+            self.assertEqual(cctx.compress(source), expected)
 
     def test_compress_large(self):
         chunks = []
         for i in range(255):
             chunks.append(struct.Struct('>B').pack(i) * 16384)
 
-        cctx = zstd.ZstdCompressor(level=3)
+        cctx = zstd.ZstdCompressor(level=3, write_content_size=False)
         result = cctx.compress(b''.join(chunks))
         self.assertEqual(len(result), 999)
         self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
 
-        # This matches the test for read_from() below.
-        cctx = zstd.ZstdCompressor(level=1)
+        # This matches the test for read_to_iter() below.
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         result = cctx.compress(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE + b'o')
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x40\x54\x00\x00'
                                  b'\x10\x66\x66\x01\x00\xfb\xff\x39\xc0'
                                  b'\x02\x09\x00\x00\x6f')
 
+    def test_negative_level(self):
+        cctx = zstd.ZstdCompressor(level=-4)
+        result = cctx.compress(b'foo' * 256)
+
+    def test_no_magic(self):
+        params = zstd.ZstdCompressionParameters.from_level(
+            1, format=zstd.FORMAT_ZSTD1)
+        cctx = zstd.ZstdCompressor(compression_params=params)
+        magic = cctx.compress(b'foobar')
+
+        params = zstd.ZstdCompressionParameters.from_level(
+            1, format=zstd.FORMAT_ZSTD1_MAGICLESS)
+        cctx = zstd.ZstdCompressor(compression_params=params)
+        no_magic = cctx.compress(b'foobar')
+
+        self.assertEqual(magic[0:4], b'\x28\xb5\x2f\xfd')
+        self.assertEqual(magic[4:], no_magic)
+
     def test_write_checksum(self):
         cctx = zstd.ZstdCompressor(level=1)
         no_checksum = cctx.compress(b'foobar')
@@ -109,15 +122,15 b' class TestCompressor_compress(unittest.T'
 
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
+        with_size = cctx.compress(b'foobar' * 256)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         no_size = cctx.compress(b'foobar' * 256)
-        cctx = zstd.ZstdCompressor(level=1, write_content_size=True)
-        with_size = cctx.compress(b'foobar' * 256)
 
         self.assertEqual(len(with_size), len(no_size) + 1)
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
-        self.assertEqual(no_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
 
     def test_no_dict_id(self):
@@ -140,7 +153,7 b' class TestCompressor_compress(unittest.T'
         no_params = zstd.get_frame_parameters(no_dict_id)
         with_params = zstd.get_frame_parameters(with_dict_id)
         self.assertEqual(no_params.dict_id, 0)
-        self.assertEqual(with_params.dict_id, 1584102229)
+        self.assertEqual(with_params.dict_id, 1387616518)
 
     def test_compress_dict_multiple(self):
         samples = []
@@ -156,6 +169,21 b' class TestCompressor_compress(unittest.T'
         for i in range(32):
             cctx.compress(b'foo bar foobar foo bar foobar')
 
+    def test_dict_precompute(self):
+        samples = []
+        for i in range(128):
+            samples.append(b'foo' * 64)
+            samples.append(b'bar' * 64)
+            samples.append(b'foobar' * 64)
+
+        d = zstd.train_dictionary(8192, samples)
+        d.precompute_compress(level=1)
+
+        cctx = zstd.ZstdCompressor(level=1, dict_data=d)
+
+        for i in range(32):
+            cctx.compress(b'foo bar foobar foo bar foobar')
+
     def test_multithreaded(self):
         chunk_size = multithreaded_chunk_size(1)
         source = b''.join([b'x' * chunk_size, b'y' * chunk_size])
@@ -171,16 +199,65 b' class TestCompressor_compress(unittest.T'
         dctx = zstd.ZstdDecompressor()
         self.assertEqual(dctx.decompress(compressed), source)
 
+    def test_multithreaded_dict(self):
+        samples = []
+        for i in range(128):
+            samples.append(b'foo' * 64)
+            samples.append(b'bar' * 64)
+            samples.append(b'foobar' * 64)
+
+        d = zstd.train_dictionary(1024, samples)
+
+        cctx = zstd.ZstdCompressor(dict_data=d, threads=2)
+
+        result = cctx.compress(b'foo')
+        params = zstd.get_frame_parameters(result);
+        self.assertEqual(params.content_size, 3);
+        self.assertEqual(params.dict_id, d.dict_id())
+
+        self.assertEqual(result,
+                         b'\x28\xb5\x2f\xfd\x23\x06\x59\xb5\x52\x03\x19\x00\x00'
+                         b'\x66\x6f\x6f')
+
+    def test_multithreaded_compression_params(self):
+        params = zstd.ZstdCompressionParameters.from_level(0, threads=2)
+        cctx = zstd.ZstdCompressor(compression_params=params)
+
+        result = cctx.compress(b'foo')
+        params = zstd.get_frame_parameters(result);
+        self.assertEqual(params.content_size, 3);
+
+        self.assertEqual(result,
+                         b'\x28\xb5\x2f\xfd\x20\x03\x19\x00\x00\x66\x6f\x6f')
+
 
 @make_cffi
 class TestCompressor_compressobj(unittest.TestCase):
     def test_compressobj_empty(self):
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cobj = cctx.compressobj()
         self.assertEqual(cobj.compress(b''), b'')
         self.assertEqual(cobj.flush(),
                          b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
+    def test_input_types(self):
+        expected = b'\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f'
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
+
+        mutable_array = bytearray(3)
+        mutable_array[:] = b'foo'
+
+        sources = [
+            memoryview(b'foo'),
+            bytearray(b'foo'),
+            mutable_array,
+        ]
+
+        for source in sources:
+            cobj = cctx.compressobj()
+            self.assertEqual(cobj.compress(source), b'')
+            self.assertEqual(cobj.flush(), expected)
+
     def test_compressobj_large(self):
         chunks = []
         for i in range(255):
@@ -194,7 +271,7 b' class TestCompressor_compressobj(unittes'
         self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd')
 
         params = zstd.get_frame_parameters(result)
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1048576)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
@@ -209,8 +286,8 b' class TestCompressor_compressobj(unittes'
 
         no_params = zstd.get_frame_parameters(no_checksum)
         with_params = zstd.get_frame_parameters(with_checksum)
-        self.assertEqual(no_params.content_size, 0)
-        self.assertEqual(with_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
@@ -221,14 +298,14 b' class TestCompressor_compressobj(unittes'
     def test_write_content_size(self):
         cctx = zstd.ZstdCompressor(level=1)
         cobj = cctx.compressobj(size=len(b'foobar' * 256))
+        with_size = cobj.compress(b'foobar' * 256) + cobj.flush()
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
+        cobj = cctx.compressobj(size=len(b'foobar' * 256))
         no_size = cobj.compress(b'foobar' * 256) + cobj.flush()
-        cctx = zstd.ZstdCompressor(level=1, write_content_size=True)
-        cobj = cctx.compressobj(size=len(b'foobar' * 256))
-        with_size = cobj.compress(b'foobar' * 256) + cobj.flush()
 
         no_params = zstd.get_frame_parameters(no_size)
         with_params = zstd.get_frame_parameters(with_size)
-        self.assertEqual(no_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
@@ -300,6 +377,34 b' class TestCompressor_compressobj(unittes'
 
         self.assertEqual(len(compressed), 295)
 
+    def test_frame_progression(self):
+        cctx = zstd.ZstdCompressor()
+
+        self.assertEqual(cctx.frame_progression(), (0, 0, 0))
+
+        cobj = cctx.compressobj()
+
+        cobj.compress(b'foobar')
+        self.assertEqual(cctx.frame_progression(), (6, 0, 0))
+
+        cobj.flush()
+        self.assertEqual(cctx.frame_progression(), (6, 6, 15))
+
+    def test_bad_size(self):
+        cctx = zstd.ZstdCompressor()
+
+        cobj = cctx.compressobj(size=2)
+        with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+            cobj.compress(b'foo')
+
+        # Try another operation on this instance.
+        with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+            cobj.compress(b'aa')
+
+        # Try another operation on the compressor.
+        cctx.compressobj(size=4)
+        cctx.compress(b'foobar')
+
 
 @make_cffi
 class TestCompressor_copy_stream(unittest.TestCase):
@@ -323,7 +428,7 b' class TestCompressor_copy_stream(unittes'
         source = io.BytesIO()
         dest = io.BytesIO()
 
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(int(r), 0)
         self.assertEqual(w, 9)
@@ -345,7 +450,7 b' class TestCompressor_copy_stream(unittes'
         self.assertEqual(w, 999)
 
         params = zstd.get_frame_parameters(dest.getvalue())
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1048576)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
@@ -367,8 +472,8 b' class TestCompressor_copy_stream(unittes'
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
-        self.assertEqual(no_params.content_size, 0)
-        self.assertEqual(with_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
@@ -378,12 +483,12 b' class TestCompressor_copy_stream(unittes'
         source = io.BytesIO(b'foobar' * 256)
         no_size = io.BytesIO()
 
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
         cctx.copy_stream(source, no_size)
 
         source.seek(0)
         with_size = io.BytesIO()
-        cctx = zstd.ZstdCompressor(level=1, write_content_size=True)
+        cctx = zstd.ZstdCompressor(level=1)
         cctx.copy_stream(source, with_size)
 
         # Source content size is unknown, so no content size written.
@@ -400,7 +505,7 b' class TestCompressor_copy_stream(unittes'
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
-        self.assertEqual(no_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
@@ -426,19 +531,18 b' class TestCompressor_copy_stream(unittes'
         source.seek(0)
 
         dest = io.BytesIO()
-        cctx = zstd.ZstdCompressor(threads=2)
+        cctx = zstd.ZstdCompressor(threads=2, write_content_size=False)
         r, w = cctx.copy_stream(source, dest)
         self.assertEqual(r, 3145728)
         self.assertEqual(w, 295)
 
         params = zstd.get_frame_parameters(dest.getvalue())
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Writing content size and checksum works.
-        cctx = zstd.ZstdCompressor(threads=2, write_content_size=True,
-                                   write_checksum=True)
+        cctx = zstd.ZstdCompressor(threads=2, write_checksum=True)
         dest = io.BytesIO()
         source.seek(0)
         cctx.copy_stream(source, dest, size=len(source.getvalue()))
@@ -448,31 +552,227 b' class TestCompressor_copy_stream(unittes'
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
 
+    def test_bad_size(self):
+        source = io.BytesIO()
+        source.write(b'a' * 32768)
+        source.write(b'b' * 32768)
+        source.seek(0)
 
-def compress(data, level):
-    buffer = io.BytesIO()
-    cctx = zstd.ZstdCompressor(level=level)
-    with cctx.write_to(buffer) as compressor:
-        compressor.write(data)
-    return buffer.getvalue()
+        dest = io.BytesIO()
+
+        cctx = zstd.ZstdCompressor()
+
+        with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+            cctx.copy_stream(source, dest, size=42)
+
+        # Try another operation on this compressor.
+        source.seek(0)
+        dest = io.BytesIO()
+        cctx.copy_stream(source, dest)
 
 
 @make_cffi
-class TestCompressor_write_to(unittest.TestCase):
+class TestCompressor_stream_reader(unittest.TestCase):
+    def test_context_manager(self):
+        cctx = zstd.ZstdCompressor()
+
+        reader = cctx.stream_reader(b'foo' * 60)
+        with self.assertRaisesRegexp(zstd.ZstdError, 'read\(\) must be called from an active'):
+            reader.read(10)
+
+        with cctx.stream_reader(b'foo') as reader:
+            with self.assertRaisesRegexp(ValueError, 'cannot __enter__ multiple times'):
+                with reader as reader2:
+                    pass
+
+    def test_not_implemented(self):
+        cctx = zstd.ZstdCompressor()
+
+        with cctx.stream_reader(b'foo' * 60) as reader:
+            with self.assertRaises(io.UnsupportedOperation):
+                reader.readline()
+
+            with self.assertRaises(io.UnsupportedOperation):
+                reader.readlines()
+
+            # This could probably be implemented someday.
+            with self.assertRaises(NotImplementedError):
+                reader.readall()
+
+            with self.assertRaises(io.UnsupportedOperation):
+                iter(reader)
+
+            with self.assertRaises(io.UnsupportedOperation):
+                next(reader)
+
+            with self.assertRaises(OSError):
+                reader.writelines([])
+
+            with self.assertRaises(OSError):
+                reader.write(b'foo')
+
+    def test_constant_methods(self):
+        cctx = zstd.ZstdCompressor()
+
+        with cctx.stream_reader(b'boo') as reader:
+            self.assertTrue(reader.readable())
+            self.assertFalse(reader.writable())
+            self.assertFalse(reader.seekable())
+            self.assertFalse(reader.isatty())
+            self.assertIsNone(reader.flush())
+
+    def test_read_closed(self):
+        cctx = zstd.ZstdCompressor()
+
+        with cctx.stream_reader(b'foo' * 60) as reader:
+            reader.close()
+            with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+                reader.read(10)
+
+    def test_read_bad_size(self):
+        cctx = zstd.ZstdCompressor()
+
+        with cctx.stream_reader(b'foo') as reader:
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
+                reader.read(-1)
+
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
+                reader.read(0)
+
+    def test_read_buffer(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        with cctx.stream_reader(source) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            # We should get entire frame in one read.
+            result = reader.read(8192)
+            self.assertEqual(result, frame)
+            self.assertEqual(reader.tell(), len(result))
+            self.assertEqual(reader.read(), b'')
+            self.assertEqual(reader.tell(), len(result))
+
+    def test_read_buffer_small_chunks(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = b'foo' * 60
+        chunks = []
+
+        with cctx.stream_reader(source) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            while True:
+                chunk = reader.read(1)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+                self.assertEqual(reader.tell(), sum(map(len, chunks)))
+
+        self.assertEqual(b''.join(chunks), cctx.compress(source))
+
+    def test_read_stream(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            chunk = reader.read(8192)
+            self.assertEqual(chunk, frame)
+            self.assertEqual(reader.tell(), len(chunk))
+            self.assertEqual(reader.read(), b'')
+            self.assertEqual(reader.tell(), len(chunk))
+
+    def test_read_stream_small_chunks(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = b'foo' * 60
+        chunks = []
+
+        with cctx.stream_reader(io.BytesIO(source), size=len(source)) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            while True:
+                chunk = reader.read(1)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+                self.assertEqual(reader.tell(), sum(map(len, chunks)))
+
+        self.assertEqual(b''.join(chunks), cctx.compress(source))
+
+    def test_read_after_exit(self):
+        cctx = zstd.ZstdCompressor()
+
+        with cctx.stream_reader(b'foo' * 60) as reader:
+            while reader.read(8192):
+                pass
+
+        with self.assertRaisesRegexp(zstd.ZstdError, 'read\(\) must be called from an active'):
+            reader.read(10)
+
+    def test_bad_size(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = io.BytesIO(b'foobar')
+
+        with cctx.stream_reader(source, size=2) as reader:
+            with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+                reader.read(10)
+
+        # Try another compression operation.
+        with cctx.stream_reader(source, size=42):
+            pass
+
+
+@make_cffi
+class TestCompressor_stream_writer(unittest.TestCase):
     def test_empty(self):
-        result = compress(b'', 1)
+        buffer = io.BytesIO()
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
+        with cctx.stream_writer(buffer) as compressor:
+            compressor.write(b'')
+
+        result = buffer.getvalue()
         self.assertEqual(result, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
         params = zstd.get_frame_parameters(result)
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 524288)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
+    def test_input_types(self):
+        expected = b'\x28\xb5\x2f\xfd\x00\x48\x19\x00\x00\x66\x6f\x6f'
+        cctx = zstd.ZstdCompressor(level=1)
+
+        mutable_array = bytearray(3)
+        mutable_array[:] = b'foo'
+
+        sources = [
+            memoryview(b'foo'),
+            bytearray(b'foo'),
+            mutable_array,
+        ]
+
+        for source in sources:
+            buffer = io.BytesIO()
+            with cctx.stream_writer(buffer) as compressor:
+                compressor.write(source)
+
+            self.assertEqual(buffer.getvalue(), expected)
+
     def test_multiple_compress(self):
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=5)
-        with cctx.write_to(buffer) as compressor:
+        with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'x' * 8192), 0)
@@ -491,35 +791,40 b' class TestCompressor_write_to(unittest.T'
 
         d = zstd.train_dictionary(8192, samples)
 
+        h = hashlib.sha1(d.as_bytes()).hexdigest()
+        self.assertEqual(h, '3040faa0ddc37d50e71a4dd28052cb8db5d9d027')
+
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=9, dict_data=d)
-        with cctx.write_to(buffer) as compressor:
+        with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
-            self.assertEqual(compressor.write(b'foo' * 16384), 634)
+            self.assertEqual(compressor.write(b'foo' * 16384), 0)
 
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
-        self.assertEqual(params.content_size, 0)
-        self.assertEqual(params.window_size, 1024)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(params.window_size, 2097152)
         self.assertEqual(params.dict_id, d.dict_id())
         self.assertFalse(params.has_checksum)
-
-        self.assertEqual(compressed[0:32],
-                         b'\x28\xb5\x2f\xfd\x03\x00\x55\x7b\x6b\x5e\x54\x00'
-                         b'\x00\x00\x02\xfc\xf4\xa5\xba\x23\x3f\x85\xb3\x54'
-                         b'\x00\x00\x18\x6f\x6f\x66\x01\x00')
-
-        h = hashlib.sha1(compressed).hexdigest()
-        self.assertEqual(h, '1c5bcd25181bcd8c1a73ea8773323e0056129f92')
+        self.assertEqual(compressed,
+                         b'\x28\xb5\x2f\xfd\x03\x58\x06\x59\xb5\x52\x5d\x00'
+                         b'\x00\x00\x02\xfc\x3d\x3f\xd9\xb0\x51\x03\x45\x89')
 
     def test_compression_params(self):
-        params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST)
+        params = zstd.ZstdCompressionParameters(
+            window_log=20,
+            chain_log=6,
+            hash_log=12,
+            min_match=5,
+            search_log=4,
+            target_length=10,
+            compression_strategy=zstd.STRATEGY_FAST)
 
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor(compression_params=params)
-        with cctx.write_to(buffer) as compressor:
+        with cctx.stream_writer(buffer) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'foobar' * 16384), 0)
@@ -527,29 +832,29 b' class TestCompressor_write_to(unittest.T'
         compressed = buffer.getvalue()
 
         params = zstd.get_frame_parameters(compressed)
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1048576)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         h = hashlib.sha1(compressed).hexdigest()
-        self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99')
+        self.assertEqual(h, '2a8111d72eb5004cdcecbdac37da9f26720d30ef')
 
     def test_write_checksum(self):
         no_checksum = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1)
-        with cctx.write_to(no_checksum) as compressor:
+        with cctx.stream_writer(no_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
 
         with_checksum = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1, write_checksum=True)
-        with cctx.write_to(with_checksum) as compressor:
+        with cctx.stream_writer(with_checksum) as compressor:
             self.assertEqual(compressor.write(b'foobar'), 0)
 
         no_params = zstd.get_frame_parameters(no_checksum.getvalue())
         with_params = zstd.get_frame_parameters(with_checksum.getvalue())
-        self.assertEqual(no_params.content_size, 0)
-        self.assertEqual(with_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
         self.assertFalse(no_params.has_checksum)
@@ -560,13 +865,13 b' class TestCompressor_write_to(unittest.T'
 
     def test_write_content_size(self):
         no_size = io.BytesIO()
-        cctx = zstd.ZstdCompressor(level=1)
-        with cctx.write_to(no_size) as compressor:
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
+        with cctx.stream_writer(no_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         with_size = io.BytesIO()
-        cctx = zstd.ZstdCompressor(level=1, write_content_size=True)
-        with cctx.write_to(with_size) as compressor:
+        cctx = zstd.ZstdCompressor(level=1)
+        with cctx.stream_writer(with_size) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         # Source size is not known in streaming mode, so header not
@@ -576,12 +881,12 b' class TestCompressor_write_to(unittest.T'
 
         # Declaring size will write the header.
         with_size = io.BytesIO()
-        with cctx.write_to(with_size, size=len(b'foobar' * 256)) as compressor:
+        with cctx.stream_writer(with_size, size=len(b'foobar' * 256)) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 256), 0)
 
         no_params = zstd.get_frame_parameters(no_size.getvalue())
         with_params = zstd.get_frame_parameters(with_size.getvalue())
-        self.assertEqual(no_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(with_params.content_size, 1536)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, 0)
@@ -602,18 +907,22 b' class TestCompressor_write_to(unittest.T'
 
         with_dict_id = io.BytesIO()
         cctx = zstd.ZstdCompressor(level=1, dict_data=d)
-        with cctx.write_to(with_dict_id) as compressor:
+        with cctx.stream_writer(with_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
 
+        self.assertEqual(with_dict_id.getvalue()[4:5], b'\x03')
+
         cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
         no_dict_id = io.BytesIO()
-        with cctx.write_to(no_dict_id) as compressor:
+        with cctx.stream_writer(no_dict_id) as compressor:
             self.assertEqual(compressor.write(b'foobarfoobar'), 0)
 
+        self.assertEqual(no_dict_id.getvalue()[4:5], b'\x00')
+
         no_params = zstd.get_frame_parameters(no_dict_id.getvalue())
         with_params = zstd.get_frame_parameters(with_dict_id.getvalue())
-        self.assertEqual(no_params.content_size, 0)
-        self.assertEqual(with_params.content_size, 0)
+        self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        self.assertEqual(with_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(no_params.dict_id, 0)
         self.assertEqual(with_params.dict_id, d.dict_id())
         self.assertFalse(no_params.has_checksum)
@@ -625,7 +934,8 b' class TestCompressor_write_to(unittest.T'
     def test_memory_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         buffer = io.BytesIO()
-        with cctx.write_to(buffer) as compressor:
+        with cctx.stream_writer(buffer) as compressor:
+            compressor.write(b'foo')
             size = compressor.memory_size()
 
         self.assertGreater(size, 100000)
@@ -633,7 +943,7 b' class TestCompressor_write_to(unittest.T'
     def test_write_size(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
-        with cctx.write_to(dest, write_size=1) as compressor:
+        with cctx.stream_writer(dest, write_size=1) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(compressor.write(b'bar'), 0)
             self.assertEqual(compressor.write(b'foobar'), 0)
@@ -643,7 +953,7 b' class TestCompressor_write_to(unittest.T'
     def test_flush_repeated(self):
         cctx = zstd.ZstdCompressor(level=3)
         dest = OpCountingBytesIO()
-        with cctx.write_to(dest) as compressor:
+        with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b'foo'), 0)
             self.assertEqual(dest._write_count, 0)
             self.assertEqual(compressor.flush(), 12)
@@ -659,7 +969,7 b' class TestCompressor_write_to(unittest.T'
     def test_flush_empty_block(self):
         cctx = zstd.ZstdCompressor(level=3, write_checksum=True)
         dest = OpCountingBytesIO()
-        with cctx.write_to(dest) as compressor:
+        with cctx.stream_writer(dest) as compressor:
             self.assertEqual(compressor.write(b'foobar' * 8192), 0)
             count = dest._write_count
             offset = dest.tell()
@@ -680,50 +990,89 b' class TestCompressor_write_to(unittest.T'
     def test_multithreaded(self):
         dest = io.BytesIO()
         cctx = zstd.ZstdCompressor(threads=2)
-        with cctx.write_to(dest) as compressor:
+        with cctx.stream_writer(dest) as compressor:
             compressor.write(b'a' * 1048576)
             compressor.write(b'b' * 1048576)
             compressor.write(b'c' * 1048576)
 
         self.assertEqual(len(dest.getvalue()), 295)
 
+    def test_tell(self):
+        dest = io.BytesIO()
+        cctx = zstd.ZstdCompressor()
+        with cctx.stream_writer(dest) as compressor:
+            self.assertEqual(compressor.tell(), 0)
+
+            for i in range(256):
+                compressor.write(b'foo' * (i + 1))
+                self.assertEqual(compressor.tell(), dest.tell())
+
+    def test_bad_size(self):
+        cctx = zstd.ZstdCompressor()
+
+        dest = io.BytesIO()
+
+        with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+            with cctx.stream_writer(dest, size=2) as compressor:
+                compressor.write(b'foo')
+
+        # Test another operation.
+        with cctx.stream_writer(dest, size=42):
+            pass
+
+    def test_tarfile_compat(self):
+        raise unittest.SkipTest('not yet fully working')
+
+        dest = io.BytesIO()
+        cctx = zstd.ZstdCompressor()
+        with cctx.stream_writer(dest) as compressor:
+            with tarfile.open('tf', mode='w', fileobj=compressor) as tf:
+                tf.add(__file__, 'test_compressor.py')
+
+        dest.seek(0)
+
+        dctx = zstd.ZstdDecompressor()
+        with dctx.stream_reader(dest) as reader:
+            with tarfile.open(mode='r:', fileobj=reader) as tf:
+                for member in tf:
+                    self.assertEqual(member.name, 'test_compressor.py')
 
 @make_cffi
-class TestCompressor_read_from(unittest.TestCase):
+class TestCompressor_read_to_iter(unittest.TestCase):
     def test_type_validation(self):
         cctx = zstd.ZstdCompressor()
 
         # Object with read() works.
-        for chunk in cctx.read_from(io.BytesIO()):
+        for chunk in cctx.read_to_iter(io.BytesIO()):
             pass
 
         # Buffer protocol works.
-        for chunk in cctx.read_from(b'foobar'):
+        for chunk in cctx.read_to_iter(b'foobar'):
             pass
 
         with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'):
-            for chunk in cctx.read_from(True):
+            for chunk in cctx.read_to_iter(True):
                 pass
 
     def test_read_empty(self):
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
-        it = cctx.read_from(source)
+        it = cctx.read_to_iter(source)
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed = b''.join(chunks)
         self.assertEqual(compressed, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00')
 
         # And again with the buffer protocol.
-        it = cctx.read_from(b'')
+        it = cctx.read_to_iter(b'')
         chunks = list(it)
         self.assertEqual(len(chunks), 1)
         compressed2 = b''.join(chunks)
         self.assertEqual(compressed2, compressed)
 
     def test_read_large(self):
-        cctx = zstd.ZstdCompressor(level=1)
+        cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
 
         source = io.BytesIO()
         source.write(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE)
@@ -732,7 +1081,7 b' class TestCompressor_read_from(unittest.'
 
         # Creating an iterator should not perform any compression until
         # first read.
-        it = cctx.read_from(source, size=len(source.getvalue()))
+        it = cctx.read_to_iter(source, size=len(source.getvalue()))
         self.assertEqual(source.tell(), 0)
 
         # We should have exactly 2 output chunks.
@@ -758,21 +1107,28 b' class TestCompressor_read_from(unittest.'
         self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue()))
 
         params = zstd.get_frame_parameters(b''.join(chunks))
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Now check the buffer protocol.
-        it = cctx.read_from(source.getvalue())
+        it = cctx.read_to_iter(source.getvalue())
         chunks = list(it)
         self.assertEqual(len(chunks), 2)
+
+        params = zstd.get_frame_parameters(b''.join(chunks))
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+        #self.assertEqual(params.window_size, 262144)
+        self.assertEqual(params.dict_id, 0)
+        self.assertFalse(params.has_checksum)
+
         self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue()))
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(b'foobarfoobar')
         cctx = zstd.ZstdCompressor(level=3)
-        for chunk in cctx.read_from(source, read_size=1, write_size=1):
+        for chunk in cctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
 
         self.assertEqual(source._read_count, len(source.getvalue()) + 1)
@@ -786,17 +1142,22 b' class TestCompressor_read_from(unittest.'
 
         cctx = zstd.ZstdCompressor(threads=2)
 
-        compressed = b''.join(cctx.read_from(source))
+        compressed = b''.join(cctx.read_to_iter(source))
         self.assertEqual(len(compressed), 295)
 
+    def test_bad_size(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = io.BytesIO(b'a' * 42)
+
+        with self.assertRaisesRegexp(zstd.ZstdError, 'Src size is incorrect'):
+            b''.join(cctx.read_to_iter(source, size=2))
+
+        # Test another operation on errored compressor.
+        b''.join(cctx.read_to_iter(source))
+
 
 class TestCompressor_multi_compress_to_buffer(unittest.TestCase):
-    def test_multithreaded_unsupported(self):
-        cctx = zstd.ZstdCompressor(threads=2)
-
-        with self.assertRaisesRegexp(zstd.ZstdError, 'function cannot be called on ZstdCompressor configured for multi-threaded compression'):
-            cctx.multi_compress_to_buffer([b'foo'])
-
     def test_invalid_inputs(self):
         cctx = zstd.ZstdCompressor()
 
@@ -819,7 +1180,7 b' class TestCompressor_multi_compress_to_b'
             cctx.multi_compress_to_buffer([b'', b'', b''])
 
     def test_list_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True, write_checksum=True)
+        cctx = zstd.ZstdCompressor(write_checksum=True)
 
         original = [b'foo' * 12, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
@@ -834,7 +1195,7 b' class TestCompressor_multi_compress_to_b'
         self.assertEqual(b[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True, write_checksum=True)
+        cctx = zstd.ZstdCompressor(write_checksum=True)
 
         original = [b'foo' * 4, b'bar' * 6]
         frames = [cctx.compress(c) for c in original]
@@ -852,7 +1213,7 b' class TestCompressor_multi_compress_to_b'
         self.assertEqual(result[1].tobytes(), frames[1])
 
     def test_buffer_with_segments_collection_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True, write_checksum=True)
+        cctx = zstd.ZstdCompressor(write_checksum=True)
 
         original = [
             b'foo1',
@@ -886,10 +1247,10 b' class TestCompressor_multi_compress_to_b'
     def test_multiple_threads(self):
         # threads argument will cause multi-threaded ZSTD APIs to be used, which will
         # make output different.
-        refcctx = zstd.ZstdCompressor(write_content_size=True, write_checksum=True)
+        refcctx = zstd.ZstdCompressor(write_checksum=True)
         reference = [refcctx.compress(b'x' * 64), refcctx.compress(b'y' * 64)]
 
-        cctx = zstd.ZstdCompressor(write_content_size=True, write_checksum=True)
+        cctx = zstd.ZstdCompressor(write_checksum=True)
 
         frames = []
         frames.extend(b'x' * 64 for i in range(256))

contrib/python-zstandard/tests/test_compressor_fuzzing.py

@@ -1,10 +1,6 b''
 import io
 import os
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import unittest
 
 try:
     import hypothesis
@@ -12,7 +8,7 b' try:'
 except ImportError:
     raise unittest.SkipTest('hypothesis not available')
 
-import zstd
+import zstandard as zstd
 
 from . common import (
     make_cffi,
@@ -22,7 +18,57 b' from . common import ('
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
-class TestCompressor_write_to_fuzzing(unittest.TestCase):
+class TestCompressor_stream_reader_fuzzing(unittest.TestCase):
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_stream_source_read_variance(self, original, level, source_read_size,
+                                         read_sizes):
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(io.BytesIO(original), size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                chunk = reader.read(read_size)
+
+                if not chunk:
+                    break
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_buffer_source_read_variance(self, original, level, source_read_size,
+                                         read_sizes):
+
+        refctx = zstd.ZstdCompressor(level=level)
+        ref_frame = refctx.compress(original)
+
+        cctx = zstd.ZstdCompressor(level=level)
+        with cctx.stream_reader(original, size=len(original),
+                                read_size=source_read_size) as reader:
+            chunks = []
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                chunk = reader.read(read_size)
+                if not chunk:
+                    break
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), ref_frame)
+
+
[e-mail]('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
+@make_cffi
+class TestCompressor_stream_writer_fuzzing(unittest.TestCase):
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                         level=strategies.integers(min_value=1, max_value=5),
                         write_size=strategies.integers(min_value=1, max_value=1048576))
@@ -32,7 +78,7 b' class TestCompressor_write_to_fuzzing(un'
 
         cctx = zstd.ZstdCompressor(level=level)
         b = io.BytesIO()
-        with cctx.write_to(b, size=len(original), write_size=write_size) as compressor:
+        with cctx.stream_writer(b, size=len(original), write_size=write_size) as compressor:
             compressor.write(original)
 
         self.assertEqual(b.getvalue(), ref_frame)
@@ -62,13 +108,12 b' class TestCompressor_copy_stream_fuzzing'
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
 class TestCompressor_compressobj_fuzzing(unittest.TestCase):
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
-                      chunk_sizes=strategies.streaming(
-                          strategies.integers(min_value=1, max_value=4096)))
+                      chunk_sizes=strategies.data())
     def test_random_input_sizes(self, original, level, chunk_sizes):
-        chunk_sizes = iter(chunk_sizes)
-
         refctx = zstd.ZstdCompressor(level=level)
         ref_frame = refctx.compress(original)
 
@@ -78,7 +123,7 b' class TestCompressor_compressobj_fuzzing'
         chunks = []
         i = 0
         while True:
-            chunk_size = next(chunk_sizes)
+            chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
             source = original[i:i + chunk_size]
             if not source:
                 break
@@ -93,7 +138,7 b' class TestCompressor_compressobj_fuzzing'
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
-class TestCompressor_read_from_fuzzing(unittest.TestCase):
+class TestCompressor_read_to_iter_fuzzing(unittest.TestCase):
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
                       read_size=strategies.integers(min_value=1, max_value=4096),
@@ -105,8 +150,9 b' class TestCompressor_read_from_fuzzing(u'
         source = io.BytesIO(original)
 
         cctx = zstd.ZstdCompressor(level=level)
-        chunks = list(cctx.read_from(source, size=len(original), read_size=read_size,
-                                     write_size=write_size))
+        chunks = list(cctx.read_to_iter(source, size=len(original),
+                                        read_size=read_size,
+                                        write_size=write_size))
 
         self.assertEqual(b''.join(chunks), ref_frame)
 
@@ -125,7 +171,6 b' class TestCompressor_multi_compress_to_b'
             kwargs['dict_data'] = zstd.ZstdCompressionDict(original[0])
 
         cctx = zstd.ZstdCompressor(level=1,
-                                   write_content_size=True,
                                    write_checksum=True,
                                    **kwargs)
 

contrib/python-zstandard/tests/test_data_structures.py

@@ -1,9 +1,7 b''
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import sys
+import unittest
 
-import zstd
+import zstandard as zstd
 
 from . common import (
     make_cffi,
@@ -12,52 +10,104 b' from . common import ('
 
 @make_cffi
 class TestCompressionParameters(unittest.TestCase):
-    def test_init_bad_arg_type(self):
-        with self.assertRaises(TypeError):
-            zstd.CompressionParameters()
-
-        with self.assertRaises(TypeError):
-            zstd.CompressionParameters(0, 1)
+    def test_bounds(self):
+        zstd.ZstdCompressionParameters(window_log=zstd.WINDOWLOG_MIN,
+                                       chain_log=zstd.CHAINLOG_MIN,
+                                       hash_log=zstd.HASHLOG_MIN,
+                                       search_log=zstd.SEARCHLOG_MIN,
+                                       min_match=zstd.SEARCHLENGTH_MIN + 1,
+                                       target_length=zstd.TARGETLENGTH_MIN,
+                                       compression_strategy=zstd.STRATEGY_FAST)
 
-    def test_bounds(self):
-        zstd.CompressionParameters(zstd.WINDOWLOG_MIN,
-                                   zstd.CHAINLOG_MIN,
-                                   zstd.HASHLOG_MIN,
-                                   zstd.SEARCHLOG_MIN,
-                                   zstd.SEARCHLENGTH_MIN + 1,
-                                   zstd.TARGETLENGTH_MIN,
-                                   zstd.STRATEGY_FAST)
+        zstd.ZstdCompressionParameters(window_log=zstd.WINDOWLOG_MAX,
+                                       chain_log=zstd.CHAINLOG_MAX,
+                                       hash_log=zstd.HASHLOG_MAX,
+                                       search_log=zstd.SEARCHLOG_MAX,
+                                       min_match=zstd.SEARCHLENGTH_MAX - 1,
+                                       compression_strategy=zstd.STRATEGY_BTULTRA)
 
-        zstd.CompressionParameters(zstd.WINDOWLOG_MAX,
-                                   zstd.CHAINLOG_MAX,
-                                   zstd.HASHLOG_MAX,
-                                   zstd.SEARCHLOG_MAX,
-                                   zstd.SEARCHLENGTH_MAX - 1,
-                                   zstd.TARGETLENGTH_MAX,
-                                   zstd.STRATEGY_BTOPT)
-
-    def test_get_compression_parameters(self):
-        p = zstd.get_compression_parameters(1)
+    def test_from_level(self):
+        p = zstd.ZstdCompressionParameters.from_level(1)
         self.assertIsInstance(p, zstd.CompressionParameters)
 
         self.assertEqual(p.window_log, 19)
 
+        p = zstd.ZstdCompressionParameters.from_level(-4)
+        self.assertEqual(p.window_log, 19)
+        self.assertEqual(p.compress_literals, 0)
+
     def test_members(self):
-        p = zstd.CompressionParameters(10, 6, 7, 4, 5, 8, 1)
+        p = zstd.ZstdCompressionParameters(window_log=10,
+                                           chain_log=6,
+                                           hash_log=7,
+                                           search_log=4,
+                                           min_match=5,
+                                           target_length=8,
+                                           compression_strategy=1)
         self.assertEqual(p.window_log, 10)
         self.assertEqual(p.chain_log, 6)
         self.assertEqual(p.hash_log, 7)
         self.assertEqual(p.search_log, 4)
-        self.assertEqual(p.search_length, 5)
+        self.assertEqual(p.min_match, 5)
         self.assertEqual(p.target_length, 8)
-        self.assertEqual(p.strategy, 1)
+        self.assertEqual(p.compression_strategy, 1)
+
+        p = zstd.ZstdCompressionParameters(compression_level=2)
+        self.assertEqual(p.compression_level, 2)
+
+        p = zstd.ZstdCompressionParameters(threads=4)
+        self.assertEqual(p.threads, 4)
+
+        p = zstd.ZstdCompressionParameters(threads=2, job_size=1048576,
+                                       overlap_size_log=6)
+        self.assertEqual(p.threads, 2)
+        self.assertEqual(p.job_size, 1048576)
+        self.assertEqual(p.overlap_size_log, 6)
+
+        p = zstd.ZstdCompressionParameters(compression_level=2)
+        self.assertEqual(p.compress_literals, 1)
+
+        p = zstd.ZstdCompressionParameters(compress_literals=False)
+        self.assertEqual(p.compress_literals, 0)
+
+        p = zstd.ZstdCompressionParameters(compression_level=-1)
+        self.assertEqual(p.compression_level, -1)
+        self.assertEqual(p.compress_literals, 0)
+
+        p = zstd.ZstdCompressionParameters(compression_level=-2, compress_literals=True)
+        self.assertEqual(p.compression_level, -2)
+        self.assertEqual(p.compress_literals, 1)
+
+        p = zstd.ZstdCompressionParameters(force_max_window=True)
+        self.assertEqual(p.force_max_window, 1)
+
+        p = zstd.ZstdCompressionParameters(enable_ldm=True)
+        self.assertEqual(p.enable_ldm, 1)
+
+        p = zstd.ZstdCompressionParameters(ldm_hash_log=7)
+        self.assertEqual(p.ldm_hash_log, 7)
+
+        p = zstd.ZstdCompressionParameters(ldm_min_match=6)
+        self.assertEqual(p.ldm_min_match, 6)
+
+        p = zstd.ZstdCompressionParameters(ldm_bucket_size_log=7)
+        self.assertEqual(p.ldm_bucket_size_log, 7)
+
+        p = zstd.ZstdCompressionParameters(ldm_hash_every_log=8)
+        self.assertEqual(p.ldm_hash_every_log, 8)
 
     def test_estimated_compression_context_size(self):
-        p = zstd.CompressionParameters(20, 16, 17,  1,  5, 16, zstd.STRATEGY_DFAST)
+        p = zstd.ZstdCompressionParameters(window_log=20,
+                                           chain_log=16,
+                                           hash_log=17,
+                                           search_log=1,
+                                           min_match=5,
+                                           target_length=16,
+                                           compression_strategy=zstd.STRATEGY_DFAST)
 
         # 32-bit has slightly different values from 64-bit.
-        self.assertAlmostEqual(p.estimated_compression_context_size(), 1287076,
-                               delta=110)
+        self.assertAlmostEqual(p.estimated_compression_context_size(), 1294072,
+                               delta=250)
 
 
 @make_cffi
@@ -66,8 +116,18 b' class TestFrameParameters(unittest.TestC'
         with self.assertRaises(TypeError):
             zstd.get_frame_parameters(None)
 
-        with self.assertRaises(TypeError):
-            zstd.get_frame_parameters(u'foobarbaz')
+        # Python 3 doesn't appear to convert unicode to Py_buffer.
+        if sys.version_info[0] >= 3:
+            with self.assertRaises(TypeError):
+                zstd.get_frame_parameters(u'foobarbaz')
+        else:
+            # CPython will convert unicode to Py_buffer. But CFFI won't.
+            if zstd.backend == 'cffi':
+                with self.assertRaises(TypeError):
+                    zstd.get_frame_parameters(u'foobarbaz')
+            else:
+                with self.assertRaises(zstd.ZstdError):
+                    zstd.get_frame_parameters(u'foobarbaz')
 
     def test_invalid_input_sizes(self):
         with self.assertRaisesRegexp(zstd.ZstdError, 'not enough data for frame'):
@@ -82,21 +142,21 b' class TestFrameParameters(unittest.TestC'
 
     def test_attributes(self):
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x00')
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
 
         # Lowest 2 bits indicate a dictionary and length. Here, the dict id is 1 byte.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x01\x00\xff')
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 255)
         self.assertFalse(params.has_checksum)
 
         # Lowest 3rd bit indicates if checksum is present.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x04\x00')
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 1024)
         self.assertEqual(params.dict_id, 0)
         self.assertTrue(params.has_checksum)
@@ -110,7 +170,7 b' class TestFrameParameters(unittest.TestC'
 
         # Window descriptor is 2nd byte after frame header.
         params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b'\x00\x40')
-        self.assertEqual(params.content_size, 0)
+        self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 0)
         self.assertFalse(params.has_checksum)
@@ -121,3 +181,22 b' class TestFrameParameters(unittest.TestC'
         self.assertEqual(params.window_size, 262144)
         self.assertEqual(params.dict_id, 15)
         self.assertTrue(params.has_checksum)
+
+    def test_input_types(self):
+        v = zstd.FRAME_HEADER + b'\x00\x00'
+
+        mutable_array = bytearray(len(v))
+        mutable_array[:] = v
+
+        sources = [
+            memoryview(v),
+            bytearray(v),
+            mutable_array,
+        ]
+
+        for source in sources:
+            params = zstd.get_frame_parameters(source)
+            self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
+            self.assertEqual(params.window_size, 1024)
+            self.assertEqual(params.dict_id, 0)
+            self.assertFalse(params.has_checksum)

contrib/python-zstandard/tests/test_data_structures_fuzzing.py

@@ -1,10 +1,7 b''
 import io
 import os
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import sys
+import unittest
 
 try:
     import hypothesis
@@ -12,7 +9,7 b' try:'
 except ImportError:
     raise unittest.SkipTest('hypothesis not available')
 
-import zstd
+import zstandard as zstd
 
 from .common import (
     make_cffi,
@@ -28,16 +25,17 b' s_hashlog = strategies.integers(min_valu'
 s_searchlog = strategies.integers(min_value=zstd.SEARCHLOG_MIN,
                                     max_value=zstd.SEARCHLOG_MAX)
 s_searchlength = strategies.integers(min_value=zstd.SEARCHLENGTH_MIN,
-                                        max_value=zstd.SEARCHLENGTH_MAX)
+                                     max_value=zstd.SEARCHLENGTH_MAX)
 s_targetlength = strategies.integers(min_value=zstd.TARGETLENGTH_MIN,
-                                        max_value=zstd.TARGETLENGTH_MAX)
+                                     max_value=2**32)
 s_strategy = strategies.sampled_from((zstd.STRATEGY_FAST,
                                         zstd.STRATEGY_DFAST,
                                         zstd.STRATEGY_GREEDY,
                                         zstd.STRATEGY_LAZY,
                                         zstd.STRATEGY_LAZY2,
                                         zstd.STRATEGY_BTLAZY2,
-                                        zstd.STRATEGY_BTOPT))
+                                        zstd.STRATEGY_BTOPT,
+                                        zstd.STRATEGY_BTULTRA))
 
 
 @make_cffi
@@ -47,24 +45,17 b' class TestCompressionParametersHypothesi'
                         s_searchlength, s_targetlength, s_strategy)
     def test_valid_init(self, windowlog, chainlog, hashlog, searchlog,
                         searchlength, targetlength, strategy):
-        # ZSTD_checkCParams moves the goal posts on us from what's advertised
-        # in the constants. So move along with them.
-        if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY):
-            searchlength += 1
-        elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
-            searchlength -= 1
-
-        p = zstd.CompressionParameters(windowlog, chainlog, hashlog,
-                                        searchlog, searchlength,
-                                        targetlength, strategy)
-
-        cctx = zstd.ZstdCompressor(compression_params=p)
-        with cctx.write_to(io.BytesIO()):
-            pass
+        zstd.ZstdCompressionParameters(window_log=windowlog,
+                                       chain_log=chainlog,
+                                       hash_log=hashlog,
+                                       search_log=searchlog,
+                                       min_match=searchlength,
+                                       target_length=targetlength,
+                                       compression_strategy=strategy)
 
     @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog,
                         s_searchlength, s_targetlength, s_strategy)
-    def test_estimate_compression_context_size(self, windowlog, chainlog,
+    def test_estimated_compression_context_size(self, windowlog, chainlog,
                                                 hashlog, searchlog,
                                                 searchlength, targetlength,
                                                 strategy):
@@ -73,7 +64,12 b' class TestCompressionParametersHypothesi'
         elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST:
             searchlength -= 1
 
-        p = zstd.CompressionParameters(windowlog, chainlog, hashlog,
-                            searchlog, searchlength,
-                            targetlength, strategy)
-        size = zstd.estimate_compression_context_size(p)
+        p = zstd.ZstdCompressionParameters(window_log=windowlog,
+                                           chain_log=chainlog,
+                                           hash_log=hashlog,
+                                           search_log=searchlog,
+                                           min_match=searchlength,
+                                           target_length=targetlength,
+                                           compression_strategy=strategy)
+        size = p.estimated_compression_context_size()
+

contrib/python-zstandard/tests/test_decompressor.py

@@ -1,16 +1,14 b''
 import io
+import os
 import random
 import struct
 import sys
+import unittest
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-import zstd
+import zstandard as zstd
 
 from .common import (
+    generate_samples,
     make_cffi,
     OpCountingBytesIO,
 )
@@ -23,35 +21,124 b' else:'
 
 
 @make_cffi
+class TestFrameHeaderSize(unittest.TestCase):
+    def test_empty(self):
+        with self.assertRaisesRegexp(
+            zstd.ZstdError, 'could not determine frame header size: Src size '
+                            'is incorrect'):
+            zstd.frame_header_size(b'')
+
+    def test_too_small(self):
+        with self.assertRaisesRegexp(
+            zstd.ZstdError, 'could not determine frame header size: Src size '
+                            'is incorrect'):
+            zstd.frame_header_size(b'foob')
+
+    def test_basic(self):
+        # It doesn't matter that it isn't a valid frame.
+        self.assertEqual(zstd.frame_header_size(b'long enough but no magic'), 6)
+
+
+@make_cffi
+class TestFrameContentSize(unittest.TestCase):
+    def test_empty(self):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                                     'error when determining content size'):
+            zstd.frame_content_size(b'')
+
+    def test_too_small(self):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                                     'error when determining content size'):
+            zstd.frame_content_size(b'foob')
+
+    def test_bad_frame(self):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                                     'error when determining content size'):
+            zstd.frame_content_size(b'invalid frame header')
+
+    def test_unknown(self):
+        cctx = zstd.ZstdCompressor(write_content_size=False)
+        frame = cctx.compress(b'foobar')
+
+        self.assertEqual(zstd.frame_content_size(frame), -1)
+
+    def test_empty(self):
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(b'')
+
+        self.assertEqual(zstd.frame_content_size(frame), 0)
+
+    def test_basic(self):
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(b'foobar')
+
+        self.assertEqual(zstd.frame_content_size(frame), 6)
+
+
+@make_cffi
+class TestDecompressor(unittest.TestCase):
+    def test_memory_size(self):
+        dctx = zstd.ZstdDecompressor()
+
+        self.assertGreater(dctx.memory_size(), 100)
+
+
+@make_cffi
 class TestDecompressor_decompress(unittest.TestCase):
     def test_empty_input(self):
         dctx = zstd.ZstdDecompressor()
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'):
+        with self.assertRaisesRegexp(zstd.ZstdError, 'error determining content size from frame header'):
             dctx.decompress(b'')
 
     def test_invalid_input(self):
         dctx = zstd.ZstdDecompressor()
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'):
+        with self.assertRaisesRegexp(zstd.ZstdError, 'error determining content size from frame header'):
             dctx.decompress(b'foobar')
 
+    def test_input_types(self):
+        cctx = zstd.ZstdCompressor(level=1)
+        compressed = cctx.compress(b'foo')
+
+        mutable_array = bytearray(len(compressed))
+        mutable_array[:] = compressed
+
+        sources = [
+            memoryview(compressed),
+            bytearray(compressed),
+            mutable_array,
+        ]
+
+        dctx = zstd.ZstdDecompressor()
+        for source in sources:
+            self.assertEqual(dctx.decompress(source), b'foo')
+
     def test_no_content_size_in_frame(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         compressed = cctx.compress(b'foobar')
 
         dctx = zstd.ZstdDecompressor()
-        with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'):
+        with self.assertRaisesRegexp(zstd.ZstdError, 'could not determine content size in frame header'):
             dctx.decompress(compressed)
 
     def test_content_size_present(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
         compressed = cctx.compress(b'foobar')
 
         dctx = zstd.ZstdDecompressor()
         decompressed = dctx.decompress(compressed)
         self.assertEqual(decompressed, b'foobar')
 
+    def test_empty_roundtrip(self):
+        cctx = zstd.ZstdCompressor()
+        compressed = cctx.compress(b'')
+
+        dctx = zstd.ZstdDecompressor()
+        decompressed = dctx.decompress(compressed)
+
+        self.assertEqual(decompressed, b'')
+
     def test_max_output_size(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         source = b'foobar' * 256
@@ -63,7 +150,8 b' class TestDecompressor_decompress(unitte'
         self.assertEqual(decompressed, source)
 
         # Input size - 1 fails
-        with self.assertRaisesRegexp(zstd.ZstdError, 'Destination buffer is too small'):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                'decompression error: did not decompress full frame'):
             dctx.decompress(compressed, max_output_size=len(source) - 1)
 
         # Input size + 1 works
@@ -94,7 +182,7 b' class TestDecompressor_decompress(unitte'
         d = zstd.train_dictionary(8192, samples)
 
         orig = b'foobar' * 16384
-        cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True)
+        cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         compressed = cctx.compress(orig)
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
@@ -113,7 +201,7 b' class TestDecompressor_decompress(unitte'
 
         sources = (b'foobar' * 8192, b'foo' * 8192, b'bar' * 8192)
         compressed = []
-        cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True)
+        cctx = zstd.ZstdCompressor(level=1, dict_data=d)
         for source in sources:
             compressed.append(cctx.compress(source))
 
@@ -122,6 +210,21 b' class TestDecompressor_decompress(unitte'
             decompressed = dctx.decompress(compressed[i])
             self.assertEqual(decompressed, sources[i])
 
+    def test_max_window_size(self):
+        with open(__file__, 'rb') as fh:
+            source = fh.read()
+
+        # If we write a content size, the decompressor engages single pass
+        # mode and the window size doesn't come into play.
+        cctx = zstd.ZstdCompressor(write_content_size=False)
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor(max_window_size=1)
+
+        with self.assertRaisesRegexp(
+            zstd.ZstdError, 'decompression error: Frame requires too much memory'):
+            dctx.decompress(frame, max_output_size=len(source))
+
 
 @make_cffi
 class TestDecompressor_copy_stream(unittest.TestCase):
@@ -186,6 +289,211 b' class TestDecompressor_copy_stream(unitt'
 
 
 @make_cffi
+class TestDecompressor_stream_reader(unittest.TestCase):
+    def test_context_manager(self):
+        dctx = zstd.ZstdDecompressor()
+
+        reader = dctx.stream_reader(b'foo')
+        with self.assertRaisesRegexp(zstd.ZstdError, 'read\(\) must be called from an active'):
+            reader.read(1)
+
+        with dctx.stream_reader(b'foo') as reader:
+            with self.assertRaisesRegexp(ValueError, 'cannot __enter__ multiple times'):
+                with reader as reader2:
+                    pass
+
+    def test_not_implemented(self):
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(b'foo') as reader:
+            with self.assertRaises(NotImplementedError):
+                reader.readline()
+
+            with self.assertRaises(NotImplementedError):
+                reader.readlines()
+
+            with self.assertRaises(NotImplementedError):
+                reader.readall()
+
+            with self.assertRaises(NotImplementedError):
+                iter(reader)
+
+            with self.assertRaises(NotImplementedError):
+                next(reader)
+
+            with self.assertRaises(io.UnsupportedOperation):
+                reader.write(b'foo')
+
+            with self.assertRaises(io.UnsupportedOperation):
+                reader.writelines([])
+
+    def test_constant_methods(self):
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(b'foo') as reader:
+            self.assertTrue(reader.readable())
+            self.assertFalse(reader.writable())
+            self.assertTrue(reader.seekable())
+            self.assertFalse(reader.isatty())
+            self.assertIsNone(reader.flush())
+
+    def test_read_closed(self):
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(b'foo') as reader:
+            reader.close()
+            with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+                reader.read(1)
+
+    def test_bad_read_size(self):
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(b'foo') as reader:
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
+                reader.read(-1)
+
+            with self.assertRaisesRegexp(ValueError, 'cannot read negative or size 0 amounts'):
+                reader.read(0)
+
+    def test_read_buffer(self):
+        cctx = zstd.ZstdCompressor()
+
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(frame) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            # We should get entire frame in one read.
+            result = reader.read(8192)
+            self.assertEqual(result, source)
+            self.assertEqual(reader.tell(), len(source))
+
+            # Read after EOF should return empty bytes.
+            self.assertEqual(reader.read(), b'')
+            self.assertEqual(reader.tell(), len(result))
+
+        self.assertTrue(reader.closed())
+
+    def test_read_buffer_small_chunks(self):
+        cctx = zstd.ZstdCompressor()
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+        chunks = []
+
+        with dctx.stream_reader(frame, read_size=1) as reader:
+            while True:
+                chunk = reader.read(1)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+                self.assertEqual(reader.tell(), sum(map(len, chunks)))
+
+        self.assertEqual(b''.join(chunks), source)
+
+    def test_read_stream(self):
+        cctx = zstd.ZstdCompressor()
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+        with dctx.stream_reader(io.BytesIO(frame)) as reader:
+            self.assertEqual(reader.tell(), 0)
+
+            chunk = reader.read(8192)
+            self.assertEqual(chunk, source)
+            self.assertEqual(reader.tell(), len(source))
+            self.assertEqual(reader.read(), b'')
+            self.assertEqual(reader.tell(), len(source))
+
+    def test_read_stream_small_chunks(self):
+        cctx = zstd.ZstdCompressor()
+        source = b''.join([b'foo' * 60, b'bar' * 60, b'baz' * 60])
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+        chunks = []
+
+        with dctx.stream_reader(io.BytesIO(frame), read_size=1) as reader:
+            while True:
+                chunk = reader.read(1)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+                self.assertEqual(reader.tell(), sum(map(len, chunks)))
+
+        self.assertEqual(b''.join(chunks), source)
+
+    def test_read_after_exit(self):
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(b'foo' * 60)
+
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(frame) as reader:
+            while reader.read(16):
+                pass
+
+        with self.assertRaisesRegexp(zstd.ZstdError, 'read\(\) must be called from an active'):
+            reader.read(10)
+
+    def test_illegal_seeks(self):
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(b'foo' * 60)
+
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(frame) as reader:
+            with self.assertRaisesRegexp(ValueError,
+                                         'cannot seek to negative position'):
+                reader.seek(-1, os.SEEK_SET)
+
+            reader.read(1)
+
+            with self.assertRaisesRegexp(
+                ValueError, 'cannot seek zstd decompression stream backwards'):
+                reader.seek(0, os.SEEK_SET)
+
+            with self.assertRaisesRegexp(
+                ValueError, 'cannot seek zstd decompression stream backwards'):
+                reader.seek(-1, os.SEEK_CUR)
+
+            with self.assertRaisesRegexp(
+                ValueError,
+                'zstd decompression streams cannot be seeked with SEEK_END'):
+                reader.seek(0, os.SEEK_END)
+
+            reader.close()
+
+            with self.assertRaisesRegexp(ValueError, 'stream is closed'):
+                reader.seek(4, os.SEEK_SET)
+
+        with self.assertRaisesRegexp(
+            zstd.ZstdError, 'seek\(\) must be called from an active context'):
+            reader.seek(0)
+
+    def test_seek(self):
+        source = b'foobar' * 60
+        cctx = zstd.ZstdCompressor()
+        frame = cctx.compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(frame) as reader:
+            reader.seek(3)
+            self.assertEqual(reader.read(3), b'bar')
+
+            reader.seek(4, os.SEEK_CUR)
+            self.assertEqual(reader.read(2), b'ar')
+
+
+@make_cffi
 class TestDecompressor_decompressobj(unittest.TestCase):
     def test_simple(self):
         data = zstd.ZstdCompressor(level=1).compress(b'foobar')
@@ -194,6 +502,24 b' class TestDecompressor_decompressobj(uni'
         dobj = dctx.decompressobj()
         self.assertEqual(dobj.decompress(data), b'foobar')
 
+    def test_input_types(self):
+        compressed = zstd.ZstdCompressor(level=1).compress(b'foo')
+
+        dctx = zstd.ZstdDecompressor()
+
+        mutable_array = bytearray(len(compressed))
+        mutable_array[:] = compressed
+
+        sources = [
+            memoryview(compressed),
+            bytearray(compressed),
+            mutable_array,
+        ]
+
+        for source in sources:
+            dobj = dctx.decompressobj()
+            self.assertEqual(dobj.decompress(source), b'foo')
+
     def test_reuse(self):
         data = zstd.ZstdCompressor(level=1).compress(b'foobar')
 
@@ -204,22 +530,58 b' class TestDecompressor_decompressobj(uni'
         with self.assertRaisesRegexp(zstd.ZstdError, 'cannot use a decompressobj'):
             dobj.decompress(data)
 
+    def test_bad_write_size(self):
+        dctx = zstd.ZstdDecompressor()
+
+        with self.assertRaisesRegexp(ValueError, 'write_size must be positive'):
+            dctx.decompressobj(write_size=0)
+
+    def test_write_size(self):
+        source = b'foo' * 64 + b'bar' * 128
+        data = zstd.ZstdCompressor(level=1).compress(source)
+
+        dctx = zstd.ZstdDecompressor()
+
+        for i in range(128):
+            dobj = dctx.decompressobj(write_size=i + 1)
+            self.assertEqual(dobj.decompress(data), source)
 
 def decompress_via_writer(data):
     buffer = io.BytesIO()
     dctx = zstd.ZstdDecompressor()
-    with dctx.write_to(buffer) as decompressor:
+    with dctx.stream_writer(buffer) as decompressor:
         decompressor.write(data)
     return buffer.getvalue()
 
 
 @make_cffi
-class TestDecompressor_write_to(unittest.TestCase):
+class TestDecompressor_stream_writer(unittest.TestCase):
     def test_empty_roundtrip(self):
         cctx = zstd.ZstdCompressor()
         empty = cctx.compress(b'')
         self.assertEqual(decompress_via_writer(empty), b'')
 
+    def test_input_types(self):
+        cctx = zstd.ZstdCompressor(level=1)
+        compressed = cctx.compress(b'foo')
+
+        mutable_array = bytearray(len(compressed))
+        mutable_array[:] = compressed
+
+        sources = [
+            memoryview(compressed),
+            bytearray(compressed),
+            mutable_array,
+        ]
+
+        dctx = zstd.ZstdDecompressor()
+        for source in sources:
+            buffer = io.BytesIO()
+            with dctx.stream_writer(buffer) as decompressor:
+                decompressor.write(source)
+
+            self.assertEqual(buffer.getvalue(), b'foo')
+
     def test_large_roundtrip(self):
         chunks = []
         for i in range(255):
@@ -242,7 +604,7 b' class TestDecompressor_write_to(unittest'
 
         buffer = io.BytesIO()
         dctx = zstd.ZstdDecompressor()
-        with dctx.write_to(buffer) as decompressor:
+        with dctx.stream_writer(buffer) as decompressor:
             pos = 0
             while pos < len(compressed):
                 pos2 = pos + 8192
@@ -262,14 +624,14 b' class TestDecompressor_write_to(unittest'
         orig = b'foobar' * 16384
         buffer = io.BytesIO()
         cctx = zstd.ZstdCompressor(dict_data=d)
-        with cctx.write_to(buffer) as compressor:
-            self.assertEqual(compressor.write(orig), 1544)
+        with cctx.stream_writer(buffer) as compressor:
+            self.assertEqual(compressor.write(orig), 0)
 
         compressed = buffer.getvalue()
         buffer = io.BytesIO()
 
         dctx = zstd.ZstdDecompressor(dict_data=d)
-        with dctx.write_to(buffer) as decompressor:
+        with dctx.stream_writer(buffer) as decompressor:
             self.assertEqual(decompressor.write(compressed), len(orig))
 
         self.assertEqual(buffer.getvalue(), orig)
@@ -277,7 +639,7 b' class TestDecompressor_write_to(unittest'
     def test_memory_size(self):
         dctx = zstd.ZstdDecompressor()
         buffer = io.BytesIO()
-        with dctx.write_to(buffer) as decompressor:
+        with dctx.stream_writer(buffer) as decompressor:
             size = decompressor.memory_size()
 
         self.assertGreater(size, 100000)
@@ -286,7 +648,7 b' class TestDecompressor_write_to(unittest'
         source = zstd.ZstdCompressor().compress(b'foobarfoobar')
         dest = OpCountingBytesIO()
         dctx = zstd.ZstdDecompressor()
-        with dctx.write_to(dest, write_size=1) as decompressor:
+        with dctx.stream_writer(dest, write_size=1) as decompressor:
             s = struct.Struct('>B')
             for c in source:
                 if not isinstance(c, str):
@@ -298,29 +660,29 b' class TestDecompressor_write_to(unittest'
 
 
 @make_cffi
-class TestDecompressor_read_from(unittest.TestCase):
+class TestDecompressor_read_to_iter(unittest.TestCase):
     def test_type_validation(self):
         dctx = zstd.ZstdDecompressor()
 
         # Object with read() works.
-        dctx.read_from(io.BytesIO())
+        dctx.read_to_iter(io.BytesIO())
 
         # Buffer protocol works.
-        dctx.read_from(b'foobar')
+        dctx.read_to_iter(b'foobar')
 
         with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'):
-            b''.join(dctx.read_from(True))
+            b''.join(dctx.read_to_iter(True))
 
     def test_empty_input(self):
         dctx = zstd.ZstdDecompressor()
 
         source = io.BytesIO()
-        it = dctx.read_from(source)
+        it = dctx.read_to_iter(source)
         # TODO this is arguably wrong. Should get an error about missing frame foo.
         with self.assertRaises(StopIteration):
             next(it)
 
-        it = dctx.read_from(b'')
+        it = dctx.read_to_iter(b'')
         with self.assertRaises(StopIteration):
             next(it)
 
@@ -328,11 +690,11 b' class TestDecompressor_read_from(unittes'
         dctx = zstd.ZstdDecompressor()
 
         source = io.BytesIO(b'foobar')
-        it = dctx.read_from(source)
+        it = dctx.read_to_iter(source)
         with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'):
             next(it)
 
-        it = dctx.read_from(b'foobar')
+        it = dctx.read_to_iter(b'foobar')
         with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'):
             next(it)
 
@@ -344,7 +706,7 b' class TestDecompressor_read_from(unittes'
         source.seek(0)
 
         dctx = zstd.ZstdDecompressor()
-        it = dctx.read_from(source)
+        it = dctx.read_to_iter(source)
 
         # No chunks should be emitted since there is no data.
         with self.assertRaises(StopIteration):
@@ -358,17 +720,17 b' class TestDecompressor_read_from(unittes'
         dctx = zstd.ZstdDecompressor()
 
         with self.assertRaisesRegexp(ValueError, 'skip_bytes must be smaller than read_size'):
-            b''.join(dctx.read_from(b'', skip_bytes=1, read_size=1))
+            b''.join(dctx.read_to_iter(b'', skip_bytes=1, read_size=1))
 
         with self.assertRaisesRegexp(ValueError, 'skip_bytes larger than first input chunk'):
-            b''.join(dctx.read_from(b'foobar', skip_bytes=10))
+            b''.join(dctx.read_to_iter(b'foobar', skip_bytes=10))
 
     def test_skip_bytes(self):
         cctx = zstd.ZstdCompressor(write_content_size=False)
         compressed = cctx.compress(b'foobar')
 
         dctx = zstd.ZstdDecompressor()
-        output = b''.join(dctx.read_from(b'hdr' + compressed, skip_bytes=3))
+        output = b''.join(dctx.read_to_iter(b'hdr' + compressed, skip_bytes=3))
         self.assertEqual(output, b'foobar')
 
     def test_large_output(self):
@@ -382,7 +744,7 b' class TestDecompressor_read_from(unittes'
         compressed.seek(0)
 
         dctx = zstd.ZstdDecompressor()
-        it = dctx.read_from(compressed)
+        it = dctx.read_to_iter(compressed)
 
         chunks = []
         chunks.append(next(it))
@@ -395,7 +757,7 b' class TestDecompressor_read_from(unittes'
         self.assertEqual(decompressed, source.getvalue())
 
         # And again with buffer protocol.
-        it = dctx.read_from(compressed.getvalue())
+        it = dctx.read_to_iter(compressed.getvalue())
         chunks = []
         chunks.append(next(it))
         chunks.append(next(it))
@@ -406,12 +768,13 b' class TestDecompressor_read_from(unittes'
         decompressed = b''.join(chunks)
         self.assertEqual(decompressed, source.getvalue())
 
+    @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
     def test_large_input(self):
         bytes = list(struct.Struct('>B').pack(i) for i in range(256))
         compressed = io.BytesIO()
         input_size = 0
         cctx = zstd.ZstdCompressor(level=1)
-        with cctx.write_to(compressed) as compressor:
+        with cctx.stream_writer(compressed) as compressor:
             while True:
                 compressor.write(random.choice(bytes))
                 input_size += 1
@@ -426,7 +789,7 b' class TestDecompressor_read_from(unittes'
                            zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE)
 
         dctx = zstd.ZstdDecompressor()
-        it = dctx.read_from(compressed)
+        it = dctx.read_to_iter(compressed)
 
         chunks = []
         chunks.append(next(it))
@@ -440,7 +803,7 b' class TestDecompressor_read_from(unittes'
         self.assertEqual(len(decompressed), input_size)
 
         # And again with buffer protocol.
-        it = dctx.read_from(compressed.getvalue())
+        it = dctx.read_to_iter(compressed.getvalue())
 
         chunks = []
         chunks.append(next(it))
@@ -460,7 +823,7 b' class TestDecompressor_read_from(unittes'
         source = io.BytesIO()
 
         compressed = io.BytesIO()
-        with cctx.write_to(compressed) as compressor:
+        with cctx.stream_writer(compressed) as compressor:
             for i in range(256):
                 chunk = b'\0' * 1024
                 compressor.write(chunk)
@@ -473,17 +836,34 b' class TestDecompressor_read_from(unittes'
         self.assertEqual(simple, source.getvalue())
 
         compressed.seek(0)
-        streamed = b''.join(dctx.read_from(compressed))
+        streamed = b''.join(dctx.read_to_iter(compressed))
         self.assertEqual(streamed, source.getvalue())
 
     def test_read_write_size(self):
         source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b'foobarfoobar'))
         dctx = zstd.ZstdDecompressor()
-        for chunk in dctx.read_from(source, read_size=1, write_size=1):
+        for chunk in dctx.read_to_iter(source, read_size=1, write_size=1):
             self.assertEqual(len(chunk), 1)
 
         self.assertEqual(source._read_count, len(source.getvalue()))
 
+    def test_magic_less(self):
+        params = zstd.CompressionParameters.from_level(
+            1, format=zstd.FORMAT_ZSTD1_MAGICLESS)
+        cctx = zstd.ZstdCompressor(compression_params=params)
+        frame = cctx.compress(b'foobar')
+
+        self.assertNotEqual(frame[0:4], b'\x28\xb5\x2f\xfd')
+
+        dctx = zstd.ZstdDecompressor()
+        with self.assertRaisesRegexp(
+            zstd.ZstdError, 'error determining content size from frame header'):
+            dctx.decompress(frame)
+
+        dctx = zstd.ZstdDecompressor(format=zstd.FORMAT_ZSTD1_MAGICLESS)
+        res = b''.join(dctx.read_to_iter(frame))
+        self.assertEqual(res, b'foobar')
+
 
 @make_cffi
 class TestDecompressor_content_dict_chain(unittest.TestCase):
@@ -511,19 +891,20 b' class TestDecompressor_content_dict_chai'
         with self.assertRaisesRegexp(ValueError, 'chunk 0 is not a valid zstd frame'):
             dctx.decompress_content_dict_chain([b'foo' * 8])
 
-        no_size = zstd.ZstdCompressor().compress(b'foo' * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b'foo' * 64)
 
         with self.assertRaisesRegexp(ValueError, 'chunk 0 missing content size in frame'):
             dctx.decompress_content_dict_chain([no_size])
 
         # Corrupt first frame.
-        frame = zstd.ZstdCompressor(write_content_size=True).compress(b'foo' * 64)
+        frame = zstd.ZstdCompressor().compress(b'foo' * 64)
         frame = frame[0:12] + frame[15:]
-        with self.assertRaisesRegexp(zstd.ZstdError, 'could not decompress chunk 0'):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                                     'chunk 0 did not decompress full frame'):
             dctx.decompress_content_dict_chain([frame])
 
     def test_bad_subsequent_input(self):
-        initial = zstd.ZstdCompressor(write_content_size=True).compress(b'foo' * 64)
+        initial = zstd.ZstdCompressor().compress(b'foo' * 64)
 
         dctx = zstd.ZstdDecompressor()
 
@@ -539,17 +920,17 b' class TestDecompressor_content_dict_chai'
         with self.assertRaisesRegexp(ValueError, 'chunk 1 is not a valid zstd frame'):
             dctx.decompress_content_dict_chain([initial, b'foo' * 8])
 
-        no_size = zstd.ZstdCompressor().compress(b'foo' * 64)
+        no_size = zstd.ZstdCompressor(write_content_size=False).compress(b'foo' * 64)
 
         with self.assertRaisesRegexp(ValueError, 'chunk 1 missing content size in frame'):
             dctx.decompress_content_dict_chain([initial, no_size])
 
         # Corrupt second frame.
-        cctx = zstd.ZstdCompressor(write_content_size=True, dict_data=zstd.ZstdCompressionDict(b'foo' * 64))
+        cctx = zstd.ZstdCompressor(dict_data=zstd.ZstdCompressionDict(b'foo' * 64))
         frame = cctx.compress(b'bar' * 64)
         frame = frame[0:12] + frame[15:]
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'could not decompress chunk 1'):
+        with self.assertRaisesRegexp(zstd.ZstdError, 'chunk 1 did not decompress full frame'):
             dctx.decompress_content_dict_chain([initial, frame])
 
     def test_simple(self):
@@ -562,10 +943,10 b' class TestDecompressor_content_dict_chai'
         ]
 
         chunks = []
-        chunks.append(zstd.ZstdCompressor(write_content_size=True).compress(original[0]))
+        chunks.append(zstd.ZstdCompressor().compress(original[0]))
         for i, chunk in enumerate(original[1:]):
             d = zstd.ZstdCompressionDict(original[i])
-            cctx = zstd.ZstdCompressor(dict_data=d, write_content_size=True)
+            cctx = zstd.ZstdCompressor(dict_data=d)
             chunks.append(cctx.compress(chunk))
 
         for i in range(1, len(original)):
@@ -594,7 +975,7 b' class TestDecompressor_multi_decompress_'
             dctx.multi_decompress_to_buffer([b'foobarbaz'])
 
     def test_list_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
 
         original = [b'foo' * 4, b'bar' * 6]
         frames = [cctx.compress(d) for d in original]
@@ -614,7 +995,7 b' class TestDecompressor_multi_decompress_'
         self.assertEqual(len(result[1]), 18)
 
     def test_list_input_frame_sizes(self):
-        cctx = zstd.ZstdCompressor(write_content_size=False)
+        cctx = zstd.ZstdCompressor()
 
         original = [b'foo' * 4, b'bar' * 6, b'baz' * 8]
         frames = [cctx.compress(d) for d in original]
@@ -630,7 +1011,7 b' class TestDecompressor_multi_decompress_'
             self.assertEqual(result[i].tobytes(), data)
 
     def test_buffer_with_segments_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
 
         original = [b'foo' * 4, b'bar' * 6]
         frames = [cctx.compress(d) for d in original]
@@ -669,7 +1050,7 b' class TestDecompressor_multi_decompress_'
             self.assertEqual(result[i].tobytes(), data)
 
     def test_buffer_with_segments_collection_input(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
 
         original = [
             b'foo0' * 2,
@@ -711,8 +1092,18 b' class TestDecompressor_multi_decompress_'
         for i in range(5):
             self.assertEqual(decompressed[i].tobytes(), original[i])
 
+    def test_dict(self):
+        d = zstd.train_dictionary(16384, generate_samples(), k=64, d=16)
+
+        cctx = zstd.ZstdCompressor(dict_data=d, level=1)
+        frames = [cctx.compress(s) for s in generate_samples()]
+
+        dctx = zstd.ZstdDecompressor(dict_data=d)
+        result = dctx.multi_decompress_to_buffer(frames)
+        self.assertEqual([o.tobytes() for o in result], generate_samples())
+
     def test_multiple_threads(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
 
         frames = []
         frames.extend(cctx.compress(b'x' * 64) for i in range(256))
@@ -727,15 +1118,22 b' class TestDecompressor_multi_decompress_'
         self.assertEqual(result[256].tobytes(), b'y' * 64)
 
     def test_item_failure(self):
-        cctx = zstd.ZstdCompressor(write_content_size=True)
+        cctx = zstd.ZstdCompressor()
         frames = [cctx.compress(b'x' * 128), cctx.compress(b'y' * 128)]
 
-        frames[1] = frames[1] + b'extra'
+        frames[1] = frames[1][0:15] + b'extra' + frames[1][15:]
 
         dctx = zstd.ZstdDecompressor()
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'error decompressing item 1: Src size incorrect'):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                                     'error decompressing item 1: ('
+                                     'Corrupted block|'
+                                     'Destination buffer is too small)'):
             dctx.multi_decompress_to_buffer(frames)
 
-        with self.assertRaisesRegexp(zstd.ZstdError, 'error decompressing item 1: Src size incorrect'):
+        with self.assertRaisesRegexp(zstd.ZstdError,
+                            'error decompressing item 1: ('
+                            'Corrupted block|'
+                            'Destination buffer is too small)'):
             dctx.multi_decompress_to_buffer(frames, threads=2)
+

contrib/python-zstandard/tests/test_decompressor_fuzzing.py

@@ -1,10 +1,6 b''
 import io
 import os
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import unittest
 
 try:
     import hypothesis
@@ -12,7 +8,7 b' try:'
 except ImportError:
     raise unittest.SkipTest('hypothesis not available')
 
-import zstd
+import zstandard as zstd
 
 from . common import (
     make_cffi,
@@ -22,15 +18,96 b' from . common import ('
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
-class TestDecompressor_write_to_fuzzing(unittest.TestCase):
+class TestDecompressor_stream_reader_fuzzing(unittest.TestCase):
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_stream_source_read_variance(self, original, level, source_read_size,
+                                         read_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+        frame = cctx.compress(original)
+
+        dctx = zstd.ZstdDecompressor()
+        source = io.BytesIO(frame)
+
+        chunks = []
+        with dctx.stream_reader(source, read_size=source_read_size) as reader:
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                chunk = reader.read(read_size)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      source_read_size=strategies.integers(1, 16384),
+                      read_sizes=strategies.data())
+    def test_buffer_source_read_variance(self, original, level, source_read_size,
+                                         read_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+        frame = cctx.compress(original)
+
+        dctx = zstd.ZstdDecompressor()
+        chunks = []
+
+        with dctx.stream_reader(frame, read_size=source_read_size) as reader:
+            while True:
+                read_size = read_sizes.draw(strategies.integers(1, 16384))
+                chunk = reader.read(read_size)
+                if not chunk:
+                    break
+
+                chunks.append(chunk)
+
+        self.assertEqual(b''.join(chunks), original)
+
+    @hypothesis.settings(
+        suppress_health_check=[hypothesis.HealthCheck.large_base_example])
+    @hypothesis.given(
+        original=strategies.sampled_from(random_input_data()),
+        level=strategies.integers(min_value=1, max_value=5),
+        source_read_size=strategies.integers(1, 16384),
+        seek_amounts=strategies.data(),
+        read_sizes=strategies.data())
+    def test_relative_seeks(self, original, level, source_read_size, seek_amounts,
+                            read_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+        frame = cctx.compress(original)
+
+        dctx = zstd.ZstdDecompressor()
+
+        with dctx.stream_reader(frame, read_size=source_read_size) as reader:
+            while True:
+                amount = seek_amounts.draw(strategies.integers(0, 16384))
+                reader.seek(amount, os.SEEK_CUR)
+
+                offset = reader.tell()
+                read_amount = read_sizes.draw(strategies.integers(1, 16384))
+                chunk = reader.read(read_amount)
+
+                if not chunk:
+                    break
+
+                self.assertEqual(original[offset:offset + len(chunk)], chunk)
+
+
[e-mail]('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
+@make_cffi
+class TestDecompressor_stream_writer_fuzzing(unittest.TestCase):
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
                       write_size=strategies.integers(min_value=1, max_value=8192),
-                      input_sizes=strategies.streaming(
-                          strategies.integers(min_value=1, max_value=4096)))
+                      input_sizes=strategies.data())
     def test_write_size_variance(self, original, level, write_size, input_sizes):
-        input_sizes = iter(input_sizes)
-
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -38,9 +115,10 b' class TestDecompressor_write_to_fuzzing('
         source = io.BytesIO(frame)
         dest = io.BytesIO()
 
-        with dctx.write_to(dest, write_size=write_size) as decompressor:
+        with dctx.stream_writer(dest, write_size=write_size) as decompressor:
             while True:
-                chunk = source.read(next(input_sizes))
+                input_size = input_sizes.draw(strategies.integers(1, 4096))
+                chunk = source.read(input_size)
                 if not chunk:
                     break
 
@@ -74,11 +152,8 b' class TestDecompressor_copy_stream_fuzzi'
 class TestDecompressor_decompressobj_fuzzing(unittest.TestCase):
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
-                      chunk_sizes=strategies.streaming(
-                          strategies.integers(min_value=1, max_value=4096)))
+                      chunk_sizes=strategies.data())
     def test_random_input_sizes(self, original, level, chunk_sizes):
-        chunk_sizes = iter(chunk_sizes)
-
         cctx = zstd.ZstdCompressor(level=level)
         frame = cctx.compress(original)
 
@@ -89,7 +164,33 b' class TestDecompressor_decompressobj_fuz'
 
         chunks = []
         while True:
-            chunk = source.read(next(chunk_sizes))
+            chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
+            chunk = source.read(chunk_size)
+            if not chunk:
+                break
+
+            chunks.append(dobj.decompress(chunk))
+
+        self.assertEqual(b''.join(chunks), original)
+
+    @hypothesis.given(original=strategies.sampled_from(random_input_data()),
+                      level=strategies.integers(min_value=1, max_value=5),
+                      write_size=strategies.integers(min_value=1,
+                                                     max_value=4 * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE),
+                      chunk_sizes=strategies.data())
+    def test_random_output_sizes(self, original, level, write_size, chunk_sizes):
+        cctx = zstd.ZstdCompressor(level=level)
+        frame = cctx.compress(original)
+
+        source = io.BytesIO(frame)
+
+        dctx = zstd.ZstdDecompressor()
+        dobj = dctx.decompressobj(write_size=write_size)
+
+        chunks = []
+        while True:
+            chunk_size = chunk_sizes.draw(strategies.integers(1, 4096))
+            chunk = source.read(chunk_size)
             if not chunk:
                 break
 
@@ -100,7 +201,7 b' class TestDecompressor_decompressobj_fuz'
 
 @unittest.skipUnless('ZSTD_SLOW_TESTS' in os.environ, 'ZSTD_SLOW_TESTS not set')
 @make_cffi
-class TestDecompressor_read_from_fuzzing(unittest.TestCase):
+class TestDecompressor_read_to_iter_fuzzing(unittest.TestCase):
     @hypothesis.given(original=strategies.sampled_from(random_input_data()),
                       level=strategies.integers(min_value=1, max_value=5),
                       read_size=strategies.integers(min_value=1, max_value=4096),
@@ -112,7 +213,7 b' class TestDecompressor_read_from_fuzzing'
         source = io.BytesIO(frame)
 
         dctx = zstd.ZstdDecompressor()
-        chunks = list(dctx.read_from(source, read_size=read_size, write_size=write_size))
+        chunks = list(dctx.read_to_iter(source, read_size=read_size, write_size=write_size))
 
         self.assertEqual(b''.join(chunks), original)
 

contrib/python-zstandard/tests/test_estimate_sizes.py

@@ -1,9 +1,6 b''
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import unittest
 
-import zstd
+import zstandard as zstd
 
 from . common import (
     make_cffi,
@@ -16,7 +13,3 b' class TestSizes(unittest.TestCase):'
         size = zstd.estimate_decompression_context_size()
         self.assertGreater(size, 100000)
 
-    def test_compression_size(self):
-        params = zstd.get_compression_parameters(3)
-        size = zstd.estimate_compression_context_size(params)
-        self.assertGreater(size, 100000)

contrib/python-zstandard/tests/test_module_attributes.py

@@ -1,11 +1,8 b''
 from __future__ import unicode_literals
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
+import unittest
 
-import zstd
+import zstandard as zstd
 
 from . common import (
     make_cffi,
@@ -15,7 +12,7 b' from . common import ('
 @make_cffi
 class TestModuleAttributes(unittest.TestCase):
     def test_version(self):
-        self.assertEqual(zstd.ZSTD_VERSION, (1, 1, 3))
+        self.assertEqual(zstd.ZSTD_VERSION, (1, 3, 4))
 
     def test_constants(self):
         self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22)
@@ -23,6 +20,8 b' class TestModuleAttributes(unittest.Test'
 
     def test_hasattr(self):
         attrs = (
+            'CONTENTSIZE_UNKNOWN',
+            'CONTENTSIZE_ERROR',
             'COMPRESSION_RECOMMENDED_INPUT_SIZE',
             'COMPRESSION_RECOMMENDED_OUTPUT_SIZE',
             'DECOMPRESSION_RECOMMENDED_INPUT_SIZE',
@@ -40,7 +39,9 b' class TestModuleAttributes(unittest.Test'
             'SEARCHLENGTH_MIN',
             'SEARCHLENGTH_MAX',
             'TARGETLENGTH_MIN',
-            'TARGETLENGTH_MAX',
+            'LDM_MINMATCH_MIN',
+            'LDM_MINMATCH_MAX',
+            'LDM_BUCKETSIZELOG_MAX',
             'STRATEGY_FAST',
             'STRATEGY_DFAST',
             'STRATEGY_GREEDY',
@@ -48,6 +49,10 b' class TestModuleAttributes(unittest.Test'
             'STRATEGY_LAZY2',
             'STRATEGY_BTLAZY2',
             'STRATEGY_BTOPT',
+            'STRATEGY_BTULTRA',
+            'DICT_TYPE_AUTO',
+            'DICT_TYPE_RAWCONTENT',
+            'DICT_TYPE_FULLDICT',
         )
 
         for a in attrs:

contrib/python-zstandard/tests/test_train_dictionary.py

@@ -1,13 +1,11 b''
+import struct
 import sys
+import unittest
 
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-import zstd
+import zstandard as zstd
 
 from . common import (
+    generate_samples,
     make_cffi,
 )
 
@@ -30,55 +28,18 b' class TestTrainDictionary(unittest.TestC'
         with self.assertRaises(ValueError):
             zstd.train_dictionary(8192, [u'foo'])
 
-    def test_basic(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'bar' * 64)
-            samples.append(b'foobar' * 64)
-            samples.append(b'baz' * 64)
-            samples.append(b'foobaz' * 64)
-            samples.append(b'bazfoo' * 64)
+    def test_no_params(self):
+        d = zstd.train_dictionary(8192, generate_samples())
+        self.assertIsInstance(d.dict_id(), int_type)
 
-        d = zstd.train_dictionary(8192, samples)
-        self.assertLessEqual(len(d), 8192)
-
-        dict_id = d.dict_id()
-        self.assertIsInstance(dict_id, int_type)
+        # The dictionary ID may be different across platforms.
+        expected = b'\x37\xa4\x30\xec' + struct.pack('<I', d.dict_id())
 
         data = d.as_bytes()
-        self.assertEqual(data[0:4], b'\x37\xa4\x30\xec')
-
-    def test_set_dict_id(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'foobar' * 64)
-
-        d = zstd.train_dictionary(8192, samples, dict_id=42)
-        self.assertEqual(d.dict_id(), 42)
-
-
-@make_cffi
-class TestTrainCoverDictionary(unittest.TestCase):
-    def test_no_args(self):
-        with self.assertRaises(TypeError):
-            zstd.train_cover_dictionary()
-
-    def test_bad_args(self):
-        with self.assertRaises(TypeError):
-            zstd.train_cover_dictionary(8192, u'foo')
-
-        with self.assertRaises(ValueError):
-            zstd.train_cover_dictionary(8192, [u'foo'])
+        self.assertEqual(data[0:8], expected)
 
     def test_basic(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'foobar' * 64)
-
-        d = zstd.train_cover_dictionary(8192, samples, k=64, d=16)
+        d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
         self.assertIsInstance(d.dict_id(), int_type)
 
         data = d.as_bytes()
@@ -88,23 +49,39 b' class TestTrainCoverDictionary(unittest.'
         self.assertEqual(d.d, 16)
 
     def test_set_dict_id(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'foobar' * 64)
-
-        d = zstd.train_cover_dictionary(8192, samples, k=64, d=16,
-                                        dict_id=42)
+        d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16,
+                                  dict_id=42)
         self.assertEqual(d.dict_id(), 42)
 
     def test_optimize(self):
-        samples = []
-        for i in range(128):
-            samples.append(b'foo' * 64)
-            samples.append(b'foobar' * 64)
+        d = zstd.train_dictionary(8192, generate_samples(), threads=-1, steps=1,
+                                  d=16)
+
+        self.assertEqual(d.k, 50)
+        self.assertEqual(d.d, 16)
+
+@make_cffi
+class TestCompressionDict(unittest.TestCase):
+    def test_bad_mode(self):
+        with self.assertRaisesRegexp(ValueError, 'invalid dictionary load mode'):
+            zstd.ZstdCompressionDict(b'foo', dict_type=42)
+
+    def test_bad_precompute_compress(self):
+        d = zstd.train_dictionary(8192, generate_samples(), k=64, d=16)
 
-        d = zstd.train_cover_dictionary(8192, samples, optimize=True,
-                                        threads=-1, steps=1, d=16)
+        with self.assertRaisesRegexp(ValueError, 'must specify one of level or '):
+            d.precompute_compress()
+
+        with self.assertRaisesRegexp(ValueError, 'must only specify one of level or '):
+            d.precompute_compress(level=3,
+                                  compression_params=zstd.CompressionParameters())
 
-        self.assertEqual(d.k, 16)
-        self.assertEqual(d.d, 16)
+    def test_precompute_compress_rawcontent(self):
+        d = zstd.ZstdCompressionDict(b'dictcontent' * 64,
+                                     dict_type=zstd.DICT_TYPE_RAWCONTENT)
+        d.precompute_compress(level=1)
+
+        d = zstd.ZstdCompressionDict(b'dictcontent' * 64,
+                                     dict_type=zstd.DICT_TYPE_FULLDICT)
+        with self.assertRaisesRegexp(zstd.ZstdError, 'unable to precompute dictionary'):
+            d.precompute_compress(level=1)

contrib/python-zstandard/zstd.c

@@ -20,12 +20,6 b''
 
 PyObject *ZstdError;
 
-PyDoc_STRVAR(estimate_compression_context_size__doc__,
-"estimate_compression_context_size(compression_parameters)\n"
-"\n"
-"Give the amount of memory allocated for a compression context given a\n"
-"CompressionParameters instance");
-
 PyDoc_STRVAR(estimate_decompression_context_size__doc__,
 "estimate_decompression_context_size()\n"
 "\n"
@@ -36,11 +30,101 b' static PyObject* estimate_decompression_'
 	return PyLong_FromSize_t(ZSTD_estimateDCtxSize());
 }
 
-PyDoc_STRVAR(get_compression_parameters__doc__,
-"get_compression_parameters(compression_level[, source_size[, dict_size]])\n"
+PyDoc_STRVAR(frame_content_size__doc__,
+"frame_content_size(data)\n"
 "\n"
-"Obtains a ``CompressionParameters`` instance from a compression level and\n"
-"optional input size and dictionary size");
+"Obtain the decompressed size of a frame."
+);
+
+static PyObject* frame_content_size(PyObject* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"source",
+		NULL
+	};
+
+	Py_buffer source;
+	PyObject* result = NULL;
+	unsigned long long size;
+
+#if PY_MAJOR_VERSION >= 3
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:frame_content_size",
+#else
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:frame_content_size",
+#endif
+		kwlist, &source)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	size = ZSTD_getFrameContentSize(source.buf, source.len);
+
+	if (size == ZSTD_CONTENTSIZE_ERROR) {
+		PyErr_SetString(ZstdError, "error when determining content size");
+	}
+	else if (size == ZSTD_CONTENTSIZE_UNKNOWN) {
+		result = PyLong_FromLong(-1);
+	}
+	else {
+		result = PyLong_FromUnsignedLongLong(size);
+	}
+
+finally:
+	PyBuffer_Release(&source);
+
+	return result;
+}
+
+PyDoc_STRVAR(frame_header_size__doc__,
+"frame_header_size(data)\n"
+"\n"
+"Obtain the size of a frame header.\n"
+);
+
+static PyObject* frame_header_size(PyObject* self, PyObject* args, PyObject* kwargs) {
+	static char* kwlist[] = {
+		"source",
+		NULL
+	};
+
+	Py_buffer source;
+	PyObject* result = NULL;
+	size_t zresult;
+
+#if PY_MAJOR_VERSION >= 3
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:frame_header_size",
+#else
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:frame_header_size",
+#endif
+		kwlist, &source)) {
+		return NULL;
+	}
+
+	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
+		PyErr_SetString(PyExc_ValueError,
+			"data buffer should be contiguous and have at most one dimension");
+		goto finally;
+	}
+
+	zresult = ZSTD_frameHeaderSize(source.buf, source.len);
+	if (ZSTD_isError(zresult)) {
+		PyErr_Format(ZstdError, "could not determine frame header size: %s",
+			ZSTD_getErrorName(zresult));
+	}
+	else {
+		result = PyLong_FromSize_t(zresult);
+	}
+
+finally:
+
+	PyBuffer_Release(&source);
+
+	return result;
+}
 
 PyDoc_STRVAR(get_frame_parameters__doc__,
 "get_frame_parameters(data)\n"
@@ -48,43 +132,48 b' PyDoc_STRVAR(get_frame_parameters__doc__'
 "Obtains a ``FrameParameters`` instance by parsing data.\n");
 
 PyDoc_STRVAR(train_dictionary__doc__,
-"train_dictionary(dict_size, samples)\n"
-"\n"
-"Train a dictionary from sample data.\n"
-"\n"
-"A compression dictionary of size ``dict_size`` will be created from the\n"
-"iterable of samples provided by ``samples``.\n"
-"\n"
-"The raw dictionary content will be returned\n");
-
-PyDoc_STRVAR(train_cover_dictionary__doc__,
-"train_cover_dictionary(dict_size, samples, k=None, d=None, notifications=0, dict_id=0, level=0)\n"
+"train_dictionary(dict_size, samples, k=None, d=None, steps=None,\n"
+"                 threads=None,notifications=0, dict_id=0, level=0)\n"
 "\n"
 "Train a dictionary from sample data using the COVER algorithm.\n"
 "\n"
-"This behaves like ``train_dictionary()`` except a different algorithm is\n"
-"used to create the dictionary. The algorithm has 2 parameters: ``k`` and\n"
-"``d``. These control the *segment size* and *dmer size*. A reasonable range\n"
-"for ``k`` is ``[16, 2048+]``. A reasonable range for ``d`` is ``[6, 16]``.\n"
+"A compression dictionary of size ``dict_size`` will be created from the\n"
+"iterable of ``samples``. The raw dictionary bytes will be returned.\n"
+"\n"
+"The COVER algorithm has 2 parameters: ``k`` and ``d``. These control the\n"
+"*segment size* and *dmer size*. A reasonable range for ``k`` is\n"
+"``[16, 2048+]``. A reasonable range for ``d`` is ``[6, 16]``.\n"
 "``d`` must be less than or equal to ``k``.\n"
+"\n"
+"``steps`` can be specified to control the number of steps through potential\n"
+"values of ``k`` and ``d`` to try. ``k`` and ``d`` will only be varied if\n"
+"those arguments are not defined. i.e. if ``d`` is ``8``, then only ``k``\n"
+"will be varied in this mode.\n"
+"\n"
+"``threads`` can specify how many threads to use to test various ``k`` and\n"
+"``d`` values. ``-1`` will use as many threads as available CPUs. By default,\n"
+"a single thread is used.\n"
+"\n"
+"When ``k`` and ``d`` are not defined, default values are used and the\n"
+"algorithm will perform multiple iterations - or steps - to try to find\n"
+"ideal parameters. If both ``k`` and ``d`` are specified, then those values\n"
+"will be used. ``steps`` or ``threads`` triggers optimization mode to test\n"
+"multiple ``k`` and ``d`` variations.\n"
 );
 
 static char zstd_doc[] = "Interface to zstandard";
 
 static PyMethodDef zstd_methods[] = {
-	/* TODO remove since it is a method on CompressionParameters. */
-	{ "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size,
-	METH_VARARGS, estimate_compression_context_size__doc__ },
 	{ "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size,
 	METH_NOARGS, estimate_decompression_context_size__doc__ },
-	{ "get_compression_parameters", (PyCFunction)get_compression_parameters,
-	METH_VARARGS, get_compression_parameters__doc__ },
+	{ "frame_content_size", (PyCFunction)frame_content_size,
+	METH_VARARGS | METH_KEYWORDS, frame_content_size__doc__ },
+	{ "frame_header_size", (PyCFunction)frame_header_size,
+	METH_VARARGS | METH_KEYWORDS, frame_header_size__doc__ },
 	{ "get_frame_parameters", (PyCFunction)get_frame_parameters,
-	METH_VARARGS, get_frame_parameters__doc__ },
+	METH_VARARGS | METH_KEYWORDS, get_frame_parameters__doc__ },
 	{ "train_dictionary", (PyCFunction)train_dictionary,
 	METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ },
-	{ "train_cover_dictionary", (PyCFunction)train_cover_dictionary,
-	METH_VARARGS | METH_KEYWORDS, train_cover_dictionary__doc__ },
 	{ NULL, NULL }
 };
 
@@ -94,10 +183,12 b' void compressor_module_init(PyObject* mo'
 void compressionparams_module_init(PyObject* mod);
 void constants_module_init(PyObject* mod);
 void compressiondict_module_init(PyObject* mod);
+void compressionreader_module_init(PyObject* mod);
 void compressionwriter_module_init(PyObject* mod);
 void compressoriterator_module_init(PyObject* mod);
 void decompressor_module_init(PyObject* mod);
 void decompressobj_module_init(PyObject* mod);
+void decompressionreader_module_init(PyObject *mod);
 void decompressionwriter_module_init(PyObject* mod);
 void decompressoriterator_module_init(PyObject* mod);
 void frameparams_module_init(PyObject* mod);
@@ -118,7 +209,7 b' void zstd_module_init(PyObject* m) {'
 	   We detect this mismatch here and refuse to load the module if this
 	   scenario is detected.
 	*/
-	if (ZSTD_VERSION_NUMBER != 10103 || ZSTD_versionNumber() != 10103) {
+	if (ZSTD_VERSION_NUMBER != 10304 || ZSTD_versionNumber() != 10304) {
 		PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version");
 		return;
 	}
@@ -128,16 +219,24 b' void zstd_module_init(PyObject* m) {'
 	compressiondict_module_init(m);
 	compressobj_module_init(m);
 	compressor_module_init(m);
+	compressionreader_module_init(m);
 	compressionwriter_module_init(m);
 	compressoriterator_module_init(m);
 	constants_module_init(m);
 	decompressor_module_init(m);
 	decompressobj_module_init(m);
+	decompressionreader_module_init(m);
 	decompressionwriter_module_init(m);
 	decompressoriterator_module_init(m);
 	frameparams_module_init(m);
 }
 
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define PYTHON_ZSTD_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+#  define PYTHON_ZSTD_VISIBILITY
+#endif
+
 #if PY_MAJOR_VERSION >= 3
 static struct PyModuleDef zstd_module = {
 	PyModuleDef_HEAD_INIT,
@@ -147,7 +246,7 b' static struct PyModuleDef zstd_module = '
 	zstd_methods
 };
 
-PyMODINIT_FUNC PyInit_zstd(void) {
+PYTHON_ZSTD_VISIBILITY PyMODINIT_FUNC PyInit_zstd(void) {
 	PyObject *m = PyModule_Create(&zstd_module);
 	if (m) {
 		zstd_module_init(m);
@@ -159,7 +258,7 b' PyMODINIT_FUNC PyInit_zstd(void) {'
 	return m;
 }
 #else
-PyMODINIT_FUNC initzstd(void) {
+PYTHON_ZSTD_VISIBILITY PyMODINIT_FUNC initzstd(void) {
 	PyObject *m = Py_InitModule3("zstd", zstd_methods, zstd_doc);
 	if (m) {
 		zstd_module_init(m);
@@ -211,3 +310,33 b' size_t roundpow2(size_t i) {'
 
 	return i;
 }
+
+/* Safer version of _PyBytes_Resize().
+ *
+ * _PyBytes_Resize() only works if the refcount is 1. In some scenarios,
+ * we can get an object with a refcount > 1, even if it was just created
+ * with PyBytes_FromStringAndSize()! That's because (at least) CPython
+ * pre-allocates PyBytes instances of size 1 for every possible byte value.
+ *
+ * If non-0 is returned, obj may or may not be NULL.
+ */
+int safe_pybytes_resize(PyObject** obj, Py_ssize_t size) {
+	PyObject* tmp;
+
+	if ((*obj)->ob_refcnt == 1) {
+		return _PyBytes_Resize(obj, size);
+	}
+
+	tmp = PyBytes_FromStringAndSize(NULL, size);
+	if (!tmp) {
+		return -1;
+	}
+
+	memcpy(PyBytes_AS_STRING(tmp), PyBytes_AS_STRING(*obj),
+		PyBytes_GET_SIZE(*obj));
+
+	Py_DECREF(*obj);
+	*obj = tmp;
+
+	return 0;
+}
  No newline at end of file

contrib/python-zstandard/zstd/common/bitstream.h

@@ -2,7 +2,7 b''
    bitstream
    Part of FSE library
    header file (to include)
-   Copyright (C) 2013-2016, Yann Collet.
+   Copyright (C) 2013-2017, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
@@ -39,7 +39,6 b''
 extern "C" {
 #endif
 
-
 /*
 *  This API consists of small unitary functions, which must be inlined for best performance.
 *  Since link-time-optimization is not available for all compilers,
@@ -53,6 +52,18 b' extern "C" {'
 #include "error_private.h"  /* error codes and messages */
 
 
+/*-*************************************
+*  Debug
+***************************************/
+#if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
+#  include <assert.h>
+#else
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
+#endif
+
+
 /*=========================================
 *  Target specific
 =========================================*/
@@ -60,18 +71,22 b' extern "C" {'
 #  include <immintrin.h>   /* support for bextr (experimental) */
 #endif
 
+#define STREAM_ACCUMULATOR_MIN_32  25
+#define STREAM_ACCUMULATOR_MIN_64  57
+#define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
+
 
 /*-******************************************
 *  bitStream encoding API (write forward)
 ********************************************/
 /* bitStream can mix input from multiple sources.
-*  A critical property of these streams is that they encode and decode in **reverse** direction.
-*  So the first bit sequence you add will be the last to be read, like a LIFO stack.
-*/
+ * A critical property of these streams is that they encode and decode in **reverse** direction.
+ * So the first bit sequence you add will be the last to be read, like a LIFO stack.
+ */
 typedef struct
 {
     size_t bitContainer;
-    int    bitPos;
+    unsigned bitPos;
     char*  startPtr;
     char*  ptr;
     char*  endPtr;
@@ -109,6 +124,7 b' typedef struct'
     unsigned bitsConsumed;
     const char* ptr;
     const char* start;
+    const char* limitPtr;
 } BIT_DStream_t;
 
 typedef enum { BIT_DStream_unfinished = 0,
@@ -151,140 +167,178 b' MEM_STATIC size_t BIT_readBitsFast(BIT_D'
 /*-**************************************************************
 *  Internal functions
 ****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (register U32 val)
+MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
+    assert(val != 0);
+    {
 #   if defined(_MSC_VER)   /* Visual */
-    unsigned long r=0;
-    _BitScanReverse ( &r, val );
-    return (unsigned) r;
+        unsigned long r=0;
+        _BitScanReverse ( &r, val );
+        return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
-    return 31 - __builtin_clz (val);
+        return 31 - __builtin_clz (val);
 #   else   /* Software version */
-    static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-    U32 v = val;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
+        static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
+                                                 11, 14, 16, 18, 22, 25,  3, 30,
+                                                  8, 12, 20, 28, 15, 17, 24,  7,
+                                                 19, 27, 23,  6, 26,  5,  4, 31 };
+        U32 v = val;
+        v |= v >> 1;
+        v |= v >> 2;
+        v |= v >> 4;
+        v |= v >> 8;
+        v |= v >> 16;
+        return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
 #   endif
+    }
 }
 
 /*=====    Local Constants   =====*/
-static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,  0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
-
+static const unsigned BIT_mask[] = {
+    0,          1,         3,         7,         0xF,       0x1F,
+    0x3F,       0x7F,      0xFF,      0x1FF,     0x3FF,     0x7FF,
+    0xFFF,      0x1FFF,    0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
+    0x3FFFF,    0x7FFFF,   0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
+    0xFFFFFF,   0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+    0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
 
 /*-**************************************************************
 *  bitStream encoding
 ****************************************************************/
 /*! BIT_initCStream() :
- *  `dstCapacity` must be > sizeof(void*)
+ *  `dstCapacity` must be > sizeof(size_t)
  *  @return : 0 if success,
-              otherwise an error code (can be tested using ERR_isError() ) */
-MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
+ *            otherwise an error code (can be tested using ERR_isError()) */
+MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
+                                  void* startPtr, size_t dstCapacity)
 {
     bitC->bitContainer = 0;
     bitC->bitPos = 0;
     bitC->startPtr = (char*)startPtr;
     bitC->ptr = bitC->startPtr;
-    bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
-    if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
+    bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
+    if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
     return 0;
 }
 
 /*! BIT_addBits() :
-    can add up to 26 bits into `bitC`.
-    Does not check for register overflow ! */
-MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
+ *  can add up to 31 bits into `bitC`.
+ *  Note : does not check for register overflow ! */
+MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
+                            size_t value, unsigned nbBits)
 {
+    MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+    assert(nbBits < BIT_MASK_SIZE);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
 
 /*! BIT_addBitsFast() :
  *  works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
-MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
+                                size_t value, unsigned nbBits)
 {
+    assert((value>>nbBits) == 0);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= value << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
 
 /*! BIT_flushBitsFast() :
+ *  assumption : bitContainer has not overflowed
  *  unsafe version; does not check buffer overflow */
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
+    assert(bitC->ptr <= bitC->endPtr);
     bitC->bitPos &= 7;
-    bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+    bitC->bitContainer >>= nbBytes*8;
 }
 
 /*! BIT_flushBits() :
+ *  assumption : bitContainer has not overflowed
  *  safe version; check for buffer overflow, and prevents it.
- *  note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
+ *  note : does not signal buffer overflow.
+ *  overflow will be revealed later on using BIT_closeCStream() */
 MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
     if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
     bitC->bitPos &= 7;
-    bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+    bitC->bitContainer >>= nbBytes*8;
 }
 
 /*! BIT_closeCStream() :
  *  @return : size of CStream, in bytes,
-              or 0 if it could not fit into dstBuffer */
+ *            or 0 if it could not fit into dstBuffer */
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
 {
     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
     BIT_flushBits(bitC);
-
-    if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
-
+    if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
     return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
 }
 
 
 /*-********************************************************
-* bitStream decoding
+*  bitStream decoding
 **********************************************************/
 /*! BIT_initDStream() :
-*   Initialize a BIT_DStream_t.
-*   `bitD` : a pointer to an already allocated BIT_DStream_t structure.
-*   `srcSize` must be the *exact* size of the bitStream, in bytes.
-*   @return : size of stream (== srcSize) or an errorCode if a problem is detected
-*/
+ *  Initialize a BIT_DStream_t.
+ * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+ * `srcSize` must be the *exact* size of the bitStream, in bytes.
+ * @return : size of stream (== srcSize), or an errorCode if a problem is detected
+ */
 MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
 {
     if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
 
+    bitD->start = (const char*)srcBuffer;
+    bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
+
     if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
-        bitD->start = (const char*)srcBuffer;
         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
           bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
     } else {
-        bitD->start = (const char*)srcBuffer;
         bitD->ptr   = bitD->start;
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
-            case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
-            case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
-            case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
-            case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
-            case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
-            case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
-            default:;
+        case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+                /* fall-through */
+
+        case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+                /* fall-through */
+
+        case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+                /* fall-through */
+
+        case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
+                /* fall-through */
+
+        case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
+                /* fall-through */
+
+        case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
+                /* fall-through */
+
+        default: break;
         }
-        { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
-          if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
+        {   BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
+            bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+            if (lastByte == 0) return ERROR(corruption_detected);  /* endMark not present */
+        }
         bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
     }
 
@@ -306,12 +360,14 b' MEM_STATIC size_t BIT_getMiddleBits(size'
 #  endif
         return _bextr_u32(bitContainer, start, nbBits);
 #else
+    assert(nbBits < BIT_MASK_SIZE);
     return (bitContainer >> start) & BIT_mask[nbBits];
 #endif
 }
 
 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
+    assert(nbBits < BIT_MASK_SIZE);
     return bitContainer & BIT_mask[nbBits];
 }
 
@@ -320,24 +376,24 b' MEM_STATIC size_t BIT_getLowerBits(size_'
  *  local register is not modified.
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
- *  @return : value extracted
- */
- MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
+ * @return : value extracted */
+MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
 #if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
     return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
 #else
-    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
-    return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
+    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+    return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
 #endif
 }
 
 /*! BIT_lookBitsFast() :
-*   unsafe version; only works only if nbBits >= 1 */
+ *  unsafe version; only works if nbBits >= 1 */
 MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
 {
-    U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
-    return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
+    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
+    assert(nbBits >= 1);
+    return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
 }
 
 MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
@@ -348,8 +404,7 b' MEM_STATIC void BIT_skipBits(BIT_DStream'
 /*! BIT_readBits() :
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
- *  @return : extracted value.
- */
+ * @return : extracted value. */
 MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
@@ -358,25 +413,26 b' MEM_STATIC size_t BIT_readBits(BIT_DStre'
 }
 
 /*! BIT_readBitsFast() :
-*   unsafe version; only works only if nbBits >= 1 */
+ *  unsafe version; only works only if nbBits >= 1 */
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
+    assert(nbBits >= 1);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
 /*! BIT_reloadDStream() :
-*   Refill `bitD` from buffer previously set in BIT_initDStream() .
-*   This function is safe, it guarantees it will not read beyond src buffer.
-*   @return : status of `BIT_DStream_t` internal register.
-              if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+ *  Refill `bitD` from buffer previously set in BIT_initDStream() .
+ *  This function is safe, it guarantees it will not read beyond src buffer.
+ * @return : status of `BIT_DStream_t` internal register.
+ *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
-	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */
-		return BIT_DStream_overflow;
+    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
+        return BIT_DStream_overflow;
 
-    if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
+    if (bitD->ptr >= bitD->limitPtr) {
         bitD->ptr -= bitD->bitsConsumed >> 3;
         bitD->bitsConsumed &= 7;
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
@@ -386,6 +442,7 b' MEM_STATIC BIT_DStream_status BIT_reload'
         if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
         return BIT_DStream_completed;
     }
+    /* start < ptr < limitPtr */
     {   U32 nbBytes = bitD->bitsConsumed >> 3;
         BIT_DStream_status result = BIT_DStream_unfinished;
         if (bitD->ptr - nbBytes < bitD->start) {
@@ -394,14 +451,14 b' MEM_STATIC BIT_DStream_status BIT_reload'
         }
         bitD->ptr -= nbBytes;
         bitD->bitsConsumed -= nbBytes*8;
-        bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD) */
+        bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
         return result;
     }
 }
 
 /*! BIT_endOfDStream() :
-*   @return Tells if DStream has exactly reached its end (all bits consumed).
-*/
+ * @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
+ */
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 {
     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));

contrib/python-zstandard/zstd/common/entropy_common.c

@@ -43,27 +43,21 b''
 #include "huf.h"
 
 
-/*-****************************************
-*  FSE Error Management
-******************************************/
+/*===   Version   ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
+
+
+/*===   Error Management   ===*/
 unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-
 const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
-
-/* **************************************************************
-*  HUF Error Management
-****************************************************************/
 unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-
 const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
 
 /*-**************************************************************
 *  FSE NCount encoding-decoding
 ****************************************************************/
-static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
-
 size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
                  const void* headerBuffer, size_t hbSize)
 {
@@ -117,21 +111,21 b' size_t FSE_readNCount (short* normalized'
             } else {
                 bitStream >>= 2;
         }   }
-        {   short const max = (short)((2*threshold-1)-remaining);
-            short count;
+        {   int const max = (2*threshold-1) - remaining;
+            int count;
 
             if ((bitStream & (threshold-1)) < (U32)max) {
-                count = (short)(bitStream & (threshold-1));
-                bitCount   += nbBits-1;
+                count = bitStream & (threshold-1);
+                bitCount += nbBits-1;
             } else {
-                count = (short)(bitStream & (2*threshold-1));
+                count = bitStream & (2*threshold-1);
                 if (count >= threshold) count -= max;
-                bitCount   += nbBits;
+                bitCount += nbBits;
             }
 
             count--;   /* extra accuracy */
-            remaining -= FSE_abs(count);
-            normalizedCounter[charnum++] = count;
+            remaining -= count < 0 ? -count : count;   /* -1 means +1 */
+            normalizedCounter[charnum++] = (short)count;
             previous0 = !count;
             while (remaining < threshold) {
                 nbBits--;

contrib/python-zstandard/zstd/common/error_private.c

@@ -1,10 +1,11 b''
-/**
+/*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 /* The purpose of this file is to have a single list of error strings embedded in binary */
@@ -20,23 +21,27 b' const char* ERR_getErrorString(ERR_enum '
     case PREFIX(GENERIC):  return "Error (generic)";
     case PREFIX(prefix_unknown): return "Unknown frame descriptor";
     case PREFIX(version_unsupported): return "Version not supported";
-    case PREFIX(parameter_unknown): return "Unknown parameter type";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
-    case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
-    case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
+    case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+    case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
     case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
-    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
-    case PREFIX(srcSize_wrong): return "Src size incorrect";
-    case PREFIX(corruption_detected): return "Corrupted block detected";
-    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
     case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
     case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
     case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+    case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
+    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+    case PREFIX(srcSize_wrong): return "Src size is incorrect";
+        /* following error codes are not stable and may be removed or changed in a future version */
+    case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
+    case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }

contrib/python-zstandard/zstd/common/error_private.h

@@ -1,10 +1,11 b''
-/**
+/*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 /* Note : this module is expected to remain private, do not expose it */
@@ -48,10 +49,9 b' typedef ZSTD_ErrorCode ERR_enum;'
 /*-****************************************
 *  Error codes handling
 ******************************************/
-#ifdef ERROR
-#  undef ERROR   /* reported already defined on VS 2015 (Rich Geldreich) */
-#endif
-#define ERROR(name) ((size_t)-PREFIX(name))
+#undef ERROR   /* reported already defined on VS 2015 (Rich Geldreich) */
+#define ERROR(name) ZSTD_ERROR(name)
+#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
 
 ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
 

contrib/python-zstandard/zstd/common/fse.h

@@ -31,13 +31,14 b''
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
-#ifndef FSE_H
-#define FSE_H
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef FSE_H
+#define FSE_H
+
 
 /*-*****************************************
 *  Dependencies
@@ -45,6 +46,32 b' extern "C" {'
 #include <stddef.h>    /* size_t, ptrdiff_t */
 
 
+/*-*****************************************
+*  FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+#  define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
+#  define FSE_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+#  define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define FSE_PUBLIC_API
+#endif
+
+/*------   Version   ------*/
+#define FSE_VERSION_MAJOR    0
+#define FSE_VERSION_MINOR    9
+#define FSE_VERSION_RELEASE  0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER  (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /**< library version number; to be used when checking dll version */
+
 /*-****************************************
 *  FSE simple functions
 ******************************************/
@@ -56,8 +83,8 b' extern "C" {'
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
                      if FSE_isError(return), compression failed (more details using FSE_getErrorName())
 */
-size_t FSE_compress(void* dst, size_t dstCapacity,
-              const void* src, size_t srcSize);
+FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
+                             const void* src, size_t srcSize);
 
 /*! FSE_decompress():
     Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
@@ -69,18 +96,18 b' size_t FSE_compress(void* dst, size_t ds'
     Why ? : making this distinction requires a header.
     Header management is intentionally delegated to the user layer, which can better manage special cases.
 */
-size_t FSE_decompress(void* dst,  size_t dstCapacity,
-                const void* cSrc, size_t cSrcSize);
+FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
+                               const void* cSrc, size_t cSrcSize);
 
 
 /*-*****************************************
 *  Tool functions
 ******************************************/
-size_t FSE_compressBound(size_t size);       /* maximum compressed size */
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size);       /* maximum compressed size */
 
 /* Error Management */
-unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
-const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
+FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return value is an error code */
+FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
 /*-*****************************************
@@ -94,7 +121,7 b' const char* FSE_getErrorName(size_t code'
                      if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
                      if FSE_isError(return), it's an error code.
 */
-size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
 
 
 /*-*****************************************
@@ -127,50 +154,50 b' or to save and provide normalized distri'
     @return : the count of the most frequent symbol (which is not identified).
               if return == srcSize, there is only one symbol.
               Can also return an error code, which can be tested with FSE_isError(). */
-size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
+FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
 
 /*! FSE_optimalTableLog():
     dynamically downsize 'tableLog' when conditions are met.
     It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
     @return : recommended tableLog (necessarily <= 'maxTableLog') */
-unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_normalizeCount():
     normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
     'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
     @return : tableLog,
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
+FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
 
 /*! FSE_NCountWriteBound():
     Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
     Typically useful for allocation purpose. */
-size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_writeNCount():
     Compactly save 'normalizedCounter' into 'buffer'.
     @return : size of the compressed table,
               or an errorCode, which can be tested using FSE_isError(). */
-size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
-void        FSE_freeCTable (FSE_CTable* ct);
+FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
     @return : 0, or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_compress_usingCTable():
     Compress `src` using `ct` into `dst` which must be already allocated.
     @return : size of compressed data (<= `dstCapacity`),
               or 0 if compressed data could not fit into `dst`,
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
+FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
 
 /*!
 Tutorial :
@@ -223,25 +250,25 b' If there is an error, the function will '
     @return : size read from 'rBuffer',
               or an errorCode, which can be tested using FSE_isError().
               maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
-size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
+FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
 
 /*! Constructor and Destructor of FSE_DTable.
     Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_DTable* FSE_createDTable(unsigned tableLog);
-void        FSE_freeDTable(FSE_DTable* dt);
+FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
+FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
 
 /*! FSE_buildDTable():
     Builds 'dt', which must be already allocated, using FSE_createDTable().
     return : 0, or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*! FSE_decompress_usingDTable():
     Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
     into `dst` which must be already allocated.
     @return : size of regenerated data (necessarily <= `dstCapacity`),
               or an errorCode, which can be tested using FSE_isError() */
-size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
+FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :
@@ -271,8 +298,10 b' FSE_decompress_usingDTable() result will'
 If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
 */
 
+#endif  /* FSE_H */
 
-#ifdef FSE_STATIC_LINKING_ONLY
+#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
+#define FSE_H_FSE_STATIC_LINKING_ONLY
 
 /* *** Dependency *** */
 #include "bitstream.h"
@@ -290,6 +319,10 b' If there is an error, the function will '
 #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue)   (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
 #define FSE_DTABLE_SIZE_U32(maxTableLog)                   (1 + (1<<maxTableLog))
 
+/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
+#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue)   (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
+#define FSE_DTABLE_SIZE(maxTableLog)                   (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
+
 
 /* *****************************************
 *  FSE advanced API
@@ -312,7 +345,7 b' size_t FSE_countFast(unsigned* count, un'
  */
 size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
 
-/*! FSE_count_simple
+/*! FSE_count_simple() :
  * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
  * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
 */
@@ -327,7 +360,7 b' unsigned FSE_optimalTableLog_internal(un'
  * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
  * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
  */
-#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) )
+#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
 size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 
 size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
@@ -351,6 +384,11 b' size_t FSE_buildDTable_rle (FSE_DTable* '
 size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
 /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
 
+typedef enum {
+   FSE_repeat_none,  /**< Cannot use the previous table */
+   FSE_repeat_check, /**< Can use the previous table but it must be checked */
+   FSE_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } FSE_repeat;
 
 /* *****************************************
 *  FSE symbol compression API
@@ -524,9 +562,9 b' MEM_STATIC void FSE_initCState2(FSE_CSta'
 
 MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
 {
-    const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
+    FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
     const U16* const stateTable = (const U16*)(statePtr->stateTable);
-    U32 nbBitsOut  = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
+    U32 const nbBitsOut  = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
     BIT_addBits(bitC, statePtr->value, nbBitsOut);
     statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
 }
@@ -664,5 +702,3 b' MEM_STATIC unsigned FSE_endOfDState(cons'
 #if defined (__cplusplus)
 }
 #endif
-
-#endif  /* FSE_H */

contrib/python-zstandard/zstd/common/fse_decompress.c

@@ -34,35 +34,15 b''
 
 
 /* **************************************************************
-*  Compiler specifics
-****************************************************************/
-#ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
-#  include <intrin.h>                    /* For Visual 2005 */
-#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
-#else
-#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-#    ifdef __GNUC__
-#      define FORCE_INLINE static inline __attribute__((always_inline))
-#    else
-#      define FORCE_INLINE static inline
-#    endif
-#  else
-#    define FORCE_INLINE static
-#  endif /* __STDC_VERSION__ */
-#endif
-
-
-/* **************************************************************
 *  Includes
 ****************************************************************/
 #include <stdlib.h>     /* malloc, free, qsort */
 #include <string.h>     /* memcpy, memset */
-#include <stdio.h>      /* printf (debug) */
 #include "bitstream.h"
+#include "compiler.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
+#include "error_private.h"
 
 
 /* **************************************************************
@@ -159,8 +139,8 b' size_t FSE_buildDTable(FSE_DTable* dt, c'
     {   U32 u;
         for (u=0; u<tableSize; u++) {
             FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
-            U16 nextState = symbolNext[symbol]++;
-            tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
+            U32 const nextState = symbolNext[symbol]++;
+            tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
             tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
     }   }
 
@@ -217,7 +197,7 b' size_t FSE_buildDTable_raw (FSE_DTable* '
     return 0;
 }
 
-FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
+FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
           void* dst, size_t maxDstSize,
     const void* cSrc, size_t cSrcSize,
     const FSE_DTable* dt, const unsigned fast)

contrib/python-zstandard/zstd/common/huf.h

@@ -31,81 +31,114 b''
    You can contact the author at :
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 ****************************************************************** */
-#ifndef HUF_H_298734234
-#define HUF_H_298734234
 
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
 
 /* *** Dependencies *** */
 #include <stddef.h>    /* size_t */
 
 
-/* *** simple functions *** */
-/**
-HUF_compress() :
-    Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
-    'dst' buffer must be already allocated.
-    Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
-    `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
-    @return : size of compressed data (<= `dstCapacity`).
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
-                     if return == 1, srcData is a single repeated byte symbol (RLE compression).
-                     if HUF_isError(return), compression failed (more details using HUF_getErrorName())
-*/
-size_t HUF_compress(void* dst, size_t dstCapacity,
-              const void* src, size_t srcSize);
+/* *** library symbols visibility *** */
+/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
+ *        HUF symbols remain "private" (internal symbols for library only).
+ *        Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
+#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
+#  define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
+#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
+#  define HUF_PUBLIC_API __declspec(dllexport)
+#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
+#  define HUF_PUBLIC_API __declspec(dllimport)  /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
+#else
+#  define HUF_PUBLIC_API
+#endif
+
+
+/* ========================== */
+/* ***  simple functions  *** */
+/* ========================== */
 
-/**
-HUF_decompress() :
-    Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
-    into already allocated buffer 'dst', of minimum size 'dstSize'.
-    `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
-    Note : in contrast with FSE, HUF_decompress can regenerate
-           RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
-           because it knows size to regenerate.
-    @return : size of regenerated data (== originalSize),
-              or an error code, which can be tested using HUF_isError()
-*/
-size_t HUF_decompress(void* dst,  size_t originalSize,
-                const void* cSrc, size_t cSrcSize);
+/** HUF_compress() :
+ *  Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
+ * 'dst' buffer must be already allocated.
+ *  Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
+ * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
+ * @return : size of compressed data (<= `dstCapacity`).
+ *  Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
+ *                   if HUF_isError(return), compression failed (more details using HUF_getErrorName())
+ */
+HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
+                             const void* src, size_t srcSize);
+
+/** HUF_decompress() :
+ *  Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
+ *  into already allocated buffer 'dst', of minimum size 'dstSize'.
+ * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
+ *  Note : in contrast with FSE, HUF_decompress can regenerate
+ *         RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
+ *         because it knows size to regenerate (originalSize).
+ * @return : size of regenerated data (== originalSize),
+ *           or an error code, which can be tested using HUF_isError()
+ */
+HUF_PUBLIC_API size_t HUF_decompress(void* dst,  size_t originalSize,
+                               const void* cSrc, size_t cSrcSize);
 
 
 /* ***   Tool functions *** */
-#define HUF_BLOCKSIZE_MAX (128 * 1024)       /**< maximum input size for a single block compressed with HUF_compress */
-size_t HUF_compressBound(size_t size);       /**< maximum compressed size (worst case) */
+#define HUF_BLOCKSIZE_MAX (128 * 1024)                  /**< maximum input size for a single block compressed with HUF_compress */
+HUF_PUBLIC_API size_t HUF_compressBound(size_t size);   /**< maximum compressed size (worst case) */
 
 /* Error Management */
-unsigned    HUF_isError(size_t code);        /**< tells if a return value is an error code */
-const char* HUF_getErrorName(size_t code);   /**< provides error code string (useful for debugging) */
+HUF_PUBLIC_API unsigned    HUF_isError(size_t code);       /**< tells if a return value is an error code */
+HUF_PUBLIC_API const char* HUF_getErrorName(size_t code);  /**< provides error code string (useful for debugging) */
 
 
 /* ***   Advanced function   *** */
 
 /** HUF_compress2() :
- *   Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` .
- *   `tableLog` must be `<= HUF_TABLELOG_MAX` . */
-size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+ *  Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
+ * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
+ * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
+HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
+                               const void* src, size_t srcSize,
+                               unsigned maxSymbolValue, unsigned tableLog);
 
 /** HUF_compress4X_wksp() :
-*   Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+ *  Same as HUF_compress2(), but uses externally allocated `workSpace`.
+ * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */
+#define HUF_WORKSPACE_SIZE (6 << 10)
+#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
+HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
+                                     const void* src, size_t srcSize,
+                                     unsigned maxSymbolValue, unsigned tableLog,
+                                     void* workSpace, size_t wkspSize);
 
-
+#endif   /* HUF_H_298734234 */
 
-#ifdef HUF_STATIC_LINKING_ONLY
+/* ******************************************************************
+ *  WARNING !!
+ *  The following section contains advanced and experimental definitions
+ *  which shall never be used in the context of a dynamic library,
+ *  because they are not guaranteed to remain stable in the future.
+ *  Only consider them in association with static linking.
+ * *****************************************************************/
+#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
+#define HUF_H_HUF_STATIC_LINKING_ONLY
 
 /* *** Dependencies *** */
 #include "mem.h"   /* U32 */
 
 
 /* *** Constants *** */
-#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#define HUF_TABLELOG_MAX  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
-#define HUF_TABLELOG_DEFAULT  11       /* tableLog by default, when not specified */
-#define HUF_SYMBOLVALUE_MAX 255
+#define HUF_TABLELOG_MAX      12      /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_TABLELOG_DEFAULT  11      /* default tableLog value when none specified */
+#define HUF_SYMBOLVALUE_MAX  255
+
+#define HUF_TABLELOG_ABSOLUTEMAX  15  /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
 #  error "HUF_TABLELOG_MAX is too large !"
 #endif
@@ -116,12 +149,14 b' size_t HUF_compress4X_wksp (void* dst, s'
 ******************************************/
 /* HUF buffer bounds */
 #define HUF_CTABLEBOUND 129
-#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8)   /* only true if incompressible pre-filtered with fast heuristic */
+#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8)   /* only true when incompressible is pre-filtered with fast heuristic */
 #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
 
 /* static allocation of HUF's Compression Table */
+#define HUF_CTABLE_SIZE_U32(maxSymbolValue)   ((maxSymbolValue)+1)   /* Use tables of U32, for proper alignment */
+#define HUF_CTABLE_SIZE(maxSymbolValue)       (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))
 #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
-    U32 name##hb[maxSymbolValue+1]; \
+    U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \
     void* name##hv = &(name##hb); \
     HUF_CElt* name = (HUF_CElt*)(name##hv)   /* no final ; */
 
@@ -142,97 +177,151 b' size_t HUF_decompress4X4 (void* dst, siz'
 
 size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
 
 
 /* ****************************************
-*  HUF detailed API
-******************************************/
-/*!
-HUF_compress() does the following:
-1. count symbol occurrence from source[] into table count[] using FSE_count()
-2. (optional) refine tableLog using HUF_optimalTableLog()
-3. build Huffman table from count using HUF_buildCTable()
-4. save Huffman table to memory buffer using HUF_writeCTable()
-5. encode the data stream using HUF_compress4X_usingCTable()
+ *  HUF detailed API
+ * ****************************************/
 
-The following API allows targeting specific sub-functions for advanced tasks.
-For example, it's possible to compress several blocks using the same 'CTable',
-or to save and regenerate 'CTable' using external methods.
-*/
-/* FSE_count() : find it within "fse.h" */
+/*! HUF_compress() does the following:
+ *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
+ *  2. (optional) refine tableLog using HUF_optimalTableLog()
+ *  3. build Huffman table from count using HUF_buildCTable()
+ *  4. save Huffman table to memory buffer using HUF_writeCTable()
+ *  5. encode the data stream using HUF_compress4X_usingCTable()
+ *
+ *  The following API allows targeting specific sub-functions for advanced tasks.
+ *  For example, it's possible to compress several blocks using the same 'CTable',
+ *  or to save and regenerate 'CTable' using external methods.
+ */
 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
 typedef struct HUF_CElt_s HUF_CElt;   /* incomplete type */
-size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
+size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);   /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 
+typedef enum {
+   HUF_repeat_none,  /**< Cannot use the previous table */
+   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+   HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } HUF_repeat;
+/** HUF_compress4X_repeat() :
+ *  Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ *  If it uses hufTable it does not modify hufTable or repeat.
+ *  If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ *  If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
+                       const void* src, size_t srcSize,
+                       unsigned maxSymbolValue, unsigned tableLog,
+                       void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
- *  `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
+ * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
  */
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
+#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
 size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
 
 /*! HUF_readStats() :
-    Read compact Huffman tree, saved by HUF_writeCTable().
-    `huffWeight` is destination buffer.
-    @return : size read from `src` , or an error Code .
-    Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
-size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
-                     U32* nbSymbolsPtr, U32* tableLogPtr,
+ *  Read compact Huffman tree, saved by HUF_writeCTable().
+ * `huffWeight` is destination buffer.
+ * @return : size read from `src` , or an error Code .
+ *  Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
+size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
+                     U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
                      const void* src, size_t srcSize);
 
 /** HUF_readCTable() :
-*   Loading a CTable saved with HUF_writeCTable() */
-size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
+ *  Loading a CTable saved with HUF_writeCTable() */
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
 
 
 /*
-HUF_decompress() does the following:
-1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
-2. build Huffman table from save, using HUF_readDTableXn()
-3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
-*/
+ * HUF_decompress() does the following:
+ * 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
+ * 2. build Huffman table from save, using HUF_readDTableX?()
+ * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
+ */
 
 /** HUF_selectDecoder() :
-*   Tells which decoder is likely to decode faster,
-*   based on a set of pre-determined metrics.
-*   @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
-*   Assumption : 0 < cSrcSize < dstSize <= 128 KB */
+ *  Tells which decoder is likely to decode faster,
+ *  based on a set of pre-computed metrics.
+ * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
+ *  Assumption : 0 < dstSize <= 128 KB */
 U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
 
+/**
+ *  The minimum workspace size for the `workSpace` used in
+ *  HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp().
+ *
+ *  The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
+ *  HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
+ *  Buffer overflow errors may potentially occur if code modifications result in
+ *  a required workspace size greater than that specified in the following
+ *  macro.
+ */
+#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
 size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
+size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
 size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
+size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
 
 size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
 size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
 size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
 
 
+/* ====================== */
 /* single stream variants */
+/* ====================== */
 
 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+/** HUF_compress1X_repeat() :
+ *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+ *  If it uses hufTable it does not modify hufTable or repeat.
+ *  If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+ *  If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
+                       const void* src, size_t srcSize,
+                       unsigned maxSymbolValue, unsigned tableLog,
+                       void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
 
 size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
 size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< single-symbol decoder */
 size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /**< double-symbols decoder */
 
 size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);   /**< automatic selection of sing or double symbol decoder, based on DTable */
 size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
 size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
 
+/* BMI2 variants.
+ * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
+ */
+size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+size_t HUF_decompress1X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+
 #endif /* HUF_STATIC_LINKING_ONLY */
 
-
 #if defined (__cplusplus)
 }
 #endif
-
-#endif   /* HUF_H_298734234 */

contrib/python-zstandard/zstd/common/mem.h

@@ -1,10 +1,11 b''
-/**
+/*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 #ifndef MEM_H_MODULE
@@ -48,14 +49,13 b' MEM_STATIC void MEM_check(void) { MEM_ST'
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
-  typedef  uint8_t BYTE;
-  typedef uint16_t U16;
-  typedef  int16_t S16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
-  typedef  int64_t S64;
-  typedef intptr_t iPtrDiff;
+  typedef   uint8_t BYTE;
+  typedef  uint16_t U16;
+  typedef   int16_t S16;
+  typedef  uint32_t U32;
+  typedef   int32_t S32;
+  typedef  uint64_t U64;
+  typedef   int64_t S64;
 #else
   typedef unsigned char      BYTE;
   typedef unsigned short      U16;
@@ -64,7 +64,6 b' MEM_STATIC void MEM_check(void) { MEM_ST'
   typedef   signed int        S32;
   typedef unsigned long long  U64;
   typedef   signed long long  S64;
-  typedef ptrdiff_t      iPtrDiff;
 #endif
 
 
@@ -76,19 +75,18 b' MEM_STATIC void MEM_check(void) { MEM_ST'
  * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
  * The below switch allow to select different access method for improved performance.
  * Method 0 (default) : use `memcpy()`. Safe and portable.
- * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
  *            It can generate buggy code on targets depending on alignment.
- *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ *            In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define MEM_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
-  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#  elif defined(__INTEL_COMPILER) || defined(__GNUC__)
 #    define MEM_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
@@ -109,7 +107,7 b' Only use if no other choice to achieve b'
 MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
 MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
 MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
-MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
+MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
 
 MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
 MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
@@ -120,21 +118,27 b' MEM_STATIC void MEM_write64(void* memPtr'
 /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
 /* currently only defined for gcc and icc */
 #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
-	__pragma( pack(push, 1) )
-    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
+    __pragma( pack(push, 1) )
+    typedef struct { U16 v; } unalign16;
+    typedef struct { U32 v; } unalign32;
+    typedef struct { U64 v; } unalign64;
+    typedef struct { size_t v; } unalignArch;
     __pragma( pack(pop) )
 #else
-    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+    typedef struct { U16 v; } __attribute__((packed)) unalign16;
+    typedef struct { U32 v; } __attribute__((packed)) unalign32;
+    typedef struct { U64 v; } __attribute__((packed)) unalign64;
+    typedef struct { size_t v; } __attribute__((packed)) unalignArch;
 #endif
 
-MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
-MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
-MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
+MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
+MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
 
-MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
-MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
 
 #else
 
@@ -182,7 +186,7 b' MEM_STATIC U32 MEM_swap32(U32 in)'
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_ulong(in);
-#elif defined (__GNUC__)
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
     return __builtin_bswap32(in);
 #else
     return  ((in << 24) & 0xff000000 ) |
@@ -196,7 +200,7 b' MEM_STATIC U64 MEM_swap64(U64 in)'
 {
 #if defined(_MSC_VER)     /* Visual Studio */
     return _byteswap_uint64(in);
-#elif defined (__GNUC__)
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
     return __builtin_bswap64(in);
 #else
     return  ((in << 56) & 0xff00000000000000ULL) |
@@ -351,20 +355,6 b' MEM_STATIC void MEM_writeBEST(void* memP'
 }
 
 
-/* function safe only for comparisons */
-MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
-{
-    switch (length)
-    {
-    default :
-    case 4 : return MEM_read32(memPtr);
-    case 3 : if (MEM_isLittleEndian())
-                return MEM_read32(memPtr)<<8;
-             else
-                return MEM_read32(memPtr)>>8;
-    }
-}
-
 #if defined (__cplusplus)
 }
 #endif

contrib/python-zstandard/zstd/common/pool.c

@@ -1,17 +1,18 b''
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 
 /* ======   Dependencies   ======= */
 #include <stddef.h>  /* size_t */
-#include <stdlib.h>  /* malloc, calloc, free */
 #include "pool.h"
+#include "zstd_internal.h"  /* ZSTD_malloc, ZSTD_free */
 
 /* ======   Compiler specifics   ====== */
 #if defined(_MSC_VER)
@@ -25,13 +26,14 b''
 
 /* A job is a function and an opaque argument */
 typedef struct POOL_job_s {
-  POOL_function function;
-  void *opaque;
+    POOL_function function;
+    void *opaque;
 } POOL_job;
 
 struct POOL_ctx_s {
+    ZSTD_customMem customMem;
     /* Keep track of the threads */
-    pthread_t *threads;
+    ZSTD_pthread_t *threads;
     size_t numThreads;
 
     /* The queue is a circular buffer */
@@ -39,12 +41,18 b' struct POOL_ctx_s {'
     size_t queueHead;
     size_t queueTail;
     size_t queueSize;
+
+    /* The number of threads working on jobs */
+    size_t numThreadsBusy;
+    /* Indicates if the queue is empty */
+    int queueEmpty;
+
     /* The mutex protects the queue */
-    pthread_mutex_t queueMutex;
+    ZSTD_pthread_mutex_t queueMutex;
     /* Condition variable for pushers to wait on when the queue is full */
-    pthread_cond_t queuePushCond;
+    ZSTD_pthread_cond_t queuePushCond;
     /* Condition variables for poppers to wait on when the queue is empty */
-    pthread_cond_t queuePopCond;
+    ZSTD_pthread_cond_t queuePopCond;
     /* Indicates if the queue is shutting down */
     int shutdown;
 };
@@ -59,55 +67,73 b' static void* POOL_thread(void* opaque) {'
     if (!ctx) { return NULL; }
     for (;;) {
         /* Lock the mutex and wait for a non-empty queue or until shutdown */
-        pthread_mutex_lock(&ctx->queueMutex);
-        while (ctx->queueHead == ctx->queueTail && !ctx->shutdown) {
-            pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
+        ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+
+        while (ctx->queueEmpty && !ctx->shutdown) {
+            ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
         }
         /* empty => shutting down: so stop */
-        if (ctx->queueHead == ctx->queueTail) {
-            pthread_mutex_unlock(&ctx->queueMutex);
+        if (ctx->queueEmpty) {
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
             return opaque;
         }
         /* Pop a job off the queue */
         {   POOL_job const job = ctx->queue[ctx->queueHead];
             ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+            ctx->numThreadsBusy++;
+            ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
             /* Unlock the mutex, signal a pusher, and run the job */
-            pthread_mutex_unlock(&ctx->queueMutex);
-            pthread_cond_signal(&ctx->queuePushCond);
+            ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+            ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+
             job.function(job.opaque);
-        }
-    }
+
+            /* If the intended queue size was 0, signal after finishing job */
+            if (ctx->queueSize == 1) {
+                ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+                ctx->numThreadsBusy--;
+                ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+                ZSTD_pthread_cond_signal(&ctx->queuePushCond);
+        }   }
+    }  /* for (;;) */
     /* Unreachable */
 }
 
-POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
-    POOL_ctx *ctx;
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+    return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
+}
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
+    POOL_ctx* ctx;
     /* Check the parameters */
-    if (!numThreads || !queueSize) { return NULL; }
+    if (!numThreads) { return NULL; }
     /* Allocate the context and zero initialize */
-    ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx));
+    ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem);
     if (!ctx) { return NULL; }
     /* Initialize the job queue.
      * It needs one extra space since one space is wasted to differentiate empty
      * and full queues.
      */
     ctx->queueSize = queueSize + 1;
-    ctx->queue = (POOL_job *)malloc(ctx->queueSize * sizeof(POOL_job));
+    ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
     ctx->queueHead = 0;
     ctx->queueTail = 0;
-    pthread_mutex_init(&ctx->queueMutex, NULL);
-    pthread_cond_init(&ctx->queuePushCond, NULL);
-    pthread_cond_init(&ctx->queuePopCond, NULL);
+    ctx->numThreadsBusy = 0;
+    ctx->queueEmpty = 1;
+    (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+    (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
-    ctx->threads = (pthread_t *)malloc(numThreads * sizeof(pthread_t));
+    ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
     ctx->numThreads = 0;
+    ctx->customMem = customMem;
     /* Check for errors */
     if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
     /* Initialize the threads */
     {   size_t i;
         for (i = 0; i < numThreads; ++i) {
-            if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
+            if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
                 ctx->numThreads = i;
                 POOL_free(ctx);
                 return NULL;
@@ -120,75 +146,138 b' POOL_ctx *POOL_create(size_t numThreads,'
 /*! POOL_join() :
     Shutdown the queue, wake any sleeping threads, and join all of the threads.
 */
-static void POOL_join(POOL_ctx *ctx) {
+static void POOL_join(POOL_ctx* ctx) {
     /* Shut down the queue */
-    pthread_mutex_lock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
     ctx->shutdown = 1;
-    pthread_mutex_unlock(&ctx->queueMutex);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
     /* Wake up sleeping threads */
-    pthread_cond_broadcast(&ctx->queuePushCond);
-    pthread_cond_broadcast(&ctx->queuePopCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
+    ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
     /* Join all of the threads */
     {   size_t i;
         for (i = 0; i < ctx->numThreads; ++i) {
-            pthread_join(ctx->threads[i], NULL);
+            ZSTD_pthread_join(ctx->threads[i], NULL);
     }   }
 }
 
 void POOL_free(POOL_ctx *ctx) {
     if (!ctx) { return; }
     POOL_join(ctx);
-    pthread_mutex_destroy(&ctx->queueMutex);
-    pthread_cond_destroy(&ctx->queuePushCond);
-    pthread_cond_destroy(&ctx->queuePopCond);
-    if (ctx->queue) free(ctx->queue);
-    if (ctx->threads) free(ctx->threads);
-    free(ctx);
+    ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
+    ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
+    ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
+    ZSTD_free(ctx->queue, ctx->customMem);
+    ZSTD_free(ctx->threads, ctx->customMem);
+    ZSTD_free(ctx, ctx->customMem);
+}
+
+size_t POOL_sizeof(POOL_ctx *ctx) {
+    if (ctx==NULL) return 0;  /* supports sizeof NULL */
+    return sizeof(*ctx)
+        + ctx->queueSize * sizeof(POOL_job)
+        + ctx->numThreads * sizeof(ZSTD_pthread_t);
+}
+
+/**
+ * Returns 1 if the queue is full and 0 otherwise.
+ *
+ * If the queueSize is 1 (the pool was created with an intended queueSize of 0),
+ * then a queue is empty if there is a thread free and no job is waiting.
+ */
+static int isQueueFull(POOL_ctx const* ctx) {
+    if (ctx->queueSize > 1) {
+        return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
+    } else {
+        return ctx->numThreadsBusy == ctx->numThreads ||
+               !ctx->queueEmpty;
+    }
+}
+
+
+static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
+{
+    POOL_job const job = {function, opaque};
+    assert(ctx != NULL);
+    if (ctx->shutdown) return;
+
+    ctx->queueEmpty = 0;
+    ctx->queue[ctx->queueTail] = job;
+    ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
+    ZSTD_pthread_cond_signal(&ctx->queuePopCond);
+}
+
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+    assert(ctx != NULL);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+    /* Wait until there is space in the queue for the new job */
+    while (isQueueFull(ctx) && (!ctx->shutdown)) {
+        ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
+    }
+    POOL_add_internal(ctx, function, opaque);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
 }
 
-void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
-    POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
-    if (!ctx) { return; }
 
-    pthread_mutex_lock(&ctx->queueMutex);
-    {   POOL_job const job = {function, opaque};
-        /* Wait until there is space in the queue for the new job */
-        size_t newTail = (ctx->queueTail + 1) % ctx->queueSize;
-        while (ctx->queueHead == newTail && !ctx->shutdown) {
-          pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
-          newTail = (ctx->queueTail + 1) % ctx->queueSize;
-        }
-        /* The queue is still going => there is space */
-        if (!ctx->shutdown) {
-            ctx->queue[ctx->queueTail] = job;
-            ctx->queueTail = newTail;
-        }
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
+{
+    assert(ctx != NULL);
+    ZSTD_pthread_mutex_lock(&ctx->queueMutex);
+    if (isQueueFull(ctx)) {
+        ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+        return 0;
     }
-    pthread_mutex_unlock(&ctx->queueMutex);
-    pthread_cond_signal(&ctx->queuePopCond);
+    POOL_add_internal(ctx, function, opaque);
+    ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
+    return 1;
 }
 
+
 #else  /* ZSTD_MULTITHREAD  not defined */
+
+/* ========================== */
 /* No multi-threading support */
+/* ========================== */
 
-/* We don't need any data, but if it is empty malloc() might return NULL. */
+
+/* We don't need any data, but if it is empty, malloc() might return NULL. */
 struct POOL_ctx_s {
-  int data;
+    int dummy;
 };
+static POOL_ctx g_ctx;
 
-POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
-  (void)numThreads;
-  (void)queueSize;
-  return (POOL_ctx *)malloc(sizeof(POOL_ctx));
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
+    return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
 }
 
-void POOL_free(POOL_ctx *ctx) {
-  if (ctx) free(ctx);
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) {
+    (void)numThreads;
+    (void)queueSize;
+    (void)customMem;
+    return &g_ctx;
+}
+
+void POOL_free(POOL_ctx* ctx) {
+    assert(!ctx || ctx == &g_ctx);
+    (void)ctx;
 }
 
-void POOL_add(void *ctx, POOL_function function, void *opaque) {
-  (void)ctx;
-  function(opaque);
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
+    (void)ctx;
+    function(opaque);
+}
+
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
+    (void)ctx;
+    function(opaque);
+    return 1;
+}
+
+size_t POOL_sizeof(POOL_ctx* ctx) {
+    if (ctx==NULL) return 0;  /* supports sizeof NULL */
+    assert(ctx == &g_ctx);
+    return sizeof(*ctx);
 }
 
 #endif  /* ZSTD_MULTITHREAD */

contrib/python-zstandard/zstd/common/pool.h

@@ -1,11 +1,13 b''
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
+
 #ifndef POOL_H
 #define POOL_H
 
@@ -15,38 +17,54 b' extern "C" {'
 
 
 #include <stddef.h>   /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_customMem */
+#include "zstd.h"
 
 typedef struct POOL_ctx_s POOL_ctx;
 
 /*! POOL_create() :
-    Create a thread pool with at most `numThreads` threads.
-    `numThreads` must be at least 1.
-    The maximum number of queued jobs before blocking is `queueSize`.
-    `queueSize` must be at least 1.
-    @return : The POOL_ctx pointer on success else NULL.
+ *  Create a thread pool with at most `numThreads` threads.
+ * `numThreads` must be at least 1.
+ *  The maximum number of queued jobs before blocking is `queueSize`.
+ * @return : POOL_ctx pointer on success, else NULL.
 */
-POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem);
 
 /*! POOL_free() :
     Free a thread pool returned by POOL_create().
 */
-void POOL_free(POOL_ctx *ctx);
+void POOL_free(POOL_ctx* ctx);
+
+/*! POOL_sizeof() :
+    return memory usage of pool returned by POOL_create().
+*/
+size_t POOL_sizeof(POOL_ctx* ctx);
 
 /*! POOL_function :
     The function type that can be added to a thread pool.
 */
-typedef void (*POOL_function)(void *);
+typedef void (*POOL_function)(void*);
 /*! POOL_add_function :
     The function type for a generic thread pool add function.
 */
-typedef void (*POOL_add_function)(void *, POOL_function, void *);
+typedef void (*POOL_add_function)(void*, POOL_function, void*);
 
 /*! POOL_add() :
-    Add the job `function(opaque)` to the thread pool.
+    Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
     Possibly blocks until there is room in the queue.
     Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
 */
-void POOL_add(void *ctx, POOL_function function, void *opaque);
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+/*! POOL_tryAdd() :
+    Add the job `function(opaque)` to the thread pool if a worker is available.
+    return immediately otherwise.
+   @return : 1 if successful, 0 if not.
+*/
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
 
 
 #if defined (__cplusplus)

contrib/python-zstandard/zstd/common/threading.c

@@ -1,11 +1,10 b''
-
 /**
  * Copyright (c) 2016 Tino Reichardt
  * All rights reserved.
  *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
  *
  * You can contact the author at:
  * - zstdmt source repository: https://github.com/mcmilk/zstdmt
@@ -15,11 +14,8 b''
  * This file will hold wrapper for systems, which do not support pthreads
  */
 
-/* ======   Compiler specifics   ====== */
-#if defined(_MSC_VER)
-#  pragma warning(disable : 4206)        /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */
-#endif
-
+/* create fake symbol to avoid empty trnaslation unit warning */
+int g_ZSTD_threading_useles_symbol;
 
 #if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
 
@@ -39,12 +35,12 b''
 
 static unsigned __stdcall worker(void *arg)
 {
-    pthread_t* const thread = (pthread_t*) arg;
+    ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
     thread->arg = thread->start_routine(thread->arg);
     return 0;
 }
 
-int pthread_create(pthread_t* thread, const void* unused,
+int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
             void* (*start_routine) (void*), void* arg)
 {
     (void)unused;
@@ -58,16 +54,16 b' int pthread_create(pthread_t* thread, co'
         return 0;
 }
 
-int _pthread_join(pthread_t * thread, void **value_ptr)
+int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 {
     DWORD result;
 
-    if (!thread->handle) return 0;
+    if (!thread.handle) return 0;
 
-    result = WaitForSingleObject(thread->handle, INFINITE);
+    result = WaitForSingleObject(thread.handle, INFINITE);
     switch (result) {
     case WAIT_OBJECT_0:
-        if (value_ptr) *value_ptr = thread->arg;
+        if (value_ptr) *value_ptr = thread.arg;
         return 0;
     case WAIT_ABANDONED:
         return EINVAL;