cext: add Python 3.10 support

* Replace "Py_TYPE(obj) = type;" with "Py_SET_TYPE(obj, type);"
* Add pythoncapi_compat.h header file to get Py_SET_TYPE() on Python
  2.7-3.8. Header file added to mercurial/ and
  contrib/python-zstandard/zstd/common/.

In Python 3.10, Py_TYPE(obj) must not longer be used as an l-value.

pythoncapi_compat.h comes from:
https://github.com/pythoncapi/pythoncapi_compat

Differential Revision: https://phab.mercurial-scm.org/D9825

contrib/python-zstandard/zstd/common/pythoncapi_compat.h

@@ -0,0 +1,278 b''
+// Header file providing new functions of the Python C API to old Python
+// versions.
+//
+// File distributed under the MIT license.
+//
+// Homepage:
+// https://github.com/pythoncapi/pythoncapi_compat
+//
+// Latest version:
+// https://raw.githubusercontent.com/pythoncapi/pythoncapi_compat/master/pythoncapi_compat.h
+
+#ifndef PYTHONCAPI_COMPAT
+#define PYTHONCAPI_COMPAT
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <Python.h>
+#include "frameobject.h"          // PyFrameObject, PyFrame_GetBack()
+
+
+// Cast argument to PyObject* type.
+#ifndef _PyObject_CAST
+#  define _PyObject_CAST(op) ((PyObject*)(op))
+#endif
+
+
+// bpo-42262 added Py_NewRef() to Python 3.10.0a3
+#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_NewRef)
+static inline PyObject* _Py_NewRef(PyObject *obj)
+{
+    Py_INCREF(obj);
+    return obj;
+}
+#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
+#endif
+
+
+// bpo-42262 added Py_XNewRef() to Python 3.10.0a3
+#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_XNewRef)
+static inline PyObject* _Py_XNewRef(PyObject *obj)
+{
+    Py_XINCREF(obj);
+    return obj;
+}
+#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
+#endif
+
+
+// bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT)
+static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt)
+{
+    ob->ob_refcnt = refcnt;
+}
+#define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT((PyObject*)(ob), refcnt)
+#endif
+
+
+// bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE)
+static inline void
+_Py_SET_TYPE(PyObject *ob, PyTypeObject *type)
+{
+    ob->ob_type = type;
+}
+#define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject*)(ob), type)
+#endif
+
+
+// bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE)
+static inline void
+_Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size)
+{
+    ob->ob_size = size;
+}
+#define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size)
+#endif
+
+
+// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyCodeObject*
+PyFrame_GetCode(PyFrameObject *frame)
+{
+    PyCodeObject *code;
+    assert(frame != NULL);
+    code = frame->f_code;
+    assert(code != NULL);
+    Py_INCREF(code);
+    return code;
+}
+#endif
+
+static inline PyCodeObject*
+_PyFrame_GetCodeBorrow(PyFrameObject *frame)
+{
+    PyCodeObject *code = PyFrame_GetCode(frame);
+    Py_DECREF(code);
+    return code;  // borrowed reference
+}
+
+
+// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyFrameObject*
+PyFrame_GetBack(PyFrameObject *frame)
+{
+    PyFrameObject *back;
+    assert(frame != NULL);
+    back = frame->f_back;
+    Py_XINCREF(back);
+    return back;
+}
+#endif
+
+static inline PyFrameObject*
+_PyFrame_GetBackBorrow(PyFrameObject *frame)
+{
+    PyFrameObject *back = PyFrame_GetBack(frame);
+    Py_XDECREF(back);
+    return back;  // borrowed reference
+}
+
+
+// bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline PyInterpreterState *
+PyThreadState_GetInterpreter(PyThreadState *tstate)
+{
+    assert(tstate != NULL);
+    return tstate->interp;
+}
+#endif
+
+
+// bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyFrameObject*
+PyThreadState_GetFrame(PyThreadState *tstate)
+{
+    PyFrameObject *frame;
+    assert(tstate != NULL);
+    frame = tstate->frame;
+    Py_XINCREF(frame);
+    return frame;
+}
+#endif
+
+static inline PyFrameObject*
+_PyThreadState_GetFrameBorrow(PyThreadState *tstate)
+{
+    PyFrameObject *frame = PyThreadState_GetFrame(tstate);
+    Py_XDECREF(frame);
+    return frame;  // borrowed reference
+}
+
+
+// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline PyInterpreterState *
+PyInterpreterState_Get(void)
+{
+    PyThreadState *tstate;
+    PyInterpreterState *interp;
+
+    tstate = PyThreadState_GET();
+    if (tstate == NULL) {
+        Py_FatalError("GIL released (tstate is NULL)");
+    }
+    interp = tstate->interp;
+    if (interp == NULL) {
+        Py_FatalError("no current interpreter");
+    }
+    return interp;
+}
+#endif
+
+
+// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6
+#if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6
+static inline uint64_t
+PyThreadState_GetID(PyThreadState *tstate)
+{
+    assert(tstate != NULL);
+    return tstate->id;
+}
+#endif
+
+
+// bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1
+#if PY_VERSION_HEX < 0x030900A1
+static inline PyObject*
+PyObject_CallNoArgs(PyObject *func)
+{
+    return PyObject_CallFunctionObjArgs(func, NULL);
+}
+#endif
+
+
+// bpo-39245 made PyObject_CallOneArg() public (previously called
+// _PyObject_CallOneArg) in Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4
+static inline PyObject*
+PyObject_CallOneArg(PyObject *func, PyObject *arg)
+{
+    return PyObject_CallFunctionObjArgs(func, arg, NULL);
+}
+#endif
+
+
+// bpo-40024 added PyModule_AddType() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline int
+PyModule_AddType(PyObject *module, PyTypeObject *type)
+{
+    const char *name, *dot;
+
+    if (PyType_Ready(type) < 0) {
+        return -1;
+    }
+
+    // inline _PyType_Name()
+    name = type->tp_name;
+    assert(name != NULL);
+    dot = strrchr(name, '.');
+    if (dot != NULL) {
+        name = dot + 1;
+    }
+
+    Py_INCREF(type);
+    if (PyModule_AddObject(module, name, (PyObject *)type) < 0) {
+        Py_DECREF(type);
+        return -1;
+    }
+
+    return 0;
+}
+#endif
+
+
+// bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6.
+// bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2.
+#if PY_VERSION_HEX < 0x030900A6
+static inline int
+PyObject_GC_IsTracked(PyObject* obj)
+{
+    return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj));
+}
+#endif
+
+// bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6.
+// bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final.
+#if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0
+static inline int
+PyObject_GC_IsFinalized(PyObject *obj)
+{
+    return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED((PyGC_Head *)(obj)-1));
+}
+#endif
+
+
+// bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE)
+static inline int
+_Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) {
+    return ob->ob_type == type;
+}
+#define Py_IS_TYPE(ob, type) _Py_IS_TYPE((const PyObject*)(ob), type)
+#endif
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // PYTHONCAPI_COMPAT

mercurial/pythoncapi_compat.h

@@ -0,0 +1,278 b''
+// Header file providing new functions of the Python C API to old Python
+// versions.
+//
+// File distributed under the MIT license.
+//
+// Homepage:
+// https://github.com/pythoncapi/pythoncapi_compat
+//
+// Latest version:
+// https://raw.githubusercontent.com/pythoncapi/pythoncapi_compat/master/pythoncapi_compat.h
+
+#ifndef PYTHONCAPI_COMPAT
+#define PYTHONCAPI_COMPAT
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <Python.h>
+#include "frameobject.h"          // PyFrameObject, PyFrame_GetBack()
+
+
+// Cast argument to PyObject* type.
+#ifndef _PyObject_CAST
+#  define _PyObject_CAST(op) ((PyObject*)(op))
+#endif
+
+
+// bpo-42262 added Py_NewRef() to Python 3.10.0a3
+#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_NewRef)
+static inline PyObject* _Py_NewRef(PyObject *obj)
+{
+    Py_INCREF(obj);
+    return obj;
+}
+#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
+#endif
+
+
+// bpo-42262 added Py_XNewRef() to Python 3.10.0a3
+#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_XNewRef)
+static inline PyObject* _Py_XNewRef(PyObject *obj)
+{
+    Py_XINCREF(obj);
+    return obj;
+}
+#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
+#endif
+
+
+// bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT)
+static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt)
+{
+    ob->ob_refcnt = refcnt;
+}
+#define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT((PyObject*)(ob), refcnt)
+#endif
+
+
+// bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE)
+static inline void
+_Py_SET_TYPE(PyObject *ob, PyTypeObject *type)
+{
+    ob->ob_type = type;
+}
+#define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject*)(ob), type)
+#endif
+
+
+// bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE)
+static inline void
+_Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size)
+{
+    ob->ob_size = size;
+}
+#define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size)
+#endif
+
+
+// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyCodeObject*
+PyFrame_GetCode(PyFrameObject *frame)
+{
+    PyCodeObject *code;
+    assert(frame != NULL);
+    code = frame->f_code;
+    assert(code != NULL);
+    Py_INCREF(code);
+    return code;
+}
+#endif
+
+static inline PyCodeObject*
+_PyFrame_GetCodeBorrow(PyFrameObject *frame)
+{
+    PyCodeObject *code = PyFrame_GetCode(frame);
+    Py_DECREF(code);
+    return code;  // borrowed reference
+}
+
+
+// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyFrameObject*
+PyFrame_GetBack(PyFrameObject *frame)
+{
+    PyFrameObject *back;
+    assert(frame != NULL);
+    back = frame->f_back;
+    Py_XINCREF(back);
+    return back;
+}
+#endif
+
+static inline PyFrameObject*
+_PyFrame_GetBackBorrow(PyFrameObject *frame)
+{
+    PyFrameObject *back = PyFrame_GetBack(frame);
+    Py_XDECREF(back);
+    return back;  // borrowed reference
+}
+
+
+// bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline PyInterpreterState *
+PyThreadState_GetInterpreter(PyThreadState *tstate)
+{
+    assert(tstate != NULL);
+    return tstate->interp;
+}
+#endif
+
+
+// bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1
+static inline PyFrameObject*
+PyThreadState_GetFrame(PyThreadState *tstate)
+{
+    PyFrameObject *frame;
+    assert(tstate != NULL);
+    frame = tstate->frame;
+    Py_XINCREF(frame);
+    return frame;
+}
+#endif
+
+static inline PyFrameObject*
+_PyThreadState_GetFrameBorrow(PyThreadState *tstate)
+{
+    PyFrameObject *frame = PyThreadState_GetFrame(tstate);
+    Py_XDECREF(frame);
+    return frame;  // borrowed reference
+}
+
+
+// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline PyInterpreterState *
+PyInterpreterState_Get(void)
+{
+    PyThreadState *tstate;
+    PyInterpreterState *interp;
+
+    tstate = PyThreadState_GET();
+    if (tstate == NULL) {
+        Py_FatalError("GIL released (tstate is NULL)");
+    }
+    interp = tstate->interp;
+    if (interp == NULL) {
+        Py_FatalError("no current interpreter");
+    }
+    return interp;
+}
+#endif
+
+
+// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6
+#if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6
+static inline uint64_t
+PyThreadState_GetID(PyThreadState *tstate)
+{
+    assert(tstate != NULL);
+    return tstate->id;
+}
+#endif
+
+
+// bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1
+#if PY_VERSION_HEX < 0x030900A1
+static inline PyObject*
+PyObject_CallNoArgs(PyObject *func)
+{
+    return PyObject_CallFunctionObjArgs(func, NULL);
+}
+#endif
+
+
+// bpo-39245 made PyObject_CallOneArg() public (previously called
+// _PyObject_CallOneArg) in Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4
+static inline PyObject*
+PyObject_CallOneArg(PyObject *func, PyObject *arg)
+{
+    return PyObject_CallFunctionObjArgs(func, arg, NULL);
+}
+#endif
+
+
+// bpo-40024 added PyModule_AddType() to Python 3.9.0a5
+#if PY_VERSION_HEX < 0x030900A5
+static inline int
+PyModule_AddType(PyObject *module, PyTypeObject *type)
+{
+    const char *name, *dot;
+
+    if (PyType_Ready(type) < 0) {
+        return -1;
+    }
+
+    // inline _PyType_Name()
+    name = type->tp_name;
+    assert(name != NULL);
+    dot = strrchr(name, '.');
+    if (dot != NULL) {
+        name = dot + 1;
+    }
+
+    Py_INCREF(type);
+    if (PyModule_AddObject(module, name, (PyObject *)type) < 0) {
+        Py_DECREF(type);
+        return -1;
+    }
+
+    return 0;
+}
+#endif
+
+
+// bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6.
+// bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2.
+#if PY_VERSION_HEX < 0x030900A6
+static inline int
+PyObject_GC_IsTracked(PyObject* obj)
+{
+    return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj));
+}
+#endif
+
+// bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6.
+// bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final.
+#if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0
+static inline int
+PyObject_GC_IsFinalized(PyObject *obj)
+{
+    return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED((PyGC_Head *)(obj)-1));
+}
+#endif
+
+
+// bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4
+#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE)
+static inline int
+_Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) {
+    return ob->ob_type == type;
+}
+#define Py_IS_TYPE(ob, type) _Py_IS_TYPE((const PyObject*)(ob), type)
+#endif
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // PYTHONCAPI_COMPAT

contrib/clang-format-ignorelist

@@ -1,11 +1,12 b''
 # Files that just need to be migrated to the formatter.
 # Do not add new files here!
 mercurial/cext/manifest.c
 mercurial/cext/osutil.c
 # Vendored code that we should never format:
 syntax: glob
 contrib/python-zstandard/**.c
 contrib/python-zstandard/**.h
 hgext/fsmonitor/pywatchman/**.c
 mercurial/thirdparty/**.c
 mercurial/thirdparty/**.h
+mercurial/pythoncapi_compat.h

contrib/python-zstandard/c-ext/bufferutil.c

@@ -1,792 +1,792 b''
 /**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(BufferWithSegments__doc__,
 "BufferWithSegments - A memory buffer holding known sub-segments.\n"
 "\n"
 "This type represents a contiguous chunk of memory containing N discrete\n"
 "items within sub-segments of that memory.\n"
 "\n"
 "Segments within the buffer are stored as an array of\n"
 "``(offset, length)`` pairs, where each element is an unsigned 64-bit\n"
 "integer using the host/native bit order representation.\n"
 "\n"
 "The type exists to facilitate operations against N>1 items without the\n"
 "overhead of Python object creation and management.\n"
 );
 
 static void BufferWithSegments_dealloc(ZstdBufferWithSegments* self) {
 	/* Backing memory is either canonically owned by a Py_buffer or by us. */
 	if (self->parent.buf) {
 		PyBuffer_Release(&self->parent);
 	}
 	else if (self->useFree) {
 		free(self->data);
 	}
 	else {
 		PyMem_Free(self->data);
 	}
 
 	self->data = NULL;
 
 	if (self->useFree) {
 		free(self->segments);
 	}
 	else {
 		PyMem_Free(self->segments);
 	}
 
 	self->segments = NULL;
 
 	PyObject_Del(self);
 }
 
 static int BufferWithSegments_init(ZstdBufferWithSegments* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		"segments",
 		NULL
 	};
 
 	Py_buffer segments;
 	Py_ssize_t segmentCount;
 	Py_ssize_t i;
 
 	memset(&self->parent, 0, sizeof(self->parent));
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*y*:BufferWithSegments",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*s*:BufferWithSegments",
 #endif
 		kwlist, &self->parent, &segments)) {
 		return -1;
 	}
 
 	if (!PyBuffer_IsContiguous(&self->parent, 'C') || self->parent.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError, "data buffer should be contiguous and have a single dimension");
 		goto except;
 	}
 
 	if (!PyBuffer_IsContiguous(&segments, 'C') || segments.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError, "segments buffer should be contiguous and have a single dimension");
 		goto except;
 	}
 
 	if (segments.len % sizeof(BufferSegment)) {
 		PyErr_Format(PyExc_ValueError, "segments array size is not a multiple of %zu",
 			sizeof(BufferSegment));
 		goto except;
 	}
 
 	segmentCount = segments.len / sizeof(BufferSegment);
 
 	/* Validate segments data, as blindly trusting it could lead to arbitrary
 	memory access. */
 	for (i = 0; i < segmentCount; i++) {
 		BufferSegment* segment = &((BufferSegment*)(segments.buf))[i];
 
 		if (segment->offset + segment->length > (unsigned long long)self->parent.len) {
 			PyErr_SetString(PyExc_ValueError, "offset within segments array references memory outside buffer");
 			goto except;
 			return -1;
 		}
 	}
 
 	/* Make a copy of the segments data. It is cheap to do so and is a guard
 	   against caller changing offsets, which has security implications. */
 	self->segments = PyMem_Malloc(segments.len);
 	if (!self->segments) {
 		PyErr_NoMemory();
 		goto except;
 	}
 
 	memcpy(self->segments, segments.buf, segments.len);
 	PyBuffer_Release(&segments);
 
 	self->data = self->parent.buf;
 	self->dataSize = self->parent.len;
 	self->segmentCount = segmentCount;
 
 	return 0;
 
 except:
 	PyBuffer_Release(&self->parent);
 	PyBuffer_Release(&segments);
 	return -1;
 }
 
 /**
  * Construct a BufferWithSegments from existing memory and offsets.
  *
  * Ownership of the backing memory and BufferSegments will be transferred to
  * the created object and freed when the BufferWithSegments is destroyed.
  */
 ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize,
 	BufferSegment* segments, Py_ssize_t segmentsSize) {
 	ZstdBufferWithSegments* result = NULL;
 	Py_ssize_t i;
 
 	if (NULL == data) {
 		PyErr_SetString(PyExc_ValueError, "data is NULL");
 		return NULL;
 	}
 
 	if (NULL == segments) {
 		PyErr_SetString(PyExc_ValueError, "segments is NULL");
 		return NULL;
 	}
 
 	for (i = 0; i < segmentsSize; i++) {
 		BufferSegment* segment = &segments[i];
 
 		if (segment->offset + segment->length > dataSize) {
 			PyErr_SetString(PyExc_ValueError, "offset in segments overflows buffer size");
 			return NULL;
 		}
 	}
 
 	result = PyObject_New(ZstdBufferWithSegments, &ZstdBufferWithSegmentsType);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	result->useFree = 0;
 
 	memset(&result->parent, 0, sizeof(result->parent));
 	result->data = data;
 	result->dataSize = dataSize;
 	result->segments = segments;
 	result->segmentCount = segmentsSize;
 
 	return result;
 }
 
 static Py_ssize_t BufferWithSegments_length(ZstdBufferWithSegments* self) {
 	return self->segmentCount;
 }
 
 static ZstdBufferSegment* BufferWithSegments_item(ZstdBufferWithSegments* self, Py_ssize_t i) {
 	ZstdBufferSegment* result = NULL;
 
 	if (i < 0) {
 		PyErr_SetString(PyExc_IndexError, "offset must be non-negative");
 		return NULL;
 	}
 
 	if (i >= self->segmentCount) {
 		PyErr_Format(PyExc_IndexError, "offset must be less than %zd", self->segmentCount);
 		return NULL;
 	}
 
 	if (self->segments[i].length > PY_SSIZE_T_MAX) {
 		PyErr_Format(PyExc_ValueError,
 			"item at offset %zd is too large for this platform", i);
 		return NULL;
 	}
 
 	result = (ZstdBufferSegment*)PyObject_CallObject((PyObject*)&ZstdBufferSegmentType, NULL);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	result->parent = (PyObject*)self;
 	Py_INCREF(self);
 
 	result->data = (char*)self->data + self->segments[i].offset;
 	result->dataSize = (Py_ssize_t)self->segments[i].length;
 	result->offset = self->segments[i].offset;
 
 	return result;
 }
 
 #if PY_MAJOR_VERSION >= 3
 static int BufferWithSegments_getbuffer(ZstdBufferWithSegments* self, Py_buffer* view, int flags) {
 	if (self->dataSize > PY_SSIZE_T_MAX) {
 		view->obj = NULL;
 		PyErr_SetString(PyExc_BufferError, "buffer is too large for this platform");
 		return -1;
 	}
 
 	return PyBuffer_FillInfo(view, (PyObject*)self, self->data, (Py_ssize_t)self->dataSize, 1, flags);
 }
 #else
 static Py_ssize_t BufferWithSegments_getreadbuffer(ZstdBufferWithSegments* self, Py_ssize_t segment, void **ptrptr) {
 	if (segment != 0) {
 		PyErr_SetString(PyExc_ValueError, "segment number must be 0");
 		return -1;
 	}
 
 	if (self->dataSize > PY_SSIZE_T_MAX) {
 		PyErr_SetString(PyExc_ValueError, "buffer is too large for this platform");
 		return -1;
 	}
 
 	*ptrptr = self->data;
 	return (Py_ssize_t)self->dataSize;
 }
 
 static Py_ssize_t BufferWithSegments_getsegcount(ZstdBufferWithSegments* self, Py_ssize_t* len) {
 	if (len) {
 		*len = 1;
 	}
 
 	return 1;
 }
 #endif
 
 PyDoc_STRVAR(BufferWithSegments_tobytes__doc__,
 "Obtain a bytes instance for this buffer.\n"
 );
 
 static PyObject* BufferWithSegments_tobytes(ZstdBufferWithSegments* self) {
 	if (self->dataSize > PY_SSIZE_T_MAX) {
 		PyErr_SetString(PyExc_ValueError, "buffer is too large for this platform");
 		return NULL;
 	}
 
 	return PyBytes_FromStringAndSize(self->data, (Py_ssize_t)self->dataSize);
 }
 
 PyDoc_STRVAR(BufferWithSegments_segments__doc__,
 "Obtain a BufferSegments describing segments in this sintance.\n"
 );
 
 static ZstdBufferSegments* BufferWithSegments_segments(ZstdBufferWithSegments* self) {
 	ZstdBufferSegments* result = (ZstdBufferSegments*)PyObject_CallObject((PyObject*)&ZstdBufferSegmentsType, NULL);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	result->parent = (PyObject*)self;
 	Py_INCREF(self);
 	result->segments = self->segments;
 	result->segmentCount = self->segmentCount;
 
 	return result;
 }
 
 static PySequenceMethods BufferWithSegments_sq = {
 	(lenfunc)BufferWithSegments_length, /* sq_length */
 	0, /* sq_concat */
 	0, /* sq_repeat */
 	(ssizeargfunc)BufferWithSegments_item, /* sq_item */
 	0, /* sq_ass_item */
 	0, /* sq_contains */
 	0, /* sq_inplace_concat */
 	0 /* sq_inplace_repeat */
 };
 
 static PyBufferProcs BufferWithSegments_as_buffer = {
 #if PY_MAJOR_VERSION >= 3
 	(getbufferproc)BufferWithSegments_getbuffer, /* bf_getbuffer */
 	0 /* bf_releasebuffer */
 #else
 	(readbufferproc)BufferWithSegments_getreadbuffer, /* bf_getreadbuffer */
 	0, /* bf_getwritebuffer */
 	(segcountproc)BufferWithSegments_getsegcount, /* bf_getsegcount */
 	0 /* bf_getcharbuffer */
 #endif
 };
 
 static PyMethodDef BufferWithSegments_methods[] = {
 	{ "segments", (PyCFunction)BufferWithSegments_segments,
 	  METH_NOARGS, BufferWithSegments_segments__doc__ },
 	{ "tobytes", (PyCFunction)BufferWithSegments_tobytes,
 	  METH_NOARGS, BufferWithSegments_tobytes__doc__ },
 	{ NULL, NULL }
 };
 
 static PyMemberDef BufferWithSegments_members[] = {
 	{ "size", T_ULONGLONG, offsetof(ZstdBufferWithSegments, dataSize),
 	  READONLY, "total size of the buffer in bytes" },
 	{ NULL }
 };
 
 PyTypeObject ZstdBufferWithSegmentsType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.BufferWithSegments", /* tp_name */
 	sizeof(ZstdBufferWithSegments),/* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)BufferWithSegments_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	&BufferWithSegments_sq,    /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	&BufferWithSegments_as_buffer, /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	BufferWithSegments__doc__, /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	BufferWithSegments_methods, /* tp_methods */
 	BufferWithSegments_members, /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	(initproc)BufferWithSegments_init, /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 PyDoc_STRVAR(BufferSegments__doc__,
 "BufferSegments - Represents segments/offsets within a BufferWithSegments\n"
 );
 
 static void BufferSegments_dealloc(ZstdBufferSegments* self) {
 	Py_CLEAR(self->parent);
 	PyObject_Del(self);
 }
 
 #if PY_MAJOR_VERSION >= 3
 static int BufferSegments_getbuffer(ZstdBufferSegments* self, Py_buffer* view, int flags) {
 	return PyBuffer_FillInfo(view, (PyObject*)self,
 		(void*)self->segments, self->segmentCount * sizeof(BufferSegment),
 		1, flags);
 }
 #else
 static Py_ssize_t BufferSegments_getreadbuffer(ZstdBufferSegments* self, Py_ssize_t segment, void **ptrptr) {
 	if (segment != 0) {
 		PyErr_SetString(PyExc_ValueError, "segment number must be 0");
 		return -1;
 	}
 
 	*ptrptr = (void*)self->segments;
 	return self->segmentCount * sizeof(BufferSegment);
 }
 
 static Py_ssize_t BufferSegments_getsegcount(ZstdBufferSegments* self, Py_ssize_t* len) {
 	if (len) {
 		*len = 1;
 	}
 
 	return 1;
 }
 #endif
 
 static PyBufferProcs BufferSegments_as_buffer = {
 #if PY_MAJOR_VERSION >= 3
 	(getbufferproc)BufferSegments_getbuffer,
 	0
 #else
 	(readbufferproc)BufferSegments_getreadbuffer,
 	0,
 	(segcountproc)BufferSegments_getsegcount,
 	0
 #endif
 };
 
 PyTypeObject ZstdBufferSegmentsType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.BufferSegments", /* tp_name */
 	sizeof(ZstdBufferSegments),/* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)BufferSegments_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	0,                         /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	&BufferSegments_as_buffer, /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	BufferSegments__doc__,     /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	0,                         /* tp_methods */
 	0,                         /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	0,                         /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 PyDoc_STRVAR(BufferSegment__doc__,
 	"BufferSegment - Represents a segment within a BufferWithSegments\n"
 );
 
 static void BufferSegment_dealloc(ZstdBufferSegment* self) {
 	Py_CLEAR(self->parent);
 	PyObject_Del(self);
 }
 
 static Py_ssize_t BufferSegment_length(ZstdBufferSegment* self) {
 	return self->dataSize;
 }
 
 #if PY_MAJOR_VERSION >= 3
 static int BufferSegment_getbuffer(ZstdBufferSegment* self, Py_buffer* view, int flags) {
 	return PyBuffer_FillInfo(view, (PyObject*)self,
 		self->data, self->dataSize, 1, flags);
 }
 #else
 static Py_ssize_t BufferSegment_getreadbuffer(ZstdBufferSegment* self, Py_ssize_t segment, void **ptrptr) {
 	if (segment != 0) {
 		PyErr_SetString(PyExc_ValueError, "segment number must be 0");
 		return -1;
 	}
 
 	*ptrptr = self->data;
 	return self->dataSize;
 }
 
 static Py_ssize_t BufferSegment_getsegcount(ZstdBufferSegment* self, Py_ssize_t* len) {
 	if (len) {
 		*len = 1;
 	}
 
 	return 1;
 }
 #endif
 
 PyDoc_STRVAR(BufferSegment_tobytes__doc__,
 "Obtain a bytes instance for this segment.\n"
 );
 
 static PyObject* BufferSegment_tobytes(ZstdBufferSegment* self) {
 	return PyBytes_FromStringAndSize(self->data, self->dataSize);
 }
 
 static PySequenceMethods BufferSegment_sq = {
 	(lenfunc)BufferSegment_length, /* sq_length */
 	0, /* sq_concat */
 	0, /* sq_repeat */
 	0, /* sq_item */
 	0, /* sq_ass_item */
 	0, /* sq_contains */
 	0, /* sq_inplace_concat */
 	0 /* sq_inplace_repeat */
 };
 
 static PyBufferProcs BufferSegment_as_buffer = {
 #if PY_MAJOR_VERSION >= 3
 	(getbufferproc)BufferSegment_getbuffer,
 	0
 #else
 	(readbufferproc)BufferSegment_getreadbuffer,
 	0,
 	(segcountproc)BufferSegment_getsegcount,
 	0
 #endif
 };
 
 static PyMethodDef BufferSegment_methods[] = {
 	{ "tobytes", (PyCFunction)BufferSegment_tobytes,
 	  METH_NOARGS, BufferSegment_tobytes__doc__ },
 	{ NULL, NULL }
 };
 
 static PyMemberDef BufferSegment_members[] = {
 	{ "offset", T_ULONGLONG, offsetof(ZstdBufferSegment, offset), READONLY,
 	  "offset of segment within parent buffer" },
 	  { NULL }
 };
 
 PyTypeObject ZstdBufferSegmentType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.BufferSegment", /* tp_name */
 	sizeof(ZstdBufferSegment),/* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)BufferSegment_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	&BufferSegment_sq,         /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	&BufferSegment_as_buffer,  /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	BufferSegment__doc__,      /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	BufferSegment_methods,     /* tp_methods */
 	BufferSegment_members,     /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	0,                         /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 PyDoc_STRVAR(BufferWithSegmentsCollection__doc__,
 "Represents a collection of BufferWithSegments.\n"
 );
 
 static void BufferWithSegmentsCollection_dealloc(ZstdBufferWithSegmentsCollection* self) {
 	Py_ssize_t i;
 
 	if (self->firstElements) {
 		PyMem_Free(self->firstElements);
 		self->firstElements = NULL;
 	}
 
 	if (self->buffers) {
 		for (i = 0; i < self->bufferCount; i++) {
 			Py_CLEAR(self->buffers[i]);
 		}
 
 		PyMem_Free(self->buffers);
 		self->buffers = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 static int BufferWithSegmentsCollection_init(ZstdBufferWithSegmentsCollection* self, PyObject* args) {
 	Py_ssize_t size;
 	Py_ssize_t i;
 	Py_ssize_t offset = 0;
 
 	size = PyTuple_Size(args);
 	if (-1 == size) {
 		return -1;
 	}
 
 	if (0 == size) {
 		PyErr_SetString(PyExc_ValueError, "must pass at least 1 argument");
 		return -1;
 	}
 
 	for (i = 0; i < size; i++) {
 		PyObject* item = PyTuple_GET_ITEM(args, i);
 		if (!PyObject_TypeCheck(item, &ZstdBufferWithSegmentsType)) {
 			PyErr_SetString(PyExc_TypeError, "arguments must be BufferWithSegments instances");
 			return -1;
 		}
 
 		if (0 == ((ZstdBufferWithSegments*)item)->segmentCount ||
 			0 == ((ZstdBufferWithSegments*)item)->dataSize) {
 			PyErr_SetString(PyExc_ValueError, "ZstdBufferWithSegments cannot be empty");
 			return -1;
 		}
 	}
 
 	self->buffers = PyMem_Malloc(size * sizeof(ZstdBufferWithSegments*));
 	if (NULL == self->buffers) {
 		PyErr_NoMemory();
 		return -1;
 	}
 
 	self->firstElements = PyMem_Malloc(size * sizeof(Py_ssize_t));
 	if (NULL == self->firstElements) {
 		PyMem_Free(self->buffers);
 		self->buffers = NULL;
 		PyErr_NoMemory();
 		return -1;
 	}
 
 	self->bufferCount = size;
 
 	for (i = 0; i < size; i++) {
 		ZstdBufferWithSegments* item = (ZstdBufferWithSegments*)PyTuple_GET_ITEM(args, i);
 
 		self->buffers[i] = item;
 		Py_INCREF(item);
 
 		if (i > 0) {
 			self->firstElements[i - 1] = offset;
 		}
 
 		offset += item->segmentCount;
 	}
 
 	self->firstElements[size - 1] = offset;
 
 	return 0;
 }
 
 static PyObject* BufferWithSegmentsCollection_size(ZstdBufferWithSegmentsCollection* self) {
 	Py_ssize_t i;
 	Py_ssize_t j;
 	unsigned long long size = 0;
 
 	for (i = 0; i < self->bufferCount; i++) {
 		for (j = 0; j < self->buffers[i]->segmentCount; j++) {
 			size += self->buffers[i]->segments[j].length;
 		}
 	}
 
 	return PyLong_FromUnsignedLongLong(size);
 }
 
 Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection* self) {
 	return self->firstElements[self->bufferCount - 1];
 }
 
 static ZstdBufferSegment* BufferWithSegmentsCollection_item(ZstdBufferWithSegmentsCollection* self, Py_ssize_t i) {
 	Py_ssize_t bufferOffset;
 
 	if (i < 0) {
 		PyErr_SetString(PyExc_IndexError, "offset must be non-negative");
 		return NULL;
 	}
 
 	if (i >= BufferWithSegmentsCollection_length(self)) {
 		PyErr_Format(PyExc_IndexError, "offset must be less than %zd",
 			BufferWithSegmentsCollection_length(self));
 		return NULL;
 	}
 
 	for (bufferOffset = 0; bufferOffset < self->bufferCount; bufferOffset++) {
 		Py_ssize_t offset = 0;
 
 		if (i < self->firstElements[bufferOffset]) {
 			if (bufferOffset > 0) {
 				offset = self->firstElements[bufferOffset - 1];
 			}
 
 			return BufferWithSegments_item(self->buffers[bufferOffset], i - offset);
 		}
 	}
 
 	PyErr_SetString(ZstdError, "error resolving segment; this should not happen");
 	return NULL;
 }
 
 static PySequenceMethods BufferWithSegmentsCollection_sq = {
 	(lenfunc)BufferWithSegmentsCollection_length, /* sq_length */
 	0, /* sq_concat */
 	0, /* sq_repeat */
 	(ssizeargfunc)BufferWithSegmentsCollection_item, /* sq_item */
 	0, /* sq_ass_item */
 	0, /* sq_contains */
 	0, /* sq_inplace_concat */
 	0 /* sq_inplace_repeat */
 };
 
 static PyMethodDef BufferWithSegmentsCollection_methods[] = {
 	{ "size", (PyCFunction)BufferWithSegmentsCollection_size,
 	  METH_NOARGS, PyDoc_STR("total size in bytes of all segments") },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdBufferWithSegmentsCollectionType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.BufferWithSegmentsCollection", /* tp_name */
 	sizeof(ZstdBufferWithSegmentsCollection),/* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)BufferWithSegmentsCollection_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	&BufferWithSegmentsCollection_sq, /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	BufferWithSegmentsCollection__doc__, /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	/* TODO implement iterator for performance. */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	BufferWithSegmentsCollection_methods, /* tp_methods */
 	0,                         /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	(initproc)BufferWithSegmentsCollection_init, /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 void bufferutil_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdBufferWithSegmentsType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdBufferWithSegmentsType, &PyType_Type);
 	if (PyType_Ready(&ZstdBufferWithSegmentsType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdBufferWithSegmentsType);
 	PyModule_AddObject(mod, "BufferWithSegments", (PyObject*)&ZstdBufferWithSegmentsType);
 
-	Py_TYPE(&ZstdBufferSegmentsType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdBufferSegmentsType, &PyType_Type);
 	if (PyType_Ready(&ZstdBufferSegmentsType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdBufferSegmentsType);
 	PyModule_AddObject(mod, "BufferSegments", (PyObject*)&ZstdBufferSegmentsType);
 
-	Py_TYPE(&ZstdBufferSegmentType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdBufferSegmentType, &PyType_Type);
 	if (PyType_Ready(&ZstdBufferSegmentType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdBufferSegmentType);
 	PyModule_AddObject(mod, "BufferSegment", (PyObject*)&ZstdBufferSegmentType);
 
-	Py_TYPE(&ZstdBufferWithSegmentsCollectionType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdBufferWithSegmentsCollectionType, &PyType_Type);
 	if (PyType_Ready(&ZstdBufferWithSegmentsCollectionType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdBufferWithSegmentsCollectionType);
 	PyModule_AddObject(mod, "BufferWithSegmentsCollection", (PyObject*)&ZstdBufferWithSegmentsCollectionType);
 }

contrib/python-zstandard/c-ext/compressionchunker.c

@@ -1,360 +1,360 b''
 /**
 * Copyright (c) 2018-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdCompressionChunkerIterator__doc__,
 	"Iterator of output chunks from ZstdCompressionChunker.\n"
 );
 
 static void ZstdCompressionChunkerIterator_dealloc(ZstdCompressionChunkerIterator* self) {
 	Py_XDECREF(self->chunker);
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdCompressionChunkerIterator_iter(PyObject* self) {
 	Py_INCREF(self);
 	return self;
 }
 
 static PyObject* ZstdCompressionChunkerIterator_iternext(ZstdCompressionChunkerIterator* self) {
 	size_t zresult;
 	PyObject* chunk;
 	ZstdCompressionChunker* chunker = self->chunker;
 	ZSTD_EndDirective zFlushMode;
 
 	if (self->mode != compressionchunker_mode_normal && chunker->input.pos != chunker->input.size) {
 		PyErr_SetString(ZstdError, "input should have been fully consumed before calling flush() or finish()");
 		return NULL;
 	}
 
 	if (chunker->finished) {
 		return NULL;
 	}
 
 	/* If we have data left in the input, consume it. */
 	while (chunker->input.pos < chunker->input.size) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(chunker->compressor->cctx, &chunker->output,
 			&chunker->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		/* Input is fully consumed. */
 		if (chunker->input.pos == chunker->input.size) {
 			chunker->input.src = NULL;
 			chunker->input.pos = 0;
 			chunker->input.size = 0;
 			PyBuffer_Release(&chunker->inBuffer);
 		}
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		/* If it produced a full output chunk, emit it. */
 		if (chunker->output.pos == chunker->output.size) {
 			chunk = PyBytes_FromStringAndSize(chunker->output.dst, chunker->output.pos);
 			if (!chunk) {
 				return NULL;
 			}
 
 			chunker->output.pos = 0;
 
 			return chunk;
 		}
 
 		/* Else continue to compress available input data. */
 	}
 
 	/* We also need this here for the special case of an empty input buffer. */
 	if (chunker->input.pos == chunker->input.size) {
 		chunker->input.src = NULL;
 		chunker->input.pos = 0;
 		chunker->input.size = 0;
 		PyBuffer_Release(&chunker->inBuffer);
 	}
 
 	/* No more input data. A partial chunk may be in chunker->output.
 	 * If we're in normal compression mode, we're done. Otherwise if we're in
 	 * flush or finish mode, we need to emit what data remains.
 	 */
 	if (self->mode == compressionchunker_mode_normal) {
 		/* We don't need to set StopIteration. */
 		return NULL;
 	}
 
 	if (self->mode == compressionchunker_mode_flush) {
 		zFlushMode = ZSTD_e_flush;
 	}
 	else if (self->mode == compressionchunker_mode_finish) {
 		zFlushMode = ZSTD_e_end;
 	}
 	else {
 		PyErr_SetString(ZstdError, "unhandled compression mode; this should never happen");
 		return NULL;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_compressStream2(chunker->compressor->cctx, &chunker->output,
 		&chunker->input, zFlushMode);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "zstd compress error: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	if (!zresult && chunker->output.pos == 0) {
 		return NULL;
 	}
 
 	chunk = PyBytes_FromStringAndSize(chunker->output.dst, chunker->output.pos);
 	if (!chunk) {
 		return NULL;
 	}
 
 	chunker->output.pos = 0;
 
 	if (!zresult && self->mode == compressionchunker_mode_finish) {
 		chunker->finished = 1;
 	}
 
 	return chunk;
 }
 
 PyTypeObject ZstdCompressionChunkerIteratorType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionChunkerIterator", /* tp_name */
 	sizeof(ZstdCompressionChunkerIterator), /* tp_basicsize */
 	0,                               /* tp_itemsize */
 	(destructor)ZstdCompressionChunkerIterator_dealloc, /* tp_dealloc */
 	0,                               /* tp_print */
 	0,                               /* tp_getattr */
 	0,                               /* tp_setattr */
 	0,                               /* tp_compare */
 	0,                               /* tp_repr */
 	0,                               /* tp_as_number */
 	0,                               /* tp_as_sequence */
 	0,                               /* tp_as_mapping */
 	0,                               /* tp_hash */
 	0,                               /* tp_call */
 	0,                               /* tp_str */
 	0,                               /* tp_getattro */
 	0,                               /* tp_setattro */
 	0,                               /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionChunkerIterator__doc__, /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	ZstdCompressionChunkerIterator_iter, /* tp_iter */
 	(iternextfunc)ZstdCompressionChunkerIterator_iternext, /* tp_iternext */
 	0,                               /* tp_methods */
 	0,                               /* tp_members */
 	0,                               /* tp_getset */
 	0,                               /* tp_base */
 	0,                               /* tp_dict */
 	0,                               /* tp_descr_get */
 	0,                               /* tp_descr_set */
 	0,                               /* tp_dictoffset */
 	0,                               /* tp_init */
 	0,                               /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 PyDoc_STRVAR(ZstdCompressionChunker__doc__,
 	"Compress chunks iteratively into exact chunk sizes.\n"
 );
 
 static void ZstdCompressionChunker_dealloc(ZstdCompressionChunker* self) {
 	PyBuffer_Release(&self->inBuffer);
 	self->input.src = NULL;
 
 	PyMem_Free(self->output.dst);
 	self->output.dst = NULL;
 
 	Py_XDECREF(self->compressor);
 
 	PyObject_Del(self);
 }
 
 static ZstdCompressionChunkerIterator* ZstdCompressionChunker_compress(ZstdCompressionChunker* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	ZstdCompressionChunkerIterator* result;
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot call compress() after compression finished");
 		return NULL;
 	}
 
 	if (self->inBuffer.obj) {
 		PyErr_SetString(ZstdError,
 			"cannot perform operation before consuming output from previous operation");
 		return NULL;
 	}
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:compress",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:compress",
 #endif
 		kwlist, &self->inBuffer)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&self->inBuffer, 'C') || self->inBuffer.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		PyBuffer_Release(&self->inBuffer);
 		return NULL;
 	}
 
 	result = (ZstdCompressionChunkerIterator*)PyObject_CallObject((PyObject*)&ZstdCompressionChunkerIteratorType, NULL);
 	if (!result) {
 		PyBuffer_Release(&self->inBuffer);
 		return NULL;
 	}
 
 	self->input.src = self->inBuffer.buf;
 	self->input.size = self->inBuffer.len;
 	self->input.pos = 0;
 
 	result->chunker = self;
 	Py_INCREF(result->chunker);
 
 	result->mode = compressionchunker_mode_normal;
 
 	return result;
 }
 
 static ZstdCompressionChunkerIterator* ZstdCompressionChunker_finish(ZstdCompressionChunker* self) {
 	ZstdCompressionChunkerIterator* result;
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot call finish() after compression finished");
 		return NULL;
 	}
 
 	if (self->inBuffer.obj) {
 		PyErr_SetString(ZstdError,
 			"cannot call finish() before consuming output from previous operation");
 		return NULL;
 	}
 
 	result = (ZstdCompressionChunkerIterator*)PyObject_CallObject((PyObject*)&ZstdCompressionChunkerIteratorType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	result->chunker = self;
 	Py_INCREF(result->chunker);
 
 	result->mode = compressionchunker_mode_finish;
 
 	return result;
 }
 
 static ZstdCompressionChunkerIterator* ZstdCompressionChunker_flush(ZstdCompressionChunker* self, PyObject* args, PyObject* kwargs) {
 	ZstdCompressionChunkerIterator* result;
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot call flush() after compression finished");
 		return NULL;
 	}
 
 	if (self->inBuffer.obj) {
 		PyErr_SetString(ZstdError,
 			"cannot call flush() before consuming output from previous operation");
 		return NULL;
 	}
 
 	result = (ZstdCompressionChunkerIterator*)PyObject_CallObject((PyObject*)&ZstdCompressionChunkerIteratorType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	result->chunker = self;
 	Py_INCREF(result->chunker);
 
 	result->mode = compressionchunker_mode_flush;
 
 	return result;
 }
 
 static PyMethodDef ZstdCompressionChunker_methods[] = {
 	{ "compress", (PyCFunction)ZstdCompressionChunker_compress, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("compress data") },
 	{ "finish", (PyCFunction)ZstdCompressionChunker_finish, METH_NOARGS,
 	PyDoc_STR("finish compression operation") },
 	{ "flush", (PyCFunction)ZstdCompressionChunker_flush, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("finish compression operation") },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdCompressionChunkerType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionChunkerType",  /* tp_name */
 	sizeof(ZstdCompressionChunker),     /* tp_basicsize */
 	0,                                  /* tp_itemsize */
 	(destructor)ZstdCompressionChunker_dealloc, /* tp_dealloc */
 	0,                                  /* tp_print */
 	0,                                  /* tp_getattr */
 	0,                                  /* tp_setattr */
 	0,                                  /* tp_compare */
 	0,                                  /* tp_repr */
 	0,                                  /* tp_as_number */
 	0,                                  /* tp_as_sequence */
 	0,                                  /* tp_as_mapping */
 	0,                                  /* tp_hash */
 	0,                                  /* tp_call */
 	0,                                  /* tp_str */
 	0,                                  /* tp_getattro */
 	0,                                  /* tp_setattro */
 	0,                                  /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionChunker__doc__,      /* tp_doc */
 	0,                                  /* tp_traverse */
 	0,                                  /* tp_clear */
 	0,                                  /* tp_richcompare */
 	0,                                  /* tp_weaklistoffset */
 	0,                                  /* tp_iter */
 	0,                                  /* tp_iternext */
 	ZstdCompressionChunker_methods,     /* tp_methods */
 	0,                                  /* tp_members */
 	0,                                  /* tp_getset */
 	0,                                  /* tp_base */
 	0,                                  /* tp_dict */
 	0,                                  /* tp_descr_get */
 	0,                                  /* tp_descr_set */
 	0,                                  /* tp_dictoffset */
 	0,                                  /* tp_init */
 	0,                                  /* tp_alloc */
 	PyType_GenericNew,                  /* tp_new */
 };
 
 void compressionchunker_module_init(PyObject* module) {
-	Py_TYPE(&ZstdCompressionChunkerIteratorType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionChunkerIteratorType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionChunkerIteratorType) < 0) {
 		return;
 	}
 
-	Py_TYPE(&ZstdCompressionChunkerType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionChunkerType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionChunkerType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/compressiondict.c

@@ -1,411 +1,411 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"dict_size",
 		"samples",
 		"k",
 		"d",
 		"notifications",
 		"dict_id",
 		"level",
 		"steps",
 		"threads",
 		NULL
 	};
 
 	size_t capacity;
 	PyObject* samples;
 	unsigned k = 0;
 	unsigned d = 0;
 	unsigned notifications = 0;
 	unsigned dictID = 0;
 	int level = 0;
 	unsigned steps = 0;
 	int threads = 0;
 	ZDICT_cover_params_t params;
 	Py_ssize_t samplesLen;
 	Py_ssize_t i;
 	size_t samplesSize = 0;
 	void* sampleBuffer = NULL;
 	size_t* sampleSizes = NULL;
 	void* sampleOffset;
 	Py_ssize_t sampleSize;
 	void* dict = NULL;
 	size_t zresult;
 	ZstdCompressionDict* result = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|IIIIiIi:train_dictionary",
 		kwlist, &capacity, &PyList_Type, &samples,
 		&k, &d, &notifications, &dictID, &level, &steps, &threads)) {
 		return NULL;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	memset(&params, 0, sizeof(params));
 	params.k = k;
 	params.d = d;
 	params.steps = steps;
 	params.nbThreads = threads;
 	params.zParams.notificationLevel = notifications;
 	params.zParams.dictID = dictID;
 	params.zParams.compressionLevel = level;
 
 	/* Figure out total size of input samples. */
 	samplesLen = PyList_Size(samples);
 	for (i = 0; i < samplesLen; i++) {
 		PyObject* sampleItem = PyList_GET_ITEM(samples, i);
 
 		if (!PyBytes_Check(sampleItem)) {
 			PyErr_SetString(PyExc_ValueError, "samples must be bytes");
 			return NULL;
 		}
 		samplesSize += PyBytes_GET_SIZE(sampleItem);
 	}
 
 	sampleBuffer = PyMem_Malloc(samplesSize);
 	if (!sampleBuffer) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	sampleSizes = PyMem_Malloc(samplesLen * sizeof(size_t));
 	if (!sampleSizes) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	sampleOffset = sampleBuffer;
 	for (i = 0; i < samplesLen; i++) {
 		PyObject* sampleItem = PyList_GET_ITEM(samples, i);
 		sampleSize = PyBytes_GET_SIZE(sampleItem);
 		sampleSizes[i] = sampleSize;
 		memcpy(sampleOffset, PyBytes_AS_STRING(sampleItem), sampleSize);
 		sampleOffset = (char*)sampleOffset + sampleSize;
 	}
 
 	dict = PyMem_Malloc(capacity);
 	if (!dict) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	/* No parameters uses the default function, which will use default params
 	   and call ZDICT_optimizeTrainFromBuffer_cover under the hood. */
 	if (!params.k && !params.d && !params.zParams.compressionLevel
 		&& !params.zParams.notificationLevel && !params.zParams.dictID) {
 		zresult = ZDICT_trainFromBuffer(dict, capacity, sampleBuffer,
 			sampleSizes, (unsigned)samplesLen);
 	}
 	/* Use optimize mode if user controlled steps or threads explicitly. */
 	else if (params.steps || params.nbThreads) {
 		zresult = ZDICT_optimizeTrainFromBuffer_cover(dict, capacity,
 			sampleBuffer, sampleSizes, (unsigned)samplesLen, &params);
 	}
 	/* Non-optimize mode with explicit control. */
 	else {
 		zresult = ZDICT_trainFromBuffer_cover(dict, capacity,
 			sampleBuffer, sampleSizes, (unsigned)samplesLen, params);
 	}
 	Py_END_ALLOW_THREADS
 
 	if (ZDICT_isError(zresult)) {
 		PyMem_Free(dict);
 		PyErr_Format(ZstdError, "cannot train dict: %s", ZDICT_getErrorName(zresult));
 		goto finally;
 	}
 
 	result = PyObject_New(ZstdCompressionDict, &ZstdCompressionDictType);
 	if (!result) {
 		PyMem_Free(dict);
 		goto finally;
 	}
 
 	result->dictData = dict;
 	result->dictSize = zresult;
 	result->dictType = ZSTD_dct_fullDict;
 	result->d = params.d;
 	result->k = params.k;
 	result->cdict = NULL;
 	result->ddict = NULL;
 
 finally:
 	PyMem_Free(sampleBuffer);
 	PyMem_Free(sampleSizes);
 
 	return result;
 }
 
 int ensure_ddict(ZstdCompressionDict* dict) {
 	if (dict->ddict) {
 		return 0;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	dict->ddict = ZSTD_createDDict_advanced(dict->dictData, dict->dictSize,
 		ZSTD_dlm_byRef, dict->dictType, ZSTD_defaultCMem);
 	Py_END_ALLOW_THREADS
 	if (!dict->ddict) {
 		PyErr_SetString(ZstdError, "could not create decompression dict");
 		return 1;
 	}
 
 	return 0;
 }
 
 PyDoc_STRVAR(ZstdCompressionDict__doc__,
 "ZstdCompressionDict(data) - Represents a computed compression dictionary\n"
 "\n"
 "This type holds the results of a computed Zstandard compression dictionary.\n"
 "Instances are obtained by calling ``train_dictionary()`` or by passing\n"
 "bytes obtained from another source into the constructor.\n"
 );
 
 static int ZstdCompressionDict_init(ZstdCompressionDict* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		"dict_type",
 		NULL
 	};
 
 	int result = -1;
 	Py_buffer source;
 	unsigned dictType = ZSTD_dct_auto;
 
 	self->dictData = NULL;
 	self->dictSize = 0;
 	self->cdict = NULL;
 	self->ddict = NULL;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|I:ZstdCompressionDict",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|I:ZstdCompressionDict",
 #endif
 		kwlist, &source, &dictType)) {
 		return -1;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	if (dictType != ZSTD_dct_auto && dictType != ZSTD_dct_rawContent
 		&& dictType != ZSTD_dct_fullDict) {
 		PyErr_Format(PyExc_ValueError,
 			"invalid dictionary load mode: %d; must use DICT_TYPE_* constants",
 			dictType);
 		goto finally;
 	}
 
 	self->dictType = dictType;
 
 	self->dictData = PyMem_Malloc(source.len);
 	if (!self->dictData) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	memcpy(self->dictData, source.buf, source.len);
 	self->dictSize = source.len;
 
 	result = 0;
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
 static void ZstdCompressionDict_dealloc(ZstdCompressionDict* self) {
 	if (self->cdict) {
 		ZSTD_freeCDict(self->cdict);
 		self->cdict = NULL;
 	}
 
 	if (self->ddict) {
 		ZSTD_freeDDict(self->ddict);
 		self->ddict = NULL;
 	}
 
 	if (self->dictData) {
 		PyMem_Free(self->dictData);
 		self->dictData = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 PyDoc_STRVAR(ZstdCompressionDict_precompute_compress__doc__,
 "Precompute a dictionary so it can be used by multiple compressors.\n"
 );
 
 static PyObject* ZstdCompressionDict_precompute_compress(ZstdCompressionDict* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"level",
 		"compression_params",
 		NULL
 	};
 
 	int level = 0;
 	ZstdCompressionParametersObject* compressionParams = NULL;
 	ZSTD_compressionParameters cParams;
 	size_t zresult;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!:precompute_compress", kwlist,
 		&level, &ZstdCompressionParametersType, &compressionParams)) {
 		return NULL;
 	}
 
 	if (level && compressionParams) {
 		PyErr_SetString(PyExc_ValueError,
 			"must only specify one of level or compression_params");
 		return NULL;
 	}
 
 	if (!level && !compressionParams) {
 		PyErr_SetString(PyExc_ValueError,
 			"must specify one of level or compression_params");
 		return NULL;
 	}
 
 	if (self->cdict) {
 		zresult = ZSTD_freeCDict(self->cdict);
 		self->cdict = NULL;
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "unable to free CDict: %s",
 				ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 	}
 
 	if (level) {
 		cParams = ZSTD_getCParams(level, 0, self->dictSize);
 	}
 	else {
 		if (to_cparams(compressionParams, &cParams)) {
 			return NULL;
 		}
 	}
 
 	assert(!self->cdict);
 	self->cdict = ZSTD_createCDict_advanced(self->dictData, self->dictSize,
 		ZSTD_dlm_byRef, self->dictType, cParams, ZSTD_defaultCMem);
 
 	if (!self->cdict) {
 		PyErr_SetString(ZstdError, "unable to precompute dictionary");
 		return NULL;
 	}
 
 	Py_RETURN_NONE;
 }
 
 static PyObject* ZstdCompressionDict_dict_id(ZstdCompressionDict* self) {
 	unsigned dictID = ZDICT_getDictID(self->dictData, self->dictSize);
 
 	return PyLong_FromLong(dictID);
 }
 
 static PyObject* ZstdCompressionDict_as_bytes(ZstdCompressionDict* self) {
 	return PyBytes_FromStringAndSize(self->dictData, self->dictSize);
 }
 
 static PyMethodDef ZstdCompressionDict_methods[] = {
 	{ "dict_id", (PyCFunction)ZstdCompressionDict_dict_id, METH_NOARGS,
 	PyDoc_STR("dict_id() -- obtain the numeric dictionary ID") },
 	{ "as_bytes", (PyCFunction)ZstdCompressionDict_as_bytes, METH_NOARGS,
 	PyDoc_STR("as_bytes() -- obtain the raw bytes constituting the dictionary data") },
 	{ "precompute_compress", (PyCFunction)ZstdCompressionDict_precompute_compress,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressionDict_precompute_compress__doc__ },
 	{ NULL, NULL }
 };
 
 static PyMemberDef ZstdCompressionDict_members[] = {
 	{ "k", T_UINT, offsetof(ZstdCompressionDict, k), READONLY,
 	  "segment size" },
 	{ "d", T_UINT, offsetof(ZstdCompressionDict, d), READONLY,
 	  "dmer size" },
 	{ NULL }
 };
 
 static Py_ssize_t ZstdCompressionDict_length(ZstdCompressionDict* self) {
 	return self->dictSize;
 }
 
 static PySequenceMethods ZstdCompressionDict_sq = {
 	(lenfunc)ZstdCompressionDict_length, /* sq_length */
 	0,                                   /* sq_concat */
 	0,                                   /* sq_repeat */
 	0,                                   /* sq_item */
 	0,                                   /* sq_ass_item */
 	0,                                   /* sq_contains */
 	0,                                   /* sq_inplace_concat */
 	0                                    /* sq_inplace_repeat */
 };
 
 PyTypeObject ZstdCompressionDictType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionDict",     /* tp_name */
 	sizeof(ZstdCompressionDict),    /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)ZstdCompressionDict_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	&ZstdCompressionDict_sq,        /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionDict__doc__,     /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdCompressionDict_methods,    /* tp_methods */
 	ZstdCompressionDict_members,    /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	(initproc)ZstdCompressionDict_init, /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void compressiondict_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdCompressionDictType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionDictType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionDictType) < 0) {
 		return;
 	}
 
 	Py_INCREF((PyObject*)&ZstdCompressionDictType);
 	PyModule_AddObject(mod, "ZstdCompressionDict",
 		(PyObject*)&ZstdCompressionDictType);
 }

contrib/python-zstandard/c-ext/compressionparams.c

@@ -1,572 +1,572 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value) {
 	size_t zresult = ZSTD_CCtxParams_setParameter(params, param, value);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "unable to set compression context parameter: %s",
 			ZSTD_getErrorName(zresult));
 		return 1;
 	}
 
 	return 0;
 }
 
 #define TRY_SET_PARAMETER(params, param, value) if (set_parameter(params, param, value)) return -1;
 
 #define TRY_COPY_PARAMETER(source, dest, param) { \
 	int result; \
 	size_t zresult = ZSTD_CCtxParams_getParameter(source, param, &result); \
 	if (ZSTD_isError(zresult)) { \
 		return 1; \
 	} \
 	zresult = ZSTD_CCtxParams_setParameter(dest, param, result); \
 	if (ZSTD_isError(zresult)) { \
 		return 1; \
 	} \
 }
 
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj) {
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_nbWorkers);
 
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_format);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_compressionLevel);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_windowLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_hashLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_chainLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_searchLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_minMatch);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_targetLength);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_strategy);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_contentSizeFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_checksumFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_dictIDFlag);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_jobSize);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_overlapLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_forceMaxWindow);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_enableLongDistanceMatching);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmMinMatch);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmBucketSizeLog);
 	TRY_COPY_PARAMETER(obj->params, params, ZSTD_c_ldmHashRateLog);
 
 	return 0;
 }
 
 int reset_params(ZstdCompressionParametersObject* params) {
 	if (params->params) {
 		ZSTD_CCtxParams_reset(params->params);
 	}
 	else {
 		params->params = ZSTD_createCCtxParams();
 		if (!params->params) {
 			PyErr_NoMemory();
 			return 1;
 		}
 	}
 
 	return set_parameters(params->params, params);
 }
 
 #define TRY_GET_PARAMETER(params, param, value) { \
     size_t zresult = ZSTD_CCtxParams_getParameter(params, param, value); \
     if (ZSTD_isError(zresult)) { \
         PyErr_Format(ZstdError, "unable to retrieve parameter: %s", ZSTD_getErrorName(zresult)); \
         return 1; \
     } \
 }
 
 int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams) {
 	int value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_windowLog, &value);
 	cparams->windowLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_chainLog, &value);
 	cparams->chainLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_hashLog, &value);
 	cparams->hashLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_searchLog, &value);
 	cparams->searchLog = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_minMatch, &value);
 	cparams->minMatch = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_targetLength, &value);
 	cparams->targetLength = value;
 
 	TRY_GET_PARAMETER(params->params, ZSTD_c_strategy, &value);
 	cparams->strategy = value;
 
 	return 0;
 }
 
 static int ZstdCompressionParameters_init(ZstdCompressionParametersObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"format",
 		"compression_level",
 		"window_log",
 		"hash_log",
 		"chain_log",
 		"search_log",
 		"min_match",
 		"target_length",
 		"compression_strategy",
 		"strategy",
 		"write_content_size",
 		"write_checksum",
 		"write_dict_id",
 		"job_size",
 		"overlap_log",
 		"overlap_size_log",
 		"force_max_window",
 		"enable_ldm",
 		"ldm_hash_log",
 		"ldm_min_match",
 		"ldm_bucket_size_log",
 		"ldm_hash_rate_log",
 		"ldm_hash_every_log",
 		"threads",
 		NULL
 	};
 
 	int format = 0;
 	int compressionLevel = 0;
 	int windowLog = 0;
 	int hashLog = 0;
 	int chainLog = 0;
 	int searchLog = 0;
 	int minMatch = 0;
 	int targetLength = 0;
 	int compressionStrategy = -1;
 	int strategy = -1;
 	int contentSizeFlag = 1;
 	int checksumFlag = 0;
 	int dictIDFlag = 0;
 	int jobSize = 0;
 	int overlapLog = -1;
 	int overlapSizeLog = -1;
 	int forceMaxWindow = 0;
 	int enableLDM = 0;
 	int ldmHashLog = 0;
 	int ldmMinMatch = 0;
 	int ldmBucketSizeLog = 0;
 	int ldmHashRateLog = -1;
 	int ldmHashEveryLog = -1;
 	int threads = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs,
 		"|iiiiiiiiiiiiiiiiiiiiiiii:CompressionParameters",
 		kwlist, &format, &compressionLevel, &windowLog, &hashLog, &chainLog,
 		&searchLog, &minMatch, &targetLength, &compressionStrategy, &strategy,
 		&contentSizeFlag, &checksumFlag, &dictIDFlag, &jobSize, &overlapLog,
 		&overlapSizeLog, &forceMaxWindow, &enableLDM, &ldmHashLog, &ldmMinMatch,
 		&ldmBucketSizeLog, &ldmHashRateLog, &ldmHashEveryLog, &threads)) {
 		return -1;
 	}
 
 	if (reset_params(self)) {
 		return -1;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	/* We need to set ZSTD_c_nbWorkers before ZSTD_c_jobSize and ZSTD_c_overlapLog
 	 * because setting ZSTD_c_nbWorkers resets the other parameters. */
 	TRY_SET_PARAMETER(self->params, ZSTD_c_nbWorkers, threads);
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_format, format);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_compressionLevel, compressionLevel);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_windowLog, windowLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_hashLog, hashLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_chainLog, chainLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_searchLog, searchLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_minMatch, minMatch);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_targetLength, targetLength);
 
 	if (compressionStrategy != -1 && strategy != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both compression_strategy and strategy");
 		return -1;
     }
 
 	if (compressionStrategy != -1) {
 		strategy = compressionStrategy;
 	}
 	else if (strategy == -1) {
 		strategy = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_strategy, strategy);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_contentSizeFlag, contentSizeFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_checksumFlag, checksumFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_dictIDFlag, dictIDFlag);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_jobSize, jobSize);
 
 	if (overlapLog != -1 && overlapSizeLog != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both overlap_log and overlap_size_log");
 		return -1;
 	}
 
 	if (overlapSizeLog != -1) {
 		overlapLog = overlapSizeLog;
 	}
 	else if (overlapLog == -1) {
 		overlapLog = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_overlapLog, overlapLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_forceMaxWindow, forceMaxWindow);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_enableLongDistanceMatching, enableLDM);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashLog, ldmHashLog);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmMinMatch, ldmMinMatch);
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmBucketSizeLog, ldmBucketSizeLog);
 
 	if (ldmHashRateLog != -1 && ldmHashEveryLog != -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot specify both ldm_hash_rate_log and ldm_hash_everyLog");
 		return -1;
 	}
 
 	if (ldmHashEveryLog != -1) {
 		ldmHashRateLog = ldmHashEveryLog;
 	}
 	else if (ldmHashRateLog == -1) {
 		ldmHashRateLog = 0;
 	}
 
 	TRY_SET_PARAMETER(self->params, ZSTD_c_ldmHashRateLog, ldmHashRateLog);
 
 	return 0;
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters_from_level__doc__,
 "Create a CompressionParameters from a compression level and target sizes\n"
 );
 
 ZstdCompressionParametersObject* CompressionParameters_from_level(PyObject* undef, PyObject* args, PyObject* kwargs) {
 	int managedKwargs = 0;
 	int level;
 	PyObject* sourceSize = NULL;
 	PyObject* dictSize = NULL;
 	unsigned PY_LONG_LONG iSourceSize = 0;
 	Py_ssize_t iDictSize = 0;
 	PyObject* val;
 	ZSTD_compressionParameters params;
 	ZstdCompressionParametersObject* result = NULL;
 	int res;
 
 	if (!PyArg_ParseTuple(args, "i:from_level",
 		&level)) {
 		return NULL;
 	}
 
 	if (!kwargs) {
 		kwargs = PyDict_New();
 		if (!kwargs) {
 			return NULL;
 		}
 		managedKwargs = 1;
 	}
 
 	sourceSize = PyDict_GetItemString(kwargs, "source_size");
 	if (sourceSize) {
 #if PY_MAJOR_VERSION >= 3
 		iSourceSize = PyLong_AsUnsignedLongLong(sourceSize);
 		if (iSourceSize == (unsigned PY_LONG_LONG)(-1)) {
 			goto cleanup;
 		}
 #else
 		iSourceSize = PyInt_AsUnsignedLongLongMask(sourceSize);
 #endif
 
 		PyDict_DelItemString(kwargs, "source_size");
 	}
 
 	dictSize = PyDict_GetItemString(kwargs, "dict_size");
 	if (dictSize) {
 #if PY_MAJOR_VERSION >= 3
 		iDictSize = PyLong_AsSsize_t(dictSize);
 #else
 		iDictSize = PyInt_AsSsize_t(dictSize);
 #endif
 		if (iDictSize == -1) {
 			goto cleanup;
 		}
 
 		PyDict_DelItemString(kwargs, "dict_size");
 	}
 
 
 	params = ZSTD_getCParams(level, iSourceSize, iDictSize);
 
 	/* Values derived from the input level and sizes are passed along to the
 	   constructor. But only if a value doesn't already exist. */
 	val = PyDict_GetItemString(kwargs, "window_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.windowLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "window_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "chain_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.chainLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "chain_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "hash_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.hashLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "hash_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "search_log");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.searchLog);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "search_log", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "min_match");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.minMatch);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "min_match", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "target_length");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.targetLength);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "target_length", val);
 		Py_DECREF(val);
 	}
 
 	val = PyDict_GetItemString(kwargs, "compression_strategy");
 	if (!val) {
 		val = PyLong_FromUnsignedLong(params.strategy);
 		if (!val) {
 			goto cleanup;
 		}
 		PyDict_SetItemString(kwargs, "compression_strategy", val);
 		Py_DECREF(val);
 	}
 
 	result = PyObject_New(ZstdCompressionParametersObject, &ZstdCompressionParametersType);
 	if (!result) {
 		goto cleanup;
 	}
 
 	result->params = NULL;
 
 	val = PyTuple_New(0);
 	if (!val) {
 		Py_CLEAR(result);
 		goto cleanup;
 	}
 
 	res = ZstdCompressionParameters_init(result, val, kwargs);
 	Py_DECREF(val);
 
 	if (res) {
 		Py_CLEAR(result);
 		goto cleanup;
 	}
 
 cleanup:
 	if (managedKwargs) {
 		Py_DECREF(kwargs);
 	}
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters_estimated_compression_context_size__doc__,
 "Estimate the size in bytes of a compression context for compression parameters\n"
 );
 
 PyObject* ZstdCompressionParameters_estimated_compression_context_size(ZstdCompressionParametersObject* self) {
 	return PyLong_FromSize_t(ZSTD_estimateCCtxSize_usingCCtxParams(self->params));
 }
 
 PyDoc_STRVAR(ZstdCompressionParameters__doc__,
 "ZstdCompressionParameters: low-level control over zstd compression");
 
 static void ZstdCompressionParameters_dealloc(ZstdCompressionParametersObject* self) {
 	if (self->params) {
 		ZSTD_freeCCtxParams(self->params);
 		self->params = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 #define PARAM_GETTER(name, param) PyObject* ZstdCompressionParameters_get_##name(PyObject* self, void* unused) { \
     int result; \
     size_t zresult; \
     ZstdCompressionParametersObject* p = (ZstdCompressionParametersObject*)(self); \
     zresult = ZSTD_CCtxParams_getParameter(p->params, param, &result); \
     if (ZSTD_isError(zresult)) { \
         PyErr_Format(ZstdError, "unable to get compression parameter: %s", \
             ZSTD_getErrorName(zresult)); \
         return NULL; \
     } \
     return PyLong_FromLong(result); \
 }
 
 PARAM_GETTER(format, ZSTD_c_format)
 PARAM_GETTER(compression_level, ZSTD_c_compressionLevel)
 PARAM_GETTER(window_log, ZSTD_c_windowLog)
 PARAM_GETTER(hash_log, ZSTD_c_hashLog)
 PARAM_GETTER(chain_log, ZSTD_c_chainLog)
 PARAM_GETTER(search_log, ZSTD_c_searchLog)
 PARAM_GETTER(min_match, ZSTD_c_minMatch)
 PARAM_GETTER(target_length, ZSTD_c_targetLength)
 PARAM_GETTER(compression_strategy, ZSTD_c_strategy)
 PARAM_GETTER(write_content_size, ZSTD_c_contentSizeFlag)
 PARAM_GETTER(write_checksum, ZSTD_c_checksumFlag)
 PARAM_GETTER(write_dict_id, ZSTD_c_dictIDFlag)
 PARAM_GETTER(job_size, ZSTD_c_jobSize)
 PARAM_GETTER(overlap_log, ZSTD_c_overlapLog)
 PARAM_GETTER(force_max_window, ZSTD_c_forceMaxWindow)
 PARAM_GETTER(enable_ldm, ZSTD_c_enableLongDistanceMatching)
 PARAM_GETTER(ldm_hash_log, ZSTD_c_ldmHashLog)
 PARAM_GETTER(ldm_min_match, ZSTD_c_ldmMinMatch)
 PARAM_GETTER(ldm_bucket_size_log, ZSTD_c_ldmBucketSizeLog)
 PARAM_GETTER(ldm_hash_rate_log, ZSTD_c_ldmHashRateLog)
 PARAM_GETTER(threads, ZSTD_c_nbWorkers)
 
 static PyMethodDef ZstdCompressionParameters_methods[] = {
 	{
 		"from_level",
 		(PyCFunction)CompressionParameters_from_level,
 		METH_VARARGS | METH_KEYWORDS | METH_STATIC,
 		ZstdCompressionParameters_from_level__doc__
 	},
 	{
 		"estimated_compression_context_size",
 		(PyCFunction)ZstdCompressionParameters_estimated_compression_context_size,
 		METH_NOARGS,
 		ZstdCompressionParameters_estimated_compression_context_size__doc__
 	},
 	{ NULL, NULL }
 };
 
 #define GET_SET_ENTRY(name) { #name, ZstdCompressionParameters_get_##name, NULL, NULL, NULL }
 
 static PyGetSetDef ZstdCompressionParameters_getset[] = {
 	GET_SET_ENTRY(format),
 	GET_SET_ENTRY(compression_level),
 	GET_SET_ENTRY(window_log),
 	GET_SET_ENTRY(hash_log),
 	GET_SET_ENTRY(chain_log),
 	GET_SET_ENTRY(search_log),
 	GET_SET_ENTRY(min_match),
 	GET_SET_ENTRY(target_length),
 	GET_SET_ENTRY(compression_strategy),
 	GET_SET_ENTRY(write_content_size),
 	GET_SET_ENTRY(write_checksum),
 	GET_SET_ENTRY(write_dict_id),
 	GET_SET_ENTRY(threads),
 	GET_SET_ENTRY(job_size),
 	GET_SET_ENTRY(overlap_log),
 	/* TODO remove this deprecated attribute */
 	{ "overlap_size_log", ZstdCompressionParameters_get_overlap_log, NULL, NULL, NULL },
 	GET_SET_ENTRY(force_max_window),
 	GET_SET_ENTRY(enable_ldm),
 	GET_SET_ENTRY(ldm_hash_log),
 	GET_SET_ENTRY(ldm_min_match),
 	GET_SET_ENTRY(ldm_bucket_size_log),
 	GET_SET_ENTRY(ldm_hash_rate_log),
 	/* TODO remove this deprecated attribute */
 	{ "ldm_hash_every_log", ZstdCompressionParameters_get_ldm_hash_rate_log, NULL, NULL, NULL },
 	{ NULL }
 };
 
 PyTypeObject ZstdCompressionParametersType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"ZstdCompressionParameters", /* tp_name */
 	sizeof(ZstdCompressionParametersObject), /* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)ZstdCompressionParameters_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	0,                         /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionParameters__doc__, /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	ZstdCompressionParameters_methods, /* tp_methods */
 	0,                          /* tp_members */
 	ZstdCompressionParameters_getset,  /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	(initproc)ZstdCompressionParameters_init, /* tp_init */
 	0,                         /* tp_alloc */
 	PyType_GenericNew,         /* tp_new */
 };
 
 void compressionparams_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdCompressionParametersType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionParametersType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionParametersType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&ZstdCompressionParametersType);
 	PyModule_AddObject(mod, "ZstdCompressionParameters",
 		(PyObject*)&ZstdCompressionParametersType);
 
 	/* TODO remove deprecated alias. */
 	Py_INCREF(&ZstdCompressionParametersType);
 	PyModule_AddObject(mod, "CompressionParameters",
 		(PyObject*)&ZstdCompressionParametersType);
 }

contrib/python-zstandard/c-ext/compressionreader.c

@@ -1,818 +1,818 b''
 /**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 static void set_unsupported_operation(void) {
 	PyObject* iomod;
 	PyObject* exc;
 
 	iomod = PyImport_ImportModule("io");
 	if (NULL == iomod) {
 		return;
 	}
 
 	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
 	if (NULL == exc) {
 		Py_DECREF(iomod);
 		return;
 	}
 
 	PyErr_SetNone(exc);
 	Py_DECREF(exc);
 	Py_DECREF(iomod);
 }
 
 static void reader_dealloc(ZstdCompressionReader* self) {
 	Py_XDECREF(self->compressor);
 	Py_XDECREF(self->reader);
 
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	PyObject_Del(self);
 }
 
 static ZstdCompressionReader* reader_enter(ZstdCompressionReader* self) {
 	if (self->entered) {
 		PyErr_SetString(PyExc_ValueError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
 	self->entered = 1;
 
 	Py_INCREF(self);
 	return self;
 }
 
 static PyObject* reader_exit(ZstdCompressionReader* self, PyObject* args) {
 	PyObject* exc_type;
 	PyObject* exc_value;
 	PyObject* exc_tb;
 
 	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
 		return NULL;
 	}
 
 	self->entered = 0;
 	self->closed = 1;
 
 	/* Release resources associated with source. */
 	Py_CLEAR(self->reader);
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
     Py_CLEAR(self->compressor);
 
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_readable(ZstdCompressionReader* self) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* reader_writable(ZstdCompressionReader* self) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_seekable(ZstdCompressionReader* self) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_readline(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_readlines(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_write(PyObject* self, PyObject* args) {
 	PyErr_SetString(PyExc_OSError, "stream is not writable");
 	return NULL;
 }
 
 static PyObject* reader_writelines(PyObject* self, PyObject* args) {
 	PyErr_SetString(PyExc_OSError, "stream is not writable");
 	return NULL;
 }
 
 static PyObject* reader_isatty(PyObject* self) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_flush(PyObject* self) {
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_close(ZstdCompressionReader* self) {
 	self->closed = 1;
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_tell(ZstdCompressionReader* self) {
 	/* TODO should this raise OSError since stream isn't seekable? */
 	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
 }
 
 int read_compressor_input(ZstdCompressionReader* self) {
 	if (self->finishedInput) {
 		return 0;
 	}
 
 	if (self->input.pos != self->input.size) {
 		return 0;
 	}
 
 	if (self->reader) {
 		Py_buffer buffer;
 
 		assert(self->readResult == NULL);
 
 		self->readResult = PyObject_CallMethod(self->reader, "read",
 		    "k", self->readSize);
 
 		if (NULL == self->readResult) {
 			return -1;
 		}
 
 		memset(&buffer, 0, sizeof(buffer));
 
 		if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
 			return -1;
 		}
 
 		/* EOF */
 		if (0 == buffer.len) {
 			self->finishedInput = 1;
 			Py_CLEAR(self->readResult);
 		}
 		else {
 			self->input.src = buffer.buf;
 			self->input.size = buffer.len;
 			self->input.pos = 0;
 		}
 
 		PyBuffer_Release(&buffer);
 	}
 	else {
 		assert(self->buffer.buf);
 
 		self->input.src = self->buffer.buf;
 		self->input.size = self->buffer.len;
 		self->input.pos = 0;
 	}
 
 	return 1;
 }
 
 int compress_input(ZstdCompressionReader* self, ZSTD_outBuffer* output) {
 	size_t oldPos;
 	size_t zresult;
 
 	/* If we have data left over, consume it. */
 	if (self->input.pos < self->input.size) {
 		oldPos = output->pos;
 
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->compressor->cctx,
 		    output, &self->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		self->bytesCompressed += output->pos - oldPos;
 
 		/* Input exhausted. Clear out state tracking. */
 		if (self->input.pos == self->input.size) {
 			memset(&self->input, 0, sizeof(self->input));
 			Py_CLEAR(self->readResult);
 
 			if (self->buffer.buf) {
 				self->finishedInput = 1;
 			}
 		}
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			return -1;
 		}
 	}
 
     if (output->pos && output->pos == output->size) {
         return 1;
     }
     else {
         return 0;
     }
 }
 
 static PyObject* reader_read(ZstdCompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	size_t zresult;
 	size_t oldPos;
 	int readResult, compressResult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (size == -1) {
 		return PyObject_CallMethod((PyObject*)self, "readall", NULL);
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	self->output.dst = resultBuffer;
 	self->output.size = resultSize;
 	self->output.pos = 0;
 
 readinput:
 
     compressResult = compress_input(self, &self->output);
 
 	if (-1 == compressResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == compressResult) {
 		/* There is room in the output. We fall through to below, which will
 		 * either get more input for us or will attempt to end the stream.
 		 */
 	}
 	else if (1 == compressResult) {
 		memset(&self->output, 0, sizeof(self->output));
 		return result;
 	}
 	else {
 		assert(0);
 	}
 
 	readResult = read_compressor_input(self);
 
 	if (-1 == readResult) {
 		return NULL;
 	}
 	else if (0 == readResult) { }
 	else if (1 == readResult) { }
 	else {
 		assert(0);
 	}
 
 	if (self->input.size) {
 		goto readinput;
 	}
 
 	/* Else EOF */
 	oldPos = self->output.pos;
 
 	zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 		&self->input, ZSTD_e_end);
 
 	self->bytesCompressed += self->output.pos - oldPos;
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error ending compression stream: %s",
 			ZSTD_getErrorName(zresult));
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	assert(self->output.pos);
 
 	if (0 == zresult) {
 		self->finishedOutput = 1;
 	}
 
 	if (safe_pybytes_resize(&result, self->output.pos)) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	memset(&self->output, 0, sizeof(self->output));
 
 	return result;
 }
 
 static PyObject* reader_read1(ZstdCompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
 	int compressResult;
 	size_t oldPos;
 	size_t zresult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n:read1", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	if (size == -1) {
 		size = ZSTD_CStreamOutSize();
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	output.dst = resultBuffer;
 	output.size = resultSize;
 	output.pos = 0;
 
 	/* read1() is supposed to use at most 1 read() from the underlying stream.
 	   However, we can't satisfy this requirement with compression because
 	   not every input will generate output. We /could/ flush the compressor,
 	   but this may not be desirable. We allow multiple read() from the
 	   underlying stream. But unlike read(), we return as soon as output data
 	   is available.
 	*/
 
 	compressResult = compress_input(self, &output);
 
 	if (-1 == compressResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == compressResult || 1 == compressResult) { }
 	else {
 		assert(0);
 	}
 
 	if (output.pos) {
 		goto finally;
 	}
 
 	while (!self->finishedInput) {
 		int readResult = read_compressor_input(self);
 
 		if (-1 == readResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == readResult || 1 == readResult) { }
 		else {
 			assert(0);
 		}
 
 		compressResult = compress_input(self, &output);
 
 		if (-1 == compressResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == compressResult || 1 == compressResult) { }
 		else {
 			assert(0);
 		}
 
 		if (output.pos) {
 			goto finally;
 		}
 	}
 
 	/* EOF */
 	oldPos = output.pos;
 
 	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
         ZSTD_e_end);
 
 	self->bytesCompressed += output.pos - oldPos;
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error ending compression stream: %s",
 		    ZSTD_getErrorName(zresult));
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	if (zresult == 0) {
 		self->finishedOutput = 1;
 	}
 
 finally:
 	if (result) {
 		if (safe_pybytes_resize(&result, output.pos)) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 	}
 
 	return result;
 }
 
 static PyObject* reader_readall(PyObject* self) {
 	PyObject* chunks = NULL;
 	PyObject* empty = NULL;
 	PyObject* result = NULL;
 
 	/* Our strategy is to collect chunks into a list then join all the
 	 * chunks at the end. We could potentially use e.g. an io.BytesIO. But
 	 * this feels simple enough to implement and avoids potentially expensive
 	 * reallocations of large buffers.
 	 */
 	chunks = PyList_New(0);
 	if (NULL == chunks) {
 		return NULL;
 	}
 
 	while (1) {
 		PyObject* chunk = PyObject_CallMethod(self, "read", "i", 1048576);
 		if (NULL == chunk) {
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		if (!PyBytes_Size(chunk)) {
 			Py_DECREF(chunk);
 			break;
 		}
 
 		if (PyList_Append(chunks, chunk)) {
 			Py_DECREF(chunk);
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		Py_DECREF(chunk);
 	}
 
 	empty = PyBytes_FromStringAndSize("", 0);
 	if (NULL == empty) {
 		Py_DECREF(chunks);
 		return NULL;
 	}
 
 	result = PyObject_CallMethod(empty, "join", "O", chunks);
 
 	Py_DECREF(empty);
 	Py_DECREF(chunks);
 
 	return result;
 }
 
 static PyObject* reader_readinto(ZstdCompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	ZSTD_outBuffer output;
 	int readResult, compressResult;
 	PyObject* result = NULL;
 	size_t zresult;
 	size_t oldPos;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 		    "destination buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 	compressResult = compress_input(self, &output);
 
 	if (-1 == compressResult) {
 		goto finally;
 	}
 	else if (0 == compressResult) {	}
 	else if (1 == compressResult) {
 		result = PyLong_FromSize_t(output.pos);
 		goto finally;
 	}
 	else {
 		assert(0);
 	}
 
 	while (!self->finishedInput) {
 		readResult = read_compressor_input(self);
 
 		if (-1 == readResult) {
 			goto finally;
 		}
 		else if (0 == readResult || 1 == readResult) {}
 		else {
 			assert(0);
 		}
 
 		compressResult = compress_input(self, &output);
 
 		if (-1 == compressResult) {
 			goto finally;
 		}
 		else if (0 == compressResult) { }
 		else if (1 == compressResult) {
 			result = PyLong_FromSize_t(output.pos);
 			goto finally;
 		}
 		else {
 			assert(0);
 		}
 	}
 
 	/* EOF */
 	oldPos = output.pos;
 
 	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
 	    ZSTD_e_end);
 
 	self->bytesCompressed += self->output.pos - oldPos;
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error ending compression stream: %s",
 		    ZSTD_getErrorName(zresult));
 		goto finally;
 	}
 
 	assert(output.pos);
 
 	if (0 == zresult) {
 		self->finishedOutput = 1;
 	}
 
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_readinto1(ZstdCompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	PyObject* result = NULL;
 	ZSTD_outBuffer output;
 	int compressResult;
 	size_t oldPos;
 	size_t zresult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto1", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 		    "destination buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 	compressResult = compress_input(self, &output);
 
 	if (-1 == compressResult) {
 		goto finally;
 	}
 	else if (0 == compressResult || 1 == compressResult) { }
 	else {
 		assert(0);
 	}
 
 	if (output.pos) {
 		result = PyLong_FromSize_t(output.pos);
 		goto finally;
 	}
 
 	while (!self->finishedInput) {
 		int readResult = read_compressor_input(self);
 
 		if (-1 == readResult) {
 			goto finally;
 		}
 		else if (0 == readResult || 1 == readResult) { }
 		else {
 			assert(0);
 		}
 
 		compressResult = compress_input(self, &output);
 
 		if (-1 == compressResult) {
 			goto finally;
 		}
 		else if (0 == compressResult) { }
 		else if (1 == compressResult) {
 			result = PyLong_FromSize_t(output.pos);
 			goto finally;
 		}
 		else {
 			assert(0);
 		}
 
 		/* If we produced output and we're not done with input, emit
 		 * that output now, as we've hit restrictions of read1().
 		 */
 		if (output.pos && !self->finishedInput) {
 			result = PyLong_FromSize_t(output.pos);
 			goto finally;
 		}
 
 		/* Otherwise we either have no output or we've exhausted the
 		 * input. Either we try to get more input or we fall through
 		 * to EOF below */
 	}
 
 	/* EOF */
 	oldPos = output.pos;
 
 	zresult = ZSTD_compressStream2(self->compressor->cctx, &output, &self->input,
 	    ZSTD_e_end);
 
 	self->bytesCompressed += self->output.pos - oldPos;
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error ending compression stream: %s",
 		    ZSTD_getErrorName(zresult));
 		goto finally;
 	}
 
 	assert(output.pos);
 
 	if (0 == zresult) {
 		self->finishedOutput = 1;
 	}
 
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_iter(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iternext(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyMethodDef reader_methods[] = {
 	{ "__enter__", (PyCFunction)reader_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context") },
 	{ "__exit__", (PyCFunction)reader_exit, METH_VARARGS,
 	PyDoc_STR("Exit a compression context") },
 	{ "close", (PyCFunction)reader_close, METH_NOARGS,
 	PyDoc_STR("Close the stream so it cannot perform any more operations") },
 	{ "flush", (PyCFunction)reader_flush, METH_NOARGS, PyDoc_STR("no-ops") },
 	{ "isatty", (PyCFunction)reader_isatty, METH_NOARGS, PyDoc_STR("Returns False") },
 	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("read compressed data") },
 	{ "read1", (PyCFunction)reader_read1, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readinto", (PyCFunction)reader_readinto, METH_VARARGS, NULL },
 	{ "readinto1", (PyCFunction)reader_readinto1, METH_VARARGS, NULL },
 	{ "readline", (PyCFunction)reader_readline, METH_VARARGS, PyDoc_STR("Not implemented") },
 	{ "readlines", (PyCFunction)reader_readlines, METH_VARARGS, PyDoc_STR("Not implemented") },
 	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,
 	PyDoc_STR("Returns False") },
 	{ "tell", (PyCFunction)reader_tell, METH_NOARGS,
 	PyDoc_STR("Returns current number of bytes compressed") },
 	{ "writable", (PyCFunction)reader_writable, METH_NOARGS,
 	PyDoc_STR("Returns False") },
 	{ "write", reader_write, METH_VARARGS, PyDoc_STR("Raises OSError") },
 	{ "writelines", reader_writelines, METH_VARARGS, PyDoc_STR("Not implemented") },
 	{ NULL, NULL }
 };
 
 static PyMemberDef reader_members[] = {
 	{ "closed", T_BOOL, offsetof(ZstdCompressionReader, closed),
 	  READONLY, "whether stream is closed" },
 	{ NULL }
 };
 
 PyTypeObject ZstdCompressionReaderType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionReader", /* tp_name */
 	sizeof(ZstdCompressionReader), /* tp_basicsize */
 	0, /* tp_itemsize */
 	(destructor)reader_dealloc, /* tp_dealloc */
 	0, /* tp_print */
 	0, /* tp_getattr */
 	0, /* tp_setattr */
 	0, /* tp_compare */
 	0, /* tp_repr */
 	0, /* tp_as_number */
 	0, /* tp_as_sequence */
 	0, /* tp_as_mapping */
 	0, /* tp_hash */
 	0, /* tp_call */
 	0, /* tp_str */
 	0, /* tp_getattro */
 	0, /* tp_setattro */
 	0, /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT, /* tp_flags */
 	0, /* tp_doc */
 	0, /* tp_traverse */
 	0, /* tp_clear */
 	0, /* tp_richcompare */
 	0, /* tp_weaklistoffset */
 	reader_iter, /* tp_iter */
 	reader_iternext, /* tp_iternext */
 	reader_methods, /* tp_methods */
 	reader_members, /* tp_members */
 	0, /* tp_getset */
 	0, /* tp_base */
 	0, /* tp_dict */
 	0, /* tp_descr_get */
 	0, /* tp_descr_set */
 	0, /* tp_dictoffset */
 	0, /* tp_init */
 	0, /* tp_alloc */
 	PyType_GenericNew, /* tp_new */
 };
 
 void compressionreader_module_init(PyObject* mod) {
 	/* TODO make reader a sub-class of io.RawIOBase */
 
-	Py_TYPE(&ZstdCompressionReaderType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionReaderType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionReaderType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/compressionwriter.c

@@ -1,372 +1,372 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdCompresssionWriter__doc__,
 """A context manager used for writing compressed output to a writer.\n"
 );
 
 static void ZstdCompressionWriter_dealloc(ZstdCompressionWriter* self) {
 	Py_XDECREF(self->compressor);
 	Py_XDECREF(self->writer);
 
 	PyMem_Free(self->output.dst);
 	self->output.dst = NULL;
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) {
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->entered) {
 		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
 	self->entered = 1;
 
 	Py_INCREF(self);
 	return (PyObject*)self;
 }
 
 static PyObject* ZstdCompressionWriter_exit(ZstdCompressionWriter* self, PyObject* args) {
 	PyObject* exc_type;
 	PyObject* exc_value;
 	PyObject* exc_tb;
 
 	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
 		return NULL;
 	}
 
 	self->entered = 0;
 
 	if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) {
 		PyObject* result = PyObject_CallMethod((PyObject*)self, "close", NULL);
 
 		if (NULL == result) {
 			return NULL;
 		}
 	}
 
 	Py_RETURN_FALSE;
 }
 
 static PyObject* ZstdCompressionWriter_memory_size(ZstdCompressionWriter* self) {
 	return PyLong_FromSize_t(ZSTD_sizeof_CCtx(self->compressor->cctx));
 }
 
 static PyObject* ZstdCompressionWriter_write(ZstdCompressionWriter* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	PyObject* result = NULL;
 	Py_buffer source;
 	size_t zresult;
 	ZSTD_inBuffer input;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:write",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	self->output.pos = 0;
 
 	input.src = source.buf;
 	input.size = source.len;
 	input.pos = 0;
 
 	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output, &input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			goto finally;
 		}
 
 		/* Copy data from output buffer to writer. */
 		if (self->output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			res = PyObject_CallMethod(self->writer, "write", "y#",
 #else
 			res = PyObject_CallMethod(self->writer, "write", "s#",
 #endif
 				self->output.dst, self->output.pos);
 			Py_XDECREF(res);
 			totalWrite += self->output.pos;
 			self->bytesCompressed += self->output.pos;
 		}
 		self->output.pos = 0;
 	}
 
 	if (self->writeReturnRead) {
 		result = PyLong_FromSize_t(input.pos);
 	}
 	else {
 		result = PyLong_FromSsize_t(totalWrite);
 	}
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
 static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"flush_mode",
 		NULL
 	};
 
 	size_t zresult;
 	ZSTD_inBuffer input;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
 	unsigned flush_mode = 0;
 	ZSTD_EndDirective flush;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|I:flush",
 		kwlist, &flush_mode)) {
 		return NULL;
 	}
 
 	switch (flush_mode) {
 		case 0:
 			flush = ZSTD_e_flush;
 			break;
 		case 1:
 			flush = ZSTD_e_end;
 			break;
 		default:
 			PyErr_Format(PyExc_ValueError, "unknown flush_mode: %d", flush_mode);
 			return NULL;
 	}
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	self->output.pos = 0;
 
 	input.src = NULL;
 	input.size = 0;
 	input.pos = 0;
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output, &input, flush);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		/* Copy data from output buffer to writer. */
 		if (self->output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			res = PyObject_CallMethod(self->writer, "write", "y#",
 #else
 			res = PyObject_CallMethod(self->writer, "write", "s#",
 #endif
 				self->output.dst, self->output.pos);
 			Py_XDECREF(res);
 			totalWrite += self->output.pos;
 			self->bytesCompressed += self->output.pos;
 		}
 
 		self->output.pos = 0;
 
 		if (!zresult) {
 			break;
 		}
 	}
 
 	return PyLong_FromSsize_t(totalWrite);
 }
 
 static PyObject* ZstdCompressionWriter_close(ZstdCompressionWriter* self) {
 	PyObject* result;
 
 	if (self->closed) {
 		Py_RETURN_NONE;
 	}
 
 	result = PyObject_CallMethod((PyObject*)self, "flush", "I", 1);
 	self->closed = 1;
 
 	if (NULL == result) {
 	    return NULL;
 	}
 
     /* Call close on underlying stream as well. */
 	if (PyObject_HasAttrString(self->writer, "close")) {
 		return PyObject_CallMethod(self->writer, "close", NULL);
 	}
 
 	Py_RETURN_NONE;
 }
 
 static PyObject* ZstdCompressionWriter_fileno(ZstdCompressionWriter* self) {
 	if (PyObject_HasAttrString(self->writer, "fileno")) {
 		return PyObject_CallMethod(self->writer, "fileno", NULL);
 	}
 	else {
 		PyErr_SetString(PyExc_OSError, "fileno not available on underlying writer");
 		return NULL;
 	}
 }
 
 static PyObject* ZstdCompressionWriter_tell(ZstdCompressionWriter* self) {
 	return PyLong_FromUnsignedLongLong(self->bytesCompressed);
 }
 
 static PyObject* ZstdCompressionWriter_writelines(PyObject* self, PyObject* args) {
 	PyErr_SetNone(PyExc_NotImplementedError);
 	return NULL;
 }
 
 static PyObject* ZstdCompressionWriter_false(PyObject* self, PyObject* args) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* ZstdCompressionWriter_true(PyObject* self, PyObject* args) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* ZstdCompressionWriter_unsupported(PyObject* self, PyObject* args, PyObject* kwargs) {
 	PyObject* iomod;
 	PyObject* exc;
 
 	iomod = PyImport_ImportModule("io");
 	if (NULL == iomod) {
 		return NULL;
 	}
 
 	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
 	if (NULL == exc) {
 		Py_DECREF(iomod);
 		return NULL;
 	}
 
 	PyErr_SetNone(exc);
 	Py_DECREF(exc);
 	Py_DECREF(iomod);
 
 	return NULL;
 }
 
 static PyMethodDef ZstdCompressionWriter_methods[] = {
 	{ "__enter__", (PyCFunction)ZstdCompressionWriter_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context.") },
 	{ "__exit__", (PyCFunction)ZstdCompressionWriter_exit, METH_VARARGS,
 	PyDoc_STR("Exit a compression context.") },
 	{ "close", (PyCFunction)ZstdCompressionWriter_close, METH_NOARGS, NULL },
 	{ "fileno", (PyCFunction)ZstdCompressionWriter_fileno, METH_NOARGS, NULL },
 	{ "isatty", (PyCFunction)ZstdCompressionWriter_false, METH_NOARGS, NULL },
 	{ "readable", (PyCFunction)ZstdCompressionWriter_false, METH_NOARGS, NULL },
 	{ "readline", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readlines", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "seek", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "seekable", ZstdCompressionWriter_false, METH_NOARGS, NULL },
 	{ "truncate", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "writable", ZstdCompressionWriter_true, METH_NOARGS, NULL },
 	{ "writelines", ZstdCompressionWriter_writelines, METH_VARARGS, NULL },
 	{ "read", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readall", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readinto", (PyCFunction)ZstdCompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "memory_size", (PyCFunction)ZstdCompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size of the underlying compressor") },
 	{ "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
 	{ "flush", (PyCFunction)ZstdCompressionWriter_flush, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Flush data and finish a zstd frame") },
 	{ "tell", (PyCFunction)ZstdCompressionWriter_tell, METH_NOARGS,
 	PyDoc_STR("Returns current number of bytes compressed") },
 	{ NULL, NULL }
 };
 
 static PyMemberDef ZstdCompressionWriter_members[] = {
 	 { "closed", T_BOOL, offsetof(ZstdCompressionWriter, closed), READONLY, NULL },
 	 { NULL }
 };
 
 PyTypeObject ZstdCompressionWriterType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionWriter",  /* tp_name */
 	sizeof(ZstdCompressionWriter),  /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)ZstdCompressionWriter_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompresssionWriter__doc__,  /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdCompressionWriter_methods,  /* tp_methods */
 	ZstdCompressionWriter_members,  /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	0,                              /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void compressionwriter_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdCompressionWriterType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionWriterType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionWriterType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/compressobj.c

@@ -1,256 +1,256 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdCompressionObj__doc__,
 "Perform compression using a standard library compatible API.\n"
 );
 
 static void ZstdCompressionObj_dealloc(ZstdCompressionObj* self) {
 	PyMem_Free(self->output.dst);
 	self->output.dst = NULL;
 
 	Py_XDECREF(self->compressor);
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdCompressionObj_compress(ZstdCompressionObj* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	Py_buffer source;
 	ZSTD_inBuffer input;
 	size_t zresult;
 	PyObject* result = NULL;
 	Py_ssize_t resultSize = 0;
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot call compress() after compressor finished");
 		return NULL;
 	}
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:compress",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:compress",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	input.src = source.buf;
 	input.size = source.len;
 	input.pos = 0;
 
 	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 				&input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			Py_CLEAR(result);
 			goto finally;
 		}
 
 		if (self->output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
 
 				if (safe_pybytes_resize(&result, resultSize + self->output.pos)) {
 					Py_CLEAR(result);
 					goto finally;
 				}
 
 				memcpy(PyBytes_AS_STRING(result) + resultSize,
 					self->output.dst, self->output.pos);
 			}
 			else {
 				result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 				if (!result) {
 					goto finally;
 				}
 			}
 
 			self->output.pos = 0;
 		}
 	}
 
 	if (NULL == result) {
 		result = PyBytes_FromString("");
 	}
 
 finally:
 	PyBuffer_Release(&source);
 
 	return result;
 }
 
 static PyObject* ZstdCompressionObj_flush(ZstdCompressionObj* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"flush_mode",
 		NULL
 	};
 
 	int flushMode = compressorobj_flush_finish;
 	size_t zresult;
 	PyObject* result = NULL;
 	Py_ssize_t resultSize = 0;
 	ZSTD_inBuffer input;
 	ZSTD_EndDirective zFlushMode;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:flush", kwlist, &flushMode)) {
 		return NULL;
 	}
 
 	if (flushMode != compressorobj_flush_finish && flushMode != compressorobj_flush_block) {
 		PyErr_SetString(PyExc_ValueError, "flush mode not recognized");
 		return NULL;
 	}
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "compressor object already finished");
 		return NULL;
 	}
 
 	switch (flushMode) {
 		case compressorobj_flush_block:
 			zFlushMode = ZSTD_e_flush;
 			break;
 
 		case compressorobj_flush_finish:
 			zFlushMode = ZSTD_e_end;
 			self->finished = 1;
 			break;
 
 		default:
 			PyErr_SetString(ZstdError, "unhandled flush mode");
 			return NULL;
 	}
 
 	assert(self->output.pos == 0);
 
 	input.src = NULL;
 	input.size = 0;
 	input.pos = 0;
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&input, zFlushMode);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		if (self->output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
 
 				if (safe_pybytes_resize(&result, resultSize + self->output.pos)) {
 					Py_XDECREF(result);
 					return NULL;
 				}
 
 				memcpy(PyBytes_AS_STRING(result) + resultSize,
 					self->output.dst, self->output.pos);
 			}
 			else {
 				result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 				if (!result) {
 					return NULL;
 				}
 			}
 
 			self->output.pos = 0;
 		}
 
 		if (!zresult) {
 			break;
 		}
 	}
 
 	if (result) {
 		return result;
 	}
 	else {
 		return PyBytes_FromString("");
 	}
 }
 
 static PyMethodDef ZstdCompressionObj_methods[] = {
 	{ "compress", (PyCFunction)ZstdCompressionObj_compress, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("compress data") },
 	{ "flush", (PyCFunction)ZstdCompressionObj_flush, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("finish compression operation") },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdCompressionObjType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressionObj",      /* tp_name */
 	sizeof(ZstdCompressionObj),     /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)ZstdCompressionObj_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressionObj__doc__,      /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdCompressionObj_methods,     /* tp_methods */
 	0,                              /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	0,                              /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void compressobj_module_init(PyObject* module) {
-	Py_TYPE(&ZstdCompressionObjType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressionObjType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressionObjType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/compressor.c

@@ -1,1670 +1,1670 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 #include "pool.h"
 
 extern PyObject* ZstdError;
 
 int setup_cctx(ZstdCompressor* compressor) {
 	size_t zresult;
 
 	assert(compressor);
 	assert(compressor->cctx);
 	assert(compressor->params);
 
 	zresult = ZSTD_CCtx_setParametersUsingCCtxParams(compressor->cctx, compressor->params);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "could not set compression parameters: %s",
 			ZSTD_getErrorName(zresult));
 		return 1;
 	}
 
 	if (compressor->dict) {
 		if (compressor->dict->cdict) {
 			zresult = ZSTD_CCtx_refCDict(compressor->cctx, compressor->dict->cdict);
 		}
 		else {
 			zresult = ZSTD_CCtx_loadDictionary_advanced(compressor->cctx,
 				compressor->dict->dictData, compressor->dict->dictSize,
 				ZSTD_dlm_byRef, compressor->dict->dictType);
 		}
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "could not load compression dictionary: %s",
 				ZSTD_getErrorName(zresult));
 			return 1;
 		}
 	}
 
 	return 0;
 }
 
 static PyObject* frame_progression(ZSTD_CCtx* cctx) {
 	PyObject* result = NULL;
 	PyObject* value;
 	ZSTD_frameProgression progression;
 
 	result = PyTuple_New(3);
 	if (!result) {
 		return NULL;
 	}
 
 	progression = ZSTD_getFrameProgression(cctx);
 
 	value = PyLong_FromUnsignedLongLong(progression.ingested);
 	if (!value) {
 		Py_DECREF(result);
 		return NULL;
 	}
 
 	PyTuple_SET_ITEM(result, 0, value);
 
 	value = PyLong_FromUnsignedLongLong(progression.consumed);
 	if (!value) {
 		Py_DECREF(result);
 		return NULL;
 	}
 
 	PyTuple_SET_ITEM(result, 1, value);
 
 	value = PyLong_FromUnsignedLongLong(progression.produced);
 	if (!value) {
 		Py_DECREF(result);
 		return NULL;
 	}
 
 	PyTuple_SET_ITEM(result, 2, value);
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor__doc__,
 "ZstdCompressor(level=None, dict_data=None, compression_params=None)\n"
 "\n"
 "Create an object used to perform Zstandard compression.\n"
 "\n"
 "An instance can compress data various ways. Instances can be used multiple\n"
 "times. Each compression operation will use the compression parameters\n"
 "defined at construction time.\n"
 "\n"
 "Compression can be configured via the following names arguments:\n"
 "\n"
 "level\n"
 "   Integer compression level.\n"
 "dict_data\n"
 "   A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n"
 "compression_params\n"
 "   A ``CompressionParameters`` instance defining low-level compression"
 "   parameters. If defined, this will overwrite the ``level`` argument.\n"
 "write_checksum\n"
 "   If True, a 4 byte content checksum will be written with the compressed\n"
 "   data, allowing the decompressor to perform content verification.\n"
 "write_content_size\n"
 "   If True (the default), the decompressed content size will be included in\n"
 "   the header of the compressed data. This data will only be written if the\n"
 "   compressor knows the size of the input data.\n"
 "write_dict_id\n"
 "   Determines whether the dictionary ID will be written into the compressed\n"
 "   data. Defaults to True. Only adds content to the compressed data if\n"
 "   a dictionary is being used.\n"
 "threads\n"
 "   Number of threads to use to compress data concurrently. When set,\n"
 "   compression operations are performed on multiple threads. The default\n"
 "   value (0) disables multi-threaded compression. A value of ``-1`` means to\n"
 "   set the number of threads to the number of detected logical CPUs.\n"
 );
 
 static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"level",
 		"dict_data",
 		"compression_params",
 		"write_checksum",
 		"write_content_size",
 		"write_dict_id",
 		"threads",
 		NULL
 	};
 
 	int level = 3;
 	ZstdCompressionDict* dict = NULL;
 	ZstdCompressionParametersObject* params = NULL;
 	PyObject* writeChecksum = NULL;
 	PyObject* writeContentSize = NULL;
 	PyObject* writeDictID = NULL;
 	int threads = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOOi:ZstdCompressor",
 		kwlist,	&level, &ZstdCompressionDictType, &dict,
 		&ZstdCompressionParametersType, &params,
 		&writeChecksum, &writeContentSize, &writeDictID, &threads)) {
 		return -1;
 	}
 
 	if (level > ZSTD_maxCLevel()) {
 		PyErr_Format(PyExc_ValueError, "level must be less than %d",
 			ZSTD_maxCLevel() + 1);
 		return -1;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	/* We create a ZSTD_CCtx for reuse among multiple operations to reduce the
 	   overhead of each compression operation. */
 	self->cctx = ZSTD_createCCtx();
 	if (!self->cctx) {
 		PyErr_NoMemory();
 		return -1;
 	}
 
 	/* TODO stuff the original parameters away somewhere so we can reset later. This
 	   will allow us to do things like automatically adjust cparams based on input
 	   size (assuming zstd isn't doing that internally). */
 
 	self->params = ZSTD_createCCtxParams();
 	if (!self->params) {
 		PyErr_NoMemory();
 		return -1;
 	}
 
 	if (params && writeChecksum) {
 		PyErr_SetString(PyExc_ValueError,
 			"cannot define compression_params and write_checksum");
 		return -1;
 	}
 
 	if (params && writeContentSize) {
 		PyErr_SetString(PyExc_ValueError,
 			"cannot define compression_params and write_content_size");
 		return -1;
 	}
 
 	if (params && writeDictID) {
 		PyErr_SetString(PyExc_ValueError,
 			"cannot define compression_params and write_dict_id");
 		return -1;
 	}
 
 	if (params && threads) {
 		PyErr_SetString(PyExc_ValueError,
 			"cannot define compression_params and threads");
 		return -1;
 	}
 
 	if (params) {
 		if (set_parameters(self->params, params)) {
 			return -1;
 		}
 	}
 	else {
 		if (set_parameter(self->params, ZSTD_c_compressionLevel, level)) {
 			return -1;
 		}
 
 		if (set_parameter(self->params, ZSTD_c_contentSizeFlag,
 			writeContentSize ? PyObject_IsTrue(writeContentSize) : 1)) {
 			return -1;
 		}
 
 		if (set_parameter(self->params, ZSTD_c_checksumFlag,
 			writeChecksum ? PyObject_IsTrue(writeChecksum) : 0)) {
 			return -1;
 		}
 
 		if (set_parameter(self->params, ZSTD_c_dictIDFlag,
 			writeDictID ? PyObject_IsTrue(writeDictID) : 1)) {
 			return -1;
 		}
 
 		if (threads) {
 			if (set_parameter(self->params, ZSTD_c_nbWorkers, threads)) {
 				return -1;
 			}
 		}
 	}
 
 	if (dict) {
 		self->dict = dict;
 		Py_INCREF(dict);
 	}
 
     if (setup_cctx(self)) {
         return -1;
     }
 
 	return 0;
 }
 
 static void ZstdCompressor_dealloc(ZstdCompressor* self) {
 	if (self->cctx) {
 		ZSTD_freeCCtx(self->cctx);
 		self->cctx = NULL;
 	}
 
 	if (self->params) {
 		ZSTD_freeCCtxParams(self->params);
 		self->params = NULL;
 	}
 
 	Py_XDECREF(self->dict);
 	PyObject_Del(self);
 }
 
 PyDoc_STRVAR(ZstdCompressor_memory_size__doc__,
 "memory_size()\n"
 "\n"
 "Obtain the memory usage of this compressor, in bytes.\n"
 );
 
 static PyObject* ZstdCompressor_memory_size(ZstdCompressor* self) {
 	if (self->cctx) {
 		return PyLong_FromSize_t(ZSTD_sizeof_CCtx(self->cctx));
 	}
 	else {
 		PyErr_SetString(ZstdError, "no compressor context found; this should never happen");
 		return NULL;
 	}
 }
 
 PyDoc_STRVAR(ZstdCompressor_frame_progression__doc__,
 "frame_progression()\n"
 "\n"
 "Return information on how much work the compressor has done.\n"
 "\n"
 "Returns a 3-tuple of (ingested, consumed, produced).\n"
 );
 
 static PyObject* ZstdCompressor_frame_progression(ZstdCompressor* self) {
 	return frame_progression(self->cctx);
 }
 
 PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__,
 "copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n"
 "compress data between streams\n"
 "\n"
 "Data will be read from ``ifh``, compressed, and written to ``ofh``.\n"
 "``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n"
 "method.\n"
 "\n"
 "An optional ``size`` argument specifies the size of the source stream.\n"
 "If defined, compression parameters will be tuned based on the size.\n"
 "\n"
 "Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n"
 "of ``read()`` and ``write()`` operations, respectively. By default, they use\n"
 "the default compression stream input and output sizes, respectively.\n"
 );
 
 static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"ifh",
 		"ofh",
 		"size",
 		"read_size",
 		"write_size",
 		NULL
 	};
 
 	PyObject* source;
 	PyObject* dest;
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t inSize = ZSTD_CStreamInSize();
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_ssize_t totalRead = 0;
 	Py_ssize_t totalWrite = 0;
 	char* readBuffer;
 	Py_ssize_t readSize;
 	PyObject* readResult = NULL;
 	PyObject* res = NULL;
 	size_t zresult;
 	PyObject* writeResult;
 	PyObject* totalReadPy;
 	PyObject* totalWritePy;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|Kkk:copy_stream", kwlist,
 		&source, &dest, &sourceSize, &inSize, &outSize)) {
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(source, "read")) {
 		PyErr_SetString(PyExc_ValueError, "first argument must have a read() method");
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(dest, "write")) {
 		PyErr_SetString(PyExc_ValueError, "second argument must have a write() method");
 		return NULL;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	/* Prevent free on uninitialized memory in finally. */
 	output.dst = PyMem_Malloc(outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
 		res = NULL;
 		goto finally;
 	}
 	output.size = outSize;
 	output.pos = 0;
 
 	input.src = NULL;
 	input.size = 0;
 	input.pos = 0;
 
 	while (1) {
 		/* Try to read from source stream. */
 		readResult = PyObject_CallMethod(source, "read", "n", inSize);
 		if (!readResult) {
 			PyErr_SetString(ZstdError, "could not read() from source");
 			goto finally;
 		}
 
 		PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 
 		/* If no data was read, we're at EOF. */
 		if (0 == readSize) {
 			break;
 		}
 
 		totalRead += readSize;
 
 		/* Send data to compressor */
 		input.src = readBuffer;
 		input.size = readSize;
 		input.pos = 0;
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_compressStream2(self->cctx, &output, &input, ZSTD_e_continue);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
 				res = NULL;
 				PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 				goto finally;
 			}
 
 			if (output.pos) {
 #if PY_MAJOR_VERSION >= 3
 				writeResult = PyObject_CallMethod(dest, "write", "y#",
 #else
 				writeResult = PyObject_CallMethod(dest, "write", "s#",
 #endif
 					output.dst, output.pos);
 				Py_XDECREF(writeResult);
 				totalWrite += output.pos;
 				output.pos = 0;
 			}
 		}
 
 		Py_CLEAR(readResult);
 	}
 
 	/* We've finished reading. Now flush the compressor stream. */
 	assert(input.pos == input.size);
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->cctx, &output, &input, ZSTD_e_end);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
 			res = NULL;
 			goto finally;
 		}
 
 		if (output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			writeResult = PyObject_CallMethod(dest, "write", "y#",
 #else
 			writeResult = PyObject_CallMethod(dest, "write", "s#",
 #endif
 				output.dst, output.pos);
 			totalWrite += output.pos;
 			Py_XDECREF(writeResult);
 			output.pos = 0;
 		}
 
 		if (!zresult) {
 			break;
 		}
 	}
 
 	totalReadPy = PyLong_FromSsize_t(totalRead);
 	totalWritePy = PyLong_FromSsize_t(totalWrite);
 	res = PyTuple_Pack(2, totalReadPy, totalWritePy);
 	Py_DECREF(totalReadPy);
 	Py_DECREF(totalWritePy);
 
 finally:
 	if (output.dst) {
 		PyMem_Free(output.dst);
 	}
 
 	Py_XDECREF(readResult);
 
 	return res;
 }
 
 PyDoc_STRVAR(ZstdCompressor_stream_reader__doc__,
 "stream_reader(source, [size=0])\n"
 "\n"
 "Obtain an object that behaves like an I/O stream.\n"
 "\n"
 "The source object can be any object with a ``read(size)`` method\n"
 "or an object that conforms to the buffer protocol.\n"
 );
 
 static ZstdCompressionReader* ZstdCompressor_stream_reader(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"source",
 		"size",
 		"read_size",
 		NULL
 	};
 
 	PyObject* source;
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t readSize = ZSTD_CStreamInSize();
 	ZstdCompressionReader* result = NULL;
 	size_t zresult;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kk:stream_reader", kwlist,
 		&source, &sourceSize, &readSize)) {
 		return NULL;
 	}
 
 	result = (ZstdCompressionReader*)PyObject_CallObject((PyObject*)&ZstdCompressionReaderType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	if (PyObject_HasAttrString(source, "read")) {
 		result->reader = source;
 		Py_INCREF(source);
 		result->readSize = readSize;
 	}
 	else if (1 == PyObject_CheckBuffer(source)) {
 		if (0 != PyObject_GetBuffer(source, &result->buffer, PyBUF_CONTIG_RO)) {
 			goto except;
 		}
 
 		assert(result->buffer.len >= 0);
 
 		sourceSize = result->buffer.len;
 	}
 	else {
 		PyErr_SetString(PyExc_TypeError,
 			"must pass an object with a read() method or that conforms to the buffer protocol");
 		goto except;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source source: %s",
 			ZSTD_getErrorName(zresult));
 		goto except;
 	}
 
 	result->compressor = self;
 	Py_INCREF(self);
 
 	return result;
 
 except:
 	Py_CLEAR(result);
 
 	return NULL;
 }
 
 PyDoc_STRVAR(ZstdCompressor_compress__doc__,
 "compress(data)\n"
 "\n"
 "Compress data in a single operation.\n"
 "\n"
 "This is the simplest mechanism to perform compression: simply pass in a\n"
 "value and get a compressed value back. It is almost the most prone to abuse.\n"
 "The input and output values must fit in memory, so passing in very large\n"
 "values can result in excessive memory usage. For this reason, one of the\n"
 "streaming based APIs is preferred for larger values.\n"
 );
 
 static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	Py_buffer source;
 	size_t destSize;
 	PyObject* output = NULL;
 	size_t zresult;
 	ZSTD_outBuffer outBuffer;
 	ZSTD_inBuffer inBuffer;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|O:compress",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|O:compress",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	destSize = ZSTD_compressBound(source.len);
 	output = PyBytes_FromStringAndSize(NULL, destSize);
 	if (!output) {
 		goto finally;
 	}
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, source.len);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		Py_CLEAR(output);
 		goto finally;
 	}
 
 	inBuffer.src = source.buf;
 	inBuffer.size = source.len;
 	inBuffer.pos = 0;
 
 	outBuffer.dst = PyBytes_AsString(output);
 	outBuffer.size = destSize;
 	outBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
 	/* By avoiding ZSTD_compress(), we don't necessarily write out content
 		size. This means the argument to ZstdCompressor to control frame
 		parameters is honored. */
 	zresult = ZSTD_compressStream2(self->cctx, &outBuffer, &inBuffer, ZSTD_e_end);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult));
 		Py_CLEAR(output);
 		goto finally;
 	}
 	else if (zresult) {
 		PyErr_SetString(ZstdError, "unexpected partial frame flush");
 		Py_CLEAR(output);
 		goto finally;
 	}
 
-	Py_SIZE(output) = outBuffer.pos;
+	Py_SET_SIZE(output, outBuffer.pos);
 
 finally:
 	PyBuffer_Release(&source);
 	return output;
 }
 
 PyDoc_STRVAR(ZstdCompressionObj__doc__,
 "compressobj()\n"
 "\n"
 "Return an object exposing ``compress(data)`` and ``flush()`` methods.\n"
 "\n"
 "The returned object exposes an API similar to ``zlib.compressobj`` and\n"
 "``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n"
 "without changing how compression is performed.\n"
 );
 
 static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	unsigned long long inSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZstdCompressionObj* result = NULL;
 	size_t zresult;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|K:compressobj", kwlist, &inSize)) {
 		return NULL;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, inSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	result = (ZstdCompressionObj*)PyObject_CallObject((PyObject*)&ZstdCompressionObjType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	result->output.dst = PyMem_Malloc(outSize);
 	if (!result->output.dst) {
 		PyErr_NoMemory();
 		Py_DECREF(result);
 		return NULL;
 	}
 	result->output.size = outSize;
 	result->compressor = self;
 	Py_INCREF(result->compressor);
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor_read_to_iter__doc__,
 "read_to_iter(reader, [size=0, read_size=default, write_size=default])\n"
 "Read uncompressed data from a reader and return an iterator\n"
 "\n"
 "Returns an iterator of compressed data produced from reading from ``reader``.\n"
 "\n"
 "Uncompressed data will be obtained from ``reader`` by calling the\n"
 "``read(size)`` method of it. The source data will be streamed into a\n"
 "compressor. As compressed data is available, it will be exposed to the\n"
 "iterator.\n"
 "\n"
 "Data is read from the source in chunks of ``read_size``. Compressed chunks\n"
 "are at most ``write_size`` bytes. Both values default to the zstd input and\n"
 "and output defaults, respectively.\n"
 "\n"
 "The caller is partially in control of how fast data is fed into the\n"
 "compressor by how it consumes the returned iterator. The compressor will\n"
 "not consume from the reader unless the caller consumes from the iterator.\n"
 );
 
 static ZstdCompressorIterator* ZstdCompressor_read_to_iter(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"reader",
 		"size",
 		"read_size",
 		"write_size",
 		NULL
 	};
 
 	PyObject* reader;
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t inSize = ZSTD_CStreamInSize();
 	size_t outSize = ZSTD_CStreamOutSize();
 	ZstdCompressorIterator* result;
 	size_t zresult;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Kkk:read_to_iter", kwlist,
 		&reader, &sourceSize, &inSize, &outSize)) {
 		return NULL;
 	}
 
 	result = (ZstdCompressorIterator*)PyObject_CallObject((PyObject*)&ZstdCompressorIteratorType, NULL);
 	if (!result) {
 		return NULL;
 	}
 	if (PyObject_HasAttrString(reader, "read")) {
 		result->reader = reader;
 		Py_INCREF(result->reader);
 	}
 	else if (1 == PyObject_CheckBuffer(reader)) {
 		if (0 != PyObject_GetBuffer(reader, &result->buffer, PyBUF_CONTIG_RO)) {
 			goto except;
 		}
 
 		sourceSize = result->buffer.len;
 	}
 	else {
 		PyErr_SetString(PyExc_ValueError,
 			"must pass an object with a read() method or conforms to buffer protocol");
 		goto except;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	result->compressor = self;
 	Py_INCREF(result->compressor);
 
 	result->inSize = inSize;
 	result->outSize = outSize;
 
 	result->output.dst = PyMem_Malloc(outSize);
 	if (!result->output.dst) {
 		PyErr_NoMemory();
 		goto except;
 	}
 	result->output.size = outSize;
 
 	goto finally;
 
 except:
 	Py_CLEAR(result);
 
 finally:
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor_stream_writer___doc__,
 "Create a context manager to write compressed data to an object.\n"
 "\n"
 "The passed object must have a ``write()`` method.\n"
 "\n"
 "The caller feeds input data to the object by calling ``compress(data)``.\n"
 "Compressed data is written to the argument given to this function.\n"
 "\n"
 "The function takes an optional ``size`` argument indicating the total size\n"
 "of the eventual input. If specified, the size will influence compression\n"
 "parameter tuning and could result in the size being written into the\n"
 "header of the compressed data.\n"
 "\n"
 "An optional ``write_size`` argument is also accepted. It defines the maximum\n"
 "byte size of chunks fed to ``write()``. By default, it uses the zstd default\n"
 "for a compressor output stream.\n"
 );
 
 static ZstdCompressionWriter* ZstdCompressor_stream_writer(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"writer",
 		"size",
 		"write_size",
 		"write_return_read",
 		NULL
 	};
 
 	PyObject* writer;
 	ZstdCompressionWriter* result;
 	size_t zresult;
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t outSize = ZSTD_CStreamOutSize();
 	PyObject* writeReturnRead = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|KkO:stream_writer", kwlist,
 		&writer, &sourceSize, &outSize, &writeReturnRead)) {
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(writer, "write")) {
 		PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method");
 		return NULL;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	result = (ZstdCompressionWriter*)PyObject_CallObject((PyObject*)&ZstdCompressionWriterType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	result->output.dst = PyMem_Malloc(outSize);
 	if (!result->output.dst) {
 		Py_DECREF(result);
 		return (ZstdCompressionWriter*)PyErr_NoMemory();
 	}
 
 	result->output.pos = 0;
 	result->output.size = outSize;
 
 	result->compressor = self;
 	Py_INCREF(result->compressor);
 
 	result->writer = writer;
 	Py_INCREF(result->writer);
 
 	result->outSize = outSize;
 	result->bytesCompressed = 0;
 	result->writeReturnRead = writeReturnRead ? PyObject_IsTrue(writeReturnRead) : 0;
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor_chunker__doc__,
 "Create an object for iterative compressing to same-sized chunks.\n"
 );
 
 static ZstdCompressionChunker* ZstdCompressor_chunker(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		"chunk_size",
 		NULL
 	};
 
 	unsigned long long sourceSize = ZSTD_CONTENTSIZE_UNKNOWN;
 	size_t chunkSize = ZSTD_CStreamOutSize();
 	ZstdCompressionChunker* chunker;
 	size_t zresult;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|Kk:chunker", kwlist,
 		&sourceSize, &chunkSize)) {
 		return NULL;
 	}
 
 	ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
 	zresult = ZSTD_CCtx_setPledgedSrcSize(self->cctx, sourceSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "error setting source size: %s",
 			ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	chunker = (ZstdCompressionChunker*)PyObject_CallObject((PyObject*)&ZstdCompressionChunkerType, NULL);
 	if (!chunker) {
 		return NULL;
 	}
 
 	chunker->output.dst = PyMem_Malloc(chunkSize);
 	if (!chunker->output.dst) {
 		PyErr_NoMemory();
 		Py_DECREF(chunker);
 		return NULL;
 	}
 	chunker->output.size = chunkSize;
 	chunker->output.pos = 0;
 
 	chunker->compressor = self;
 	Py_INCREF(chunker->compressor);
 
 	chunker->chunkSize = chunkSize;
 
 	return chunker;
 }
 
 typedef struct {
 	void* sourceData;
 	size_t sourceSize;
 } DataSource;
 
 typedef struct {
 	DataSource* sources;
 	Py_ssize_t sourcesSize;
 	unsigned long long totalSourceSize;
 } DataSources;
 
 typedef struct {
 	void* dest;
 	Py_ssize_t destSize;
 	BufferSegment* segments;
 	Py_ssize_t segmentsSize;
 } DestBuffer;
 
 typedef enum {
 	WorkerError_none = 0,
 	WorkerError_zstd = 1,
 	WorkerError_no_memory = 2,
 	WorkerError_nospace = 3,
 } WorkerError;
 
 /**
  * Holds state for an individual worker performing multi_compress_to_buffer work.
  */
 typedef struct {
 	/* Used for compression. */
 	ZSTD_CCtx* cctx;
 
 	/* What to compress. */
 	DataSource* sources;
 	Py_ssize_t sourcesSize;
 	Py_ssize_t startOffset;
 	Py_ssize_t endOffset;
 	unsigned long long totalSourceSize;
 
 	/* Result storage. */
 	DestBuffer* destBuffers;
 	Py_ssize_t destCount;
 
 	/* Error tracking. */
 	WorkerError error;
 	size_t zresult;
 	Py_ssize_t errorOffset;
 } WorkerState;
 
 static void compress_worker(WorkerState* state) {
 	Py_ssize_t inputOffset = state->startOffset;
 	Py_ssize_t remainingItems = state->endOffset - state->startOffset + 1;
 	Py_ssize_t currentBufferStartOffset = state->startOffset;
 	size_t zresult;
 	void* newDest;
 	size_t allocationSize;
 	size_t boundSize;
 	Py_ssize_t destOffset = 0;
 	DataSource* sources = state->sources;
 	DestBuffer* destBuffer;
 
 	assert(!state->destBuffers);
 	assert(0 == state->destCount);
 
 	/*
 	 * The total size of the compressed data is unknown until we actually
 	 * compress data. That means we can't pre-allocate the exact size we need.
 	 *
 	 * There is a cost to every allocation and reallocation. So, it is in our
 	 * interest to minimize the number of allocations.
 	 *
 	 * There is also a cost to too few allocations. If allocations are too
 	 * large they may fail. If buffers are shared and all inputs become
 	 * irrelevant at different lifetimes, then a reference to one segment
 	 * in the buffer will keep the entire buffer alive. This leads to excessive
 	 * memory usage.
 	 *
 	 * Our current strategy is to assume a compression ratio of 16:1 and
 	 * allocate buffers of that size, rounded up to the nearest power of 2
 	 * (because computers like round numbers). That ratio is greater than what
 	 * most inputs achieve. This is by design: we don't want to over-allocate.
 	 * But we don't want to under-allocate and lead to too many buffers either.
 	 */
 
 	state->destCount = 1;
 
 	state->destBuffers = calloc(1, sizeof(DestBuffer));
 	if (NULL == state->destBuffers) {
 		state->error = WorkerError_no_memory;
 		return;
 	}
 
 	destBuffer = &state->destBuffers[state->destCount - 1];
 
 	/*
 	 * Rather than track bounds and grow the segments buffer, allocate space
 	 * to hold remaining items then truncate when we're done with it.
 	 */
 	destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
 	if (NULL == destBuffer->segments) {
 		state->error = WorkerError_no_memory;
 		return;
 	}
 
 	destBuffer->segmentsSize = remainingItems;
 
 	assert(state->totalSourceSize <= SIZE_MAX);
 	allocationSize = roundpow2((size_t)state->totalSourceSize >> 4);
 
 	/* If the maximum size of the output is larger than that, round up. */
 	boundSize = ZSTD_compressBound(sources[inputOffset].sourceSize);
 
 	if (boundSize > allocationSize) {
 		allocationSize = roundpow2(boundSize);
 	}
 
 	destBuffer->dest = malloc(allocationSize);
 	if (NULL == destBuffer->dest) {
 		state->error = WorkerError_no_memory;
 		return;
 	}
 
 	destBuffer->destSize = allocationSize;
 
 	for (inputOffset = state->startOffset; inputOffset <= state->endOffset; inputOffset++) {
 		void* source = sources[inputOffset].sourceData;
 		size_t sourceSize = sources[inputOffset].sourceSize;
 		size_t destAvailable;
 		void* dest;
 		ZSTD_outBuffer opOutBuffer;
 		ZSTD_inBuffer opInBuffer;
 
 		destAvailable = destBuffer->destSize - destOffset;
 		boundSize = ZSTD_compressBound(sourceSize);
 
 		/*
 		 * Not enough space in current buffer to hold largest compressed output.
 		 * So allocate and switch to a new output buffer.
 		 */
 		if (boundSize > destAvailable) {
 			/*
 			 * The downsizing of the existing buffer is optional. It should be cheap
 			 * (unlike growing). So we just do it.
 			 */
 			if (destAvailable) {
 				newDest = realloc(destBuffer->dest, destOffset);
 				if (NULL == newDest) {
 					state->error = WorkerError_no_memory;
 					return;
 				}
 
 				destBuffer->dest = newDest;
 				destBuffer->destSize = destOffset;
 			}
 
 			/* Truncate segments buffer. */
 			newDest = realloc(destBuffer->segments,
 				(inputOffset - currentBufferStartOffset + 1) * sizeof(BufferSegment));
 			if (NULL == newDest) {
 				state->error = WorkerError_no_memory;
 				return;
 			}
 
 			destBuffer->segments = newDest;
 			destBuffer->segmentsSize = inputOffset - currentBufferStartOffset;
 
 			/* Grow space for new struct. */
 			/* TODO consider over-allocating so we don't do this every time. */
 			newDest = realloc(state->destBuffers, (state->destCount + 1) * sizeof(DestBuffer));
 			if (NULL == newDest) {
 				state->error = WorkerError_no_memory;
 				return;
 			}
 
 			state->destBuffers = newDest;
 			state->destCount++;
 
 			destBuffer = &state->destBuffers[state->destCount - 1];
 
 			/* Don't take any chances with non-NULL pointers. */
 			memset(destBuffer, 0, sizeof(DestBuffer));
 
 			/**
 			 * We could dynamically update allocation size based on work done so far.
 			 * For now, keep is simple.
 			 */
 			assert(state->totalSourceSize <= SIZE_MAX);
 			allocationSize = roundpow2((size_t)state->totalSourceSize >> 4);
 
 			if (boundSize > allocationSize) {
 				allocationSize = roundpow2(boundSize);
 			}
 
 			destBuffer->dest = malloc(allocationSize);
 			if (NULL == destBuffer->dest) {
 				state->error = WorkerError_no_memory;
 				return;
 			}
 
 			destBuffer->destSize = allocationSize;
 			destAvailable = allocationSize;
 			destOffset = 0;
 
 			destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
 			if (NULL == destBuffer->segments) {
 				state->error = WorkerError_no_memory;
 				return;
 			}
 
 			destBuffer->segmentsSize = remainingItems;
 			currentBufferStartOffset = inputOffset;
 		}
 
 		dest = (char*)destBuffer->dest + destOffset;
 
 		opInBuffer.src = source;
 		opInBuffer.size = sourceSize;
 		opInBuffer.pos = 0;
 
 		opOutBuffer.dst = dest;
 		opOutBuffer.size = destAvailable;
 		opOutBuffer.pos = 0;
 
 		zresult = ZSTD_CCtx_setPledgedSrcSize(state->cctx, sourceSize);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;
 			state->errorOffset = inputOffset;
 			break;
 		}
 
 		zresult = ZSTD_compressStream2(state->cctx, &opOutBuffer, &opInBuffer, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;
 			state->errorOffset = inputOffset;
 			break;
 		}
 		else if (zresult) {
 			state->error = WorkerError_nospace;
 			state->errorOffset = inputOffset;
 			break;
 		}
 
 		destBuffer->segments[inputOffset - currentBufferStartOffset].offset = destOffset;
 		destBuffer->segments[inputOffset - currentBufferStartOffset].length = opOutBuffer.pos;
 
 		destOffset += opOutBuffer.pos;
 		remainingItems--;
 	}
 
 	if (destBuffer->destSize > destOffset) {
 		newDest = realloc(destBuffer->dest, destOffset);
 		if (NULL == newDest) {
 			state->error = WorkerError_no_memory;
 			return;
 		}
 
 		destBuffer->dest = newDest;
 		destBuffer->destSize = destOffset;
 	}
 }
 
 ZstdBufferWithSegmentsCollection* compress_from_datasources(ZstdCompressor* compressor,
 	DataSources* sources, Py_ssize_t threadCount) {
 	unsigned long long bytesPerWorker;
 	POOL_ctx* pool = NULL;
 	WorkerState* workerStates = NULL;
 	Py_ssize_t i;
 	unsigned long long workerBytes = 0;
 	Py_ssize_t workerStartOffset = 0;
 	Py_ssize_t currentThread = 0;
 	int errored = 0;
 	Py_ssize_t segmentsCount = 0;
 	Py_ssize_t segmentIndex;
 	PyObject* segmentsArg = NULL;
 	ZstdBufferWithSegments* buffer;
 	ZstdBufferWithSegmentsCollection* result = NULL;
 
 	assert(sources->sourcesSize > 0);
 	assert(sources->totalSourceSize > 0);
 	assert(threadCount >= 1);
 
 	/* More threads than inputs makes no sense. */
 	threadCount = sources->sourcesSize < threadCount ? sources->sourcesSize
 													 : threadCount;
 
 	/* TODO lower thread count when input size is too small and threads would add
 	overhead. */
 
 	workerStates = PyMem_Malloc(threadCount * sizeof(WorkerState));
 	if (NULL == workerStates) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	memset(workerStates, 0, threadCount * sizeof(WorkerState));
 
 	if (threadCount > 1) {
 		pool = POOL_create(threadCount, 1);
 		if (NULL == pool) {
 			PyErr_SetString(ZstdError, "could not initialize zstd thread pool");
 			goto finally;
 		}
 	}
 
 	bytesPerWorker = sources->totalSourceSize / threadCount;
 
 	for (i = 0; i < threadCount; i++) {
 		size_t zresult;
 
 		workerStates[i].cctx = ZSTD_createCCtx();
 		if (!workerStates[i].cctx) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		zresult = ZSTD_CCtx_setParametersUsingCCtxParams(workerStates[i].cctx,
 			compressor->params);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "could not set compression parameters: %s",
 				ZSTD_getErrorName(zresult));
 			goto finally;
 		}
 
 		if (compressor->dict) {
 			if (compressor->dict->cdict) {
 				zresult = ZSTD_CCtx_refCDict(workerStates[i].cctx, compressor->dict->cdict);
 			}
 			else {
 				zresult = ZSTD_CCtx_loadDictionary_advanced(
 					workerStates[i].cctx,
 					compressor->dict->dictData,
 					compressor->dict->dictSize,
 					ZSTD_dlm_byRef,
 					compressor->dict->dictType);
 			}
 
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not load compression dictionary: %s",
 					ZSTD_getErrorName(zresult));
 				goto finally;
 			}
 
 		}
 
 		workerStates[i].sources = sources->sources;
 		workerStates[i].sourcesSize = sources->sourcesSize;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	for (i = 0; i < sources->sourcesSize; i++) {
 		workerBytes += sources->sources[i].sourceSize;
 
 		/*
 		 * The last worker/thread needs to handle all remaining work. Don't
 		 * trigger it prematurely. Defer to the block outside of the loop
 		 * to run the last worker/thread. But do still process this loop
 		 * so workerBytes is correct.
 		 */
 		if (currentThread == threadCount - 1) {
 			continue;
 		}
 
 		if (workerBytes >= bytesPerWorker) {
 			assert(currentThread < threadCount);
 			workerStates[currentThread].totalSourceSize = workerBytes;
 			workerStates[currentThread].startOffset = workerStartOffset;
 			workerStates[currentThread].endOffset = i;
 
 			if (threadCount > 1) {
 				POOL_add(pool, (POOL_function)compress_worker, &workerStates[currentThread]);
 			}
 			else {
 				compress_worker(&workerStates[currentThread]);
 			}
 
 			currentThread++;
 			workerStartOffset = i + 1;
 			workerBytes = 0;
 		}
 	}
 
 	if (workerBytes) {
 		assert(currentThread < threadCount);
 		workerStates[currentThread].totalSourceSize = workerBytes;
 		workerStates[currentThread].startOffset = workerStartOffset;
 		workerStates[currentThread].endOffset = sources->sourcesSize - 1;
 
 		if (threadCount > 1) {
 			POOL_add(pool, (POOL_function)compress_worker, &workerStates[currentThread]);
 		}
 		else {
 			compress_worker(&workerStates[currentThread]);
 		}
 	}
 
 	if (threadCount > 1) {
 		POOL_free(pool);
 		pool = NULL;
 	}
 
 	Py_END_ALLOW_THREADS
 
 	for (i = 0; i < threadCount; i++) {
 		switch (workerStates[i].error) {
 		case WorkerError_no_memory:
 			PyErr_NoMemory();
 			errored = 1;
 			break;
 
 		case WorkerError_zstd:
 			PyErr_Format(ZstdError, "error compressing item %zd: %s",
 				workerStates[i].errorOffset, ZSTD_getErrorName(workerStates[i].zresult));
 			errored = 1;
 			break;
 
 		case WorkerError_nospace:
 			PyErr_Format(ZstdError, "error compressing item %zd: not enough space in output",
 				workerStates[i].errorOffset);
 			errored = 1;
 			break;
 
 		default:
 			;
 		}
 
 		if (errored) {
 			break;
 		}
 
 	}
 
 	if (errored) {
 		goto finally;
 	}
 
 	segmentsCount = 0;
 	for (i = 0; i < threadCount; i++) {
 		WorkerState* state = &workerStates[i];
 		segmentsCount += state->destCount;
 	}
 
 	segmentsArg = PyTuple_New(segmentsCount);
 	if (NULL == segmentsArg) {
 		goto finally;
 	}
 
 	segmentIndex = 0;
 
 	for (i = 0; i < threadCount; i++) {
 		Py_ssize_t j;
 		WorkerState* state = &workerStates[i];
 
 		for (j = 0; j < state->destCount; j++) {
 			DestBuffer* destBuffer = &state->destBuffers[j];
 			buffer = BufferWithSegments_FromMemory(destBuffer->dest, destBuffer->destSize,
 				destBuffer->segments, destBuffer->segmentsSize);
 
 			if (NULL == buffer) {
 				goto finally;
 			}
 
 			/* Tell instance to use free() instsead of PyMem_Free(). */
 			buffer->useFree = 1;
 
 			/*
 			 * BufferWithSegments_FromMemory takes ownership of the backing memory.
 			 * Unset it here so it doesn't get freed below.
 			 */
 			destBuffer->dest = NULL;
 			destBuffer->segments = NULL;
 
 			PyTuple_SET_ITEM(segmentsArg, segmentIndex++, (PyObject*)buffer);
 		}
 	}
 
 	result = (ZstdBufferWithSegmentsCollection*)PyObject_CallObject(
 		(PyObject*)&ZstdBufferWithSegmentsCollectionType, segmentsArg);
 
 finally:
 	Py_CLEAR(segmentsArg);
 
 	if (pool) {
 		POOL_free(pool);
 	}
 
 	if (workerStates) {
 		Py_ssize_t j;
 
 		for (i = 0; i < threadCount; i++) {
 			WorkerState state = workerStates[i];
 
 			if (state.cctx) {
 				ZSTD_freeCCtx(state.cctx);
 			}
 
 			/* malloc() is used in worker thread. */
 
 			for (j = 0; j < state.destCount; j++) {
 				if (state.destBuffers) {
 					free(state.destBuffers[j].dest);
 					free(state.destBuffers[j].segments);
 				}
 			}
 
 
 			free(state.destBuffers);
 		}
 
 		PyMem_Free(workerStates);
 	}
 
 	return result;
 }
 
 PyDoc_STRVAR(ZstdCompressor_multi_compress_to_buffer__doc__,
 "Compress multiple pieces of data as a single operation\n"
 "\n"
 "Receives a ``BufferWithSegmentsCollection``, a ``BufferWithSegments``, or\n"
 "a list of bytes like objects holding data to compress.\n"
 "\n"
 "Returns a ``BufferWithSegmentsCollection`` holding compressed data.\n"
 "\n"
 "This function is optimized to perform multiple compression operations as\n"
 "as possible with as little overhead as possbile.\n"
 );
 
 static ZstdBufferWithSegmentsCollection* ZstdCompressor_multi_compress_to_buffer(ZstdCompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		"threads",
 		NULL
 	};
 
 	PyObject* data;
 	int threads = 0;
 	Py_buffer* dataBuffers = NULL;
 	DataSources sources;
 	Py_ssize_t i;
 	Py_ssize_t sourceCount = 0;
 	ZstdBufferWithSegmentsCollection* result = NULL;
 
 	memset(&sources, 0, sizeof(sources));
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:multi_compress_to_buffer", kwlist,
 		&data, &threads)) {
 		return NULL;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	if (threads < 2) {
 		threads = 1;
 	}
 
 	if (PyObject_TypeCheck(data, &ZstdBufferWithSegmentsType)) {
 		ZstdBufferWithSegments* buffer = (ZstdBufferWithSegments*)data;
 
 		sources.sources = PyMem_Malloc(buffer->segmentCount * sizeof(DataSource));
 		if (NULL == sources.sources) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		for (i = 0; i < buffer->segmentCount; i++) {
 			if (buffer->segments[i].length > SIZE_MAX) {
 				PyErr_Format(PyExc_ValueError,
 					"buffer segment %zd is too large for this platform", i);
 				goto finally;
 			}
 
 			sources.sources[i].sourceData = (char*)buffer->data + buffer->segments[i].offset;
 			sources.sources[i].sourceSize = (size_t)buffer->segments[i].length;
 			sources.totalSourceSize += buffer->segments[i].length;
 		}
 
 		sources.sourcesSize = buffer->segmentCount;
 	}
 	else if (PyObject_TypeCheck(data, &ZstdBufferWithSegmentsCollectionType)) {
 		Py_ssize_t j;
 		Py_ssize_t offset = 0;
 		ZstdBufferWithSegments* buffer;
 		ZstdBufferWithSegmentsCollection* collection = (ZstdBufferWithSegmentsCollection*)data;
 
 		sourceCount = BufferWithSegmentsCollection_length(collection);
 
 		sources.sources = PyMem_Malloc(sourceCount * sizeof(DataSource));
 		if (NULL == sources.sources) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		for (i = 0; i < collection->bufferCount; i++) {
 			buffer = collection->buffers[i];
 
 			for (j = 0; j < buffer->segmentCount; j++) {
 				if (buffer->segments[j].length > SIZE_MAX) {
 					PyErr_Format(PyExc_ValueError,
 						"buffer segment %zd in buffer %zd is too large for this platform",
 						j, i);
 					goto finally;
 				}
 
 				sources.sources[offset].sourceData = (char*)buffer->data + buffer->segments[j].offset;
 				sources.sources[offset].sourceSize = (size_t)buffer->segments[j].length;
 				sources.totalSourceSize += buffer->segments[j].length;
 
 				offset++;
 			}
 		}
 
 		sources.sourcesSize = sourceCount;
 	}
 	else if (PyList_Check(data)) {
 		sourceCount = PyList_GET_SIZE(data);
 
 		sources.sources = PyMem_Malloc(sourceCount * sizeof(DataSource));
 		if (NULL == sources.sources) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		dataBuffers = PyMem_Malloc(sourceCount * sizeof(Py_buffer));
 		if (NULL == dataBuffers) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		memset(dataBuffers, 0, sourceCount * sizeof(Py_buffer));
 
 		for (i = 0; i < sourceCount; i++) {
 			if (0 != PyObject_GetBuffer(PyList_GET_ITEM(data, i),
 				&dataBuffers[i], PyBUF_CONTIG_RO)) {
 				PyErr_Clear();
 				PyErr_Format(PyExc_TypeError, "item %zd not a bytes like object", i);
 				goto finally;
 			}
 
 			sources.sources[i].sourceData = dataBuffers[i].buf;
 			sources.sources[i].sourceSize = dataBuffers[i].len;
 			sources.totalSourceSize += dataBuffers[i].len;
 		}
 
 		sources.sourcesSize = sourceCount;
 	}
 	else {
 		PyErr_SetString(PyExc_TypeError, "argument must be list of BufferWithSegments");
 		goto finally;
 	}
 
 	if (0 == sources.sourcesSize) {
 		PyErr_SetString(PyExc_ValueError, "no source elements found");
 		goto finally;
 	}
 
 	if (0 == sources.totalSourceSize) {
 		PyErr_SetString(PyExc_ValueError, "source elements are empty");
 		goto finally;
 	}
 
 	if (sources.totalSourceSize > SIZE_MAX) {
 		PyErr_SetString(PyExc_ValueError, "sources are too large for this platform");
 		goto finally;
 	}
 
 	result = compress_from_datasources(self, &sources, threads);
 
 finally:
 	PyMem_Free(sources.sources);
 
 	if (dataBuffers) {
 		for (i = 0; i < sourceCount; i++) {
 			PyBuffer_Release(&dataBuffers[i]);
 		}
 
 		PyMem_Free(dataBuffers);
 	}
 
 	return result;
 }
 
 static PyMethodDef ZstdCompressor_methods[] = {
 	{ "chunker", (PyCFunction)ZstdCompressor_chunker,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_chunker__doc__ },
 	{ "compress", (PyCFunction)ZstdCompressor_compress,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_compress__doc__ },
 	{ "compressobj", (PyCFunction)ZstdCompressor_compressobj,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ },
 	{ "copy_stream", (PyCFunction)ZstdCompressor_copy_stream,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ },
 	{ "stream_reader", (PyCFunction)ZstdCompressor_stream_reader,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_reader__doc__ },
 	{ "stream_writer", (PyCFunction)ZstdCompressor_stream_writer,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_writer___doc__ },
 	{ "read_to_iter", (PyCFunction)ZstdCompressor_read_to_iter,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_to_iter__doc__ },
 	/* TODO Remove deprecated API */
 	{ "read_from", (PyCFunction)ZstdCompressor_read_to_iter,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_to_iter__doc__ },
 	/* TODO remove deprecated API */
 	{ "write_to", (PyCFunction)ZstdCompressor_stream_writer,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_stream_writer___doc__ },
 	{ "multi_compress_to_buffer", (PyCFunction)ZstdCompressor_multi_compress_to_buffer,
 	METH_VARARGS | METH_KEYWORDS, ZstdCompressor_multi_compress_to_buffer__doc__ },
 	{ "memory_size", (PyCFunction)ZstdCompressor_memory_size,
 	METH_NOARGS, ZstdCompressor_memory_size__doc__ },
 	{ "frame_progression", (PyCFunction)ZstdCompressor_frame_progression,
 	METH_NOARGS, ZstdCompressor_frame_progression__doc__ },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdCompressorType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressor",         /* tp_name */
 	sizeof(ZstdCompressor),        /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)ZstdCompressor_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressor__doc__,          /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdCompressor_methods,         /* tp_methods */
 	0,                              /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	(initproc)ZstdCompressor_init,  /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void compressor_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdCompressorType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressorType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressorType) < 0) {
 		return;
 	}
 
 	Py_INCREF((PyObject*)&ZstdCompressorType);
 	PyModule_AddObject(mod, "ZstdCompressor",
 		(PyObject*)&ZstdCompressorType);
 }

contrib/python-zstandard/c-ext/compressoriterator.c

@@ -1,235 +1,235 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 #define min(a, b) (((a) < (b)) ? (a) : (b))
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdCompressorIterator__doc__,
 "Represents an iterator of compressed data.\n"
 );
 
 static void ZstdCompressorIterator_dealloc(ZstdCompressorIterator* self) {
 	Py_XDECREF(self->readResult);
 	Py_XDECREF(self->compressor);
 	Py_XDECREF(self->reader);
 
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	if (self->output.dst) {
 		PyMem_Free(self->output.dst);
 		self->output.dst = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdCompressorIterator_iter(PyObject* self) {
 	Py_INCREF(self);
 	return self;
 }
 
 static PyObject* ZstdCompressorIterator_iternext(ZstdCompressorIterator* self) {
 	size_t zresult;
 	PyObject* readResult = NULL;
 	PyObject* chunk;
 	char* readBuffer;
 	Py_ssize_t readSize = 0;
 	Py_ssize_t bufferRemaining;
 
 	if (self->finishedOutput) {
 		PyErr_SetString(PyExc_StopIteration, "output flushed");
 		return NULL;
 	}
 
 feedcompressor:
 
 	/* If we have data left in the input, consume it. */
 	if (self->input.pos < self->input.size) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&self->input, ZSTD_e_continue);
 		Py_END_ALLOW_THREADS
 
 		/* Release the Python object holding the input buffer. */
 		if (self->input.pos == self->input.size) {
 			self->input.src = NULL;
 			self->input.pos = 0;
 			self->input.size = 0;
 			Py_DECREF(self->readResult);
 			self->readResult = NULL;
 		}
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		/* If it produced output data, emit it. */
 		if (self->output.pos) {
 			chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 			self->output.pos = 0;
 			return chunk;
 		}
 	}
 
 	/* We should never have output data sitting around after a previous call. */
 	assert(self->output.pos == 0);
 
 	/* The code above should have either emitted a chunk and returned or consumed
 	the entire input buffer. So the state of the input buffer is not
 	relevant. */
 	if (!self->finishedInput) {
 		if (self->reader) {
 			readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize);
 			if (!readResult) {
 				PyErr_SetString(ZstdError, "could not read() from source");
 				return NULL;
 			}
 
 			PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 		}
 		else {
 			assert(self->buffer.buf);
 
 			/* Only support contiguous C arrays. */
 			assert(self->buffer.strides == NULL && self->buffer.suboffsets == NULL);
 			assert(self->buffer.itemsize == 1);
 
 			readBuffer = (char*)self->buffer.buf + self->bufferOffset;
 			bufferRemaining = self->buffer.len - self->bufferOffset;
 			readSize = min(bufferRemaining, (Py_ssize_t)self->inSize);
 			self->bufferOffset += readSize;
 		}
 
 		if (0 == readSize) {
 			Py_XDECREF(readResult);
 			self->finishedInput = 1;
 		}
 		else {
 			self->readResult = readResult;
 		}
 	}
 
 	/* EOF */
 	if (0 == readSize) {
 		self->input.src = NULL;
 		self->input.size = 0;
 		self->input.pos = 0;
 
 		zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 			&self->input, ZSTD_e_end);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "error ending compression stream: %s",
 				ZSTD_getErrorName(zresult));
 			return NULL;
 		}
 
 		assert(self->output.pos);
 
 		if (0 == zresult) {
 			self->finishedOutput = 1;
 		}
 
 		chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 		self->output.pos = 0;
 		return chunk;
 	}
 
 	/* New data from reader. Feed into compressor. */
 	self->input.src = readBuffer;
 	self->input.size = readSize;
 	self->input.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_compressStream2(self->compressor->cctx, &self->output,
 		&self->input, ZSTD_e_continue);
 	Py_END_ALLOW_THREADS
 
 	/* The input buffer currently points to memory managed by Python
 	(readBuffer). This object was allocated by this function. If it wasn't
 	fully consumed, we need to release it in a subsequent function call.
 	If it is fully consumed, do that now.
 	*/
 	if (self->input.pos == self->input.size) {
 		self->input.src = NULL;
 		self->input.pos = 0;
 		self->input.size = 0;
 		Py_XDECREF(self->readResult);
 		self->readResult = NULL;
 	}
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult));
 		return NULL;
 	}
 
 	assert(self->input.pos <= self->input.size);
 
 	/* If we didn't write anything, start the process over. */
 	if (0 == self->output.pos) {
 		goto feedcompressor;
 	}
 
 	chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos);
 	self->output.pos = 0;
 	return chunk;
 }
 
 PyTypeObject ZstdCompressorIteratorType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdCompressorIterator",   /* tp_name */
 	sizeof(ZstdCompressorIterator),  /* tp_basicsize */
 	0,                               /* tp_itemsize */
 	(destructor)ZstdCompressorIterator_dealloc, /* tp_dealloc */
 	0,                               /* tp_print */
 	0,                               /* tp_getattr */
 	0,                               /* tp_setattr */
 	0,                               /* tp_compare */
 	0,                               /* tp_repr */
 	0,                               /* tp_as_number */
 	0,                               /* tp_as_sequence */
 	0,                               /* tp_as_mapping */
 	0,                               /* tp_hash */
 	0,                               /* tp_call */
 	0,                               /* tp_str */
 	0,                               /* tp_getattro */
 	0,                               /* tp_setattro */
 	0,                               /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdCompressorIterator__doc__,   /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	ZstdCompressorIterator_iter,     /* tp_iter */
 	(iternextfunc)ZstdCompressorIterator_iternext, /* tp_iternext */
 	0,                               /* tp_methods */
 	0,                               /* tp_members */
 	0,                               /* tp_getset */
 	0,                               /* tp_base */
 	0,                               /* tp_dict */
 	0,                               /* tp_descr_get */
 	0,                               /* tp_descr_set */
 	0,                               /* tp_dictoffset */
 	0,                               /* tp_init */
 	0,                               /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void compressoriterator_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdCompressorIteratorType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdCompressorIteratorType, &PyType_Type);
 	if (PyType_Ready(&ZstdCompressorIteratorType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/decompressionreader.c

@@ -1,781 +1,781 b''
 /**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 static void set_unsupported_operation(void) {
 	PyObject* iomod;
 	PyObject* exc;
 
 	iomod = PyImport_ImportModule("io");
 	if (NULL == iomod) {
 		return;
 	}
 
 	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
 	if (NULL == exc) {
 		Py_DECREF(iomod);
 		return;
 	}
 
 	PyErr_SetNone(exc);
 	Py_DECREF(exc);
 	Py_DECREF(iomod);
 }
 
 static void reader_dealloc(ZstdDecompressionReader* self) {
 	Py_XDECREF(self->decompressor);
 	Py_XDECREF(self->reader);
 
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 	}
 
 	PyObject_Del(self);
 }
 
 static ZstdDecompressionReader* reader_enter(ZstdDecompressionReader* self) {
 	if (self->entered) {
 		PyErr_SetString(PyExc_ValueError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
 	self->entered = 1;
 
 	Py_INCREF(self);
 	return self;
 }
 
 static PyObject* reader_exit(ZstdDecompressionReader* self, PyObject* args) {
 	PyObject* exc_type;
 	PyObject* exc_value;
 	PyObject* exc_tb;
 
 	if (!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type, &exc_value, &exc_tb)) {
 		return NULL;
 	}
 
 	self->entered = 0;
 	self->closed = 1;
 
 	/* Release resources. */
 	Py_CLEAR(self->reader);
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	Py_CLEAR(self->decompressor);
 
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_readable(PyObject* self) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* reader_writable(PyObject* self) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* reader_seekable(PyObject* self) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* reader_close(ZstdDecompressionReader* self) {
 	self->closed = 1;
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_flush(PyObject* self) {
 	Py_RETURN_NONE;
 }
 
 static PyObject* reader_isatty(PyObject* self) {
 	Py_RETURN_FALSE;
 }
 
 /**
  * Read available input.
  *
  * Returns 0 if no data was added to input.
  * Returns 1 if new input data is available.
  * Returns -1 on error and sets a Python exception as a side-effect.
  */
 int read_decompressor_input(ZstdDecompressionReader* self) {
 	if (self->finishedInput) {
 		return 0;
 	}
 
 	if (self->input.pos != self->input.size) {
 		return 0;
 	}
 
 	if (self->reader) {
         Py_buffer buffer;
 
         assert(self->readResult == NULL);
         self->readResult = PyObject_CallMethod(self->reader, "read",
             "k", self->readSize);
         if (NULL == self->readResult) {
             return -1;
         }
 
         memset(&buffer, 0, sizeof(buffer));
 
         if (0 != PyObject_GetBuffer(self->readResult, &buffer, PyBUF_CONTIG_RO)) {
             return -1;
         }
 
         /* EOF */
         if (0 == buffer.len) {
             self->finishedInput = 1;
             Py_CLEAR(self->readResult);
         }
         else {
             self->input.src = buffer.buf;
             self->input.size = buffer.len;
             self->input.pos = 0;
         }
 
         PyBuffer_Release(&buffer);
 	}
 	else {
 		assert(self->buffer.buf);
         /*
          * We should only get here once since expectation is we always
          * exhaust input buffer before reading again.
          */
         assert(self->input.src == NULL);
 
 		self->input.src = self->buffer.buf;
         self->input.size = self->buffer.len;
         self->input.pos = 0;
 	}
 
 	return 1;
 }
 
 /**
  * Decompresses available input into an output buffer.
  *
  * Returns 0 if we need more input.
  * Returns 1 if output buffer should be emitted.
  * Returns -1 on error and sets a Python exception.
  */
 int decompress_input(ZstdDecompressionReader* self, ZSTD_outBuffer* output) {
 	size_t zresult;
 
 	if (self->input.pos >= self->input.size) {
 		return 0;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_decompressStream(self->decompressor->dctx, output, &self->input);
 	Py_END_ALLOW_THREADS
 
 	/* Input exhausted. Clear our state tracking. */
 	if (self->input.pos == self->input.size) {
 		memset(&self->input, 0, sizeof(self->input));
 		Py_CLEAR(self->readResult);
 
 		if (self->buffer.buf) {
 			self->finishedInput = 1;
 		}
 	}
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "zstd decompress error: %s", ZSTD_getErrorName(zresult));
 		return -1;
 	}
 
 	/* We fulfilled the full read request. Signal to emit. */
 	if (output->pos && output->pos == output->size) {
 		return 1;
 	}
 	/* We're at the end of a frame and we aren't allowed to return data
 	   spanning frames. */
 	else if (output->pos && zresult == 0 && !self->readAcrossFrames) {
 		return 1;
 	}
 
 	/* There is more room in the output. Signal to collect more data. */
 	return 0;
 }
 
 static PyObject* reader_read(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
 	int decompressResult, readResult;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (size == -1) {
 		return PyObject_CallMethod((PyObject*)self, "readall", NULL);
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	output.dst = resultBuffer;
 	output.size = resultSize;
 	output.pos = 0;
 
 readinput:
 
 	decompressResult = decompress_input(self, &output);
 
 	if (-1 == decompressResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == decompressResult) { }
 	else if (1 == decompressResult) {
 		self->bytesDecompressed += output.pos;
 
 		if (output.pos != output.size) {
 			if (safe_pybytes_resize(&result, output.pos)) {
 				Py_XDECREF(result);
 				return NULL;
 			}
 		}
 		return result;
 	}
 	else {
 		assert(0);
 	}
 
 	readResult = read_decompressor_input(self);
 
 	if (-1 == readResult) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 	else if (0 == readResult) {}
 	else if (1 == readResult) {}
 	else {
 		assert(0);
 	}
 
 	if (self->input.size) {
 		goto readinput;
 	}
 
 	/* EOF */
 	self->bytesDecompressed += output.pos;
 
 	if (safe_pybytes_resize(&result, output.pos)) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	return result;
 }
 
 static PyObject* reader_read1(ZstdDecompressionReader* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"size",
 		NULL
 	};
 
 	Py_ssize_t size = -1;
 	PyObject* result = NULL;
 	char* resultBuffer;
 	Py_ssize_t resultSize;
 	ZSTD_outBuffer output;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &size)) {
 		return NULL;
 	}
 
 	if (size < -1) {
 		PyErr_SetString(PyExc_ValueError, "cannot read negative amounts less than -1");
 		return NULL;
 	}
 
 	if (self->finishedOutput || size == 0) {
 		return PyBytes_FromStringAndSize("", 0);
 	}
 
 	if (size == -1) {
 		size = ZSTD_DStreamOutSize();
 	}
 
 	result = PyBytes_FromStringAndSize(NULL, size);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	PyBytes_AsStringAndSize(result, &resultBuffer, &resultSize);
 
 	output.dst = resultBuffer;
 	output.size = resultSize;
 	output.pos = 0;
 
 	/* read1() is supposed to use at most 1 read() from the underlying stream.
 	 * However, we can't satisfy this requirement with decompression due to the
 	 * nature of how decompression works. Our strategy is to read + decompress
 	 * until we get any output, at which point we return. This satisfies the
 	 * intent of the read1() API to limit read operations.
 	 */
 	while (!self->finishedInput) {
 		int readResult, decompressResult;
 
 		readResult = read_decompressor_input(self);
 		if (-1 == readResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == readResult || 1 == readResult) { }
 		else {
 			assert(0);
 		}
 
 		decompressResult = decompress_input(self, &output);
 
 		if (-1 == decompressResult) {
 			Py_XDECREF(result);
 			return NULL;
 		}
 		else if (0 == decompressResult || 1 == decompressResult) { }
 		else {
 			assert(0);
 		}
 
 		if (output.pos) {
 		    break;
 		}
 	}
 
 	self->bytesDecompressed += output.pos;
 	if (safe_pybytes_resize(&result, output.pos)) {
 		Py_XDECREF(result);
 		return NULL;
 	}
 
 	return result;
 }
 
 static PyObject* reader_readinto(ZstdDecompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	ZSTD_outBuffer output;
 	int decompressResult, readResult;
 	PyObject* result = NULL;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"destination buffer should be contiguous and have at most one dimension");
 	    goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 readinput:
 
 	decompressResult = decompress_input(self, &output);
 
 	if (-1 == decompressResult) {
 		goto finally;
 	}
 	else if (0 == decompressResult) { }
 	else if (1 == decompressResult) {
 		self->bytesDecompressed += output.pos;
 		result = PyLong_FromSize_t(output.pos);
 		goto finally;
 	}
 	else {
 		assert(0);
 	}
 
 	readResult = read_decompressor_input(self);
 
 	if (-1 == readResult) {
 		goto finally;
 	}
 	else if (0 == readResult) {}
 	else if (1 == readResult) {}
 	else {
 		assert(0);
 	}
 
 	if (self->input.size) {
 		goto readinput;
 	}
 
 	/* EOF */
 	self->bytesDecompressed += output.pos;
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_readinto1(ZstdDecompressionReader* self, PyObject* args) {
 	Py_buffer dest;
 	ZSTD_outBuffer output;
 	PyObject* result = NULL;
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->finishedOutput) {
 		return PyLong_FromLong(0);
 	}
 
 	if (!PyArg_ParseTuple(args, "w*:readinto1", &dest)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&dest, 'C') || dest.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"destination buffer should be contiguous and have at most one dimension");
 	    goto finally;
 	}
 
 	output.dst = dest.buf;
 	output.size = dest.len;
 	output.pos = 0;
 
 	while (!self->finishedInput && !self->finishedOutput) {
 		int decompressResult, readResult;
 
 		readResult = read_decompressor_input(self);
 
 		if (-1 == readResult) {
 			goto finally;
 		}
 		else if (0 == readResult || 1 == readResult) {}
 		else {
 			assert(0);
 		}
 
 		decompressResult = decompress_input(self, &output);
 
 		if (-1 == decompressResult) {
 			goto finally;
 		}
 		else if (0 == decompressResult || 1 == decompressResult) {}
 		else {
 			assert(0);
 		}
 
 		if (output.pos) {
 			break;
 		}
 	}
 
 	self->bytesDecompressed += output.pos;
 	result = PyLong_FromSize_t(output.pos);
 
 finally:
 	PyBuffer_Release(&dest);
 
 	return result;
 }
 
 static PyObject* reader_readall(PyObject* self) {
 	PyObject* chunks = NULL;
 	PyObject* empty = NULL;
 	PyObject* result = NULL;
 
 	/* Our strategy is to collect chunks into a list then join all the
 	 * chunks at the end. We could potentially use e.g. an io.BytesIO. But
 	 * this feels simple enough to implement and avoids potentially expensive
 	 * reallocations of large buffers.
 	 */
 	chunks = PyList_New(0);
 	if (NULL == chunks) {
 		return NULL;
 	}
 
 	while (1) {
 		PyObject* chunk = PyObject_CallMethod(self, "read", "i", 1048576);
 		if (NULL == chunk) {
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		if (!PyBytes_Size(chunk)) {
 			Py_DECREF(chunk);
 			break;
 		}
 
 		if (PyList_Append(chunks, chunk)) {
 			Py_DECREF(chunk);
 			Py_DECREF(chunks);
 			return NULL;
 		}
 
 		Py_DECREF(chunk);
 	}
 
 	empty = PyBytes_FromStringAndSize("", 0);
 	if (NULL == empty) {
 		Py_DECREF(chunks);
 		return NULL;
 	}
 
 	result = PyObject_CallMethod(empty, "join", "O", chunks);
 
 	Py_DECREF(empty);
 	Py_DECREF(chunks);
 
 	return result;
 }
 
 static PyObject* reader_readline(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_readlines(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_seek(ZstdDecompressionReader* self, PyObject* args) {
 	Py_ssize_t pos;
 	int whence = 0;
 	unsigned long long readAmount = 0;
 	size_t defaultOutSize = ZSTD_DStreamOutSize();
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &whence)) {
 		return NULL;
 	}
 
 	if (whence == SEEK_SET) {
 		if (pos < 0) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek to negative position with SEEK_SET");
 			return NULL;
 		}
 
 		if ((unsigned long long)pos < self->bytesDecompressed) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek zstd decompression stream backwards");
 			return NULL;
 		}
 
 		readAmount = pos - self->bytesDecompressed;
 	}
 	else if (whence == SEEK_CUR) {
 		if (pos < 0) {
 			PyErr_SetString(PyExc_ValueError,
 				"cannot seek zstd decompression stream backwards");
 			return NULL;
 		}
 
 		readAmount = pos;
 	}
 	else if (whence == SEEK_END) {
 		/* We /could/ support this with pos==0. But let's not do that until someone
 		   needs it. */
 		PyErr_SetString(PyExc_ValueError,
 			"zstd decompression streams cannot be seeked with SEEK_END");
 		return NULL;
 	}
 
 	/* It is a bit inefficient to do this via the Python API. But since there
 	   is a bit of state tracking involved to read from this type, it is the
 	   easiest to implement. */
 	while (readAmount) {
 		Py_ssize_t readSize;
 		PyObject* readResult = PyObject_CallMethod((PyObject*)self, "read", "K",
 			readAmount < defaultOutSize ? readAmount : defaultOutSize);
 
 		if (!readResult) {
 			return NULL;
 		}
 
 		readSize = PyBytes_GET_SIZE(readResult);
 
 		Py_CLEAR(readResult);
 
 		/* Empty read means EOF. */
 		if (!readSize) {
 			break;
 		}
 
 		readAmount -= readSize;
 	}
 
 	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
 }
 
 static PyObject* reader_tell(ZstdDecompressionReader* self) {
 	/* TODO should this raise OSError since stream isn't seekable? */
 	return PyLong_FromUnsignedLongLong(self->bytesDecompressed);
 }
 
 static PyObject* reader_write(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_writelines(PyObject* self, PyObject* args) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iter(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyObject* reader_iternext(PyObject* self) {
 	set_unsupported_operation();
 	return NULL;
 }
 
 static PyMethodDef reader_methods[] = {
 	{ "__enter__", (PyCFunction)reader_enter, METH_NOARGS,
 	PyDoc_STR("Enter a compression context") },
 	{ "__exit__", (PyCFunction)reader_exit, METH_VARARGS,
 	PyDoc_STR("Exit a compression context") },
 	{ "close", (PyCFunction)reader_close, METH_NOARGS,
 	PyDoc_STR("Close the stream so it cannot perform any more operations") },
 	{ "flush", (PyCFunction)reader_flush, METH_NOARGS, PyDoc_STR("no-ops") },
 	{ "isatty", (PyCFunction)reader_isatty, METH_NOARGS, PyDoc_STR("Returns False") },
 	{ "readable", (PyCFunction)reader_readable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "read", (PyCFunction)reader_read, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("read compressed data") },
 	{ "read1", (PyCFunction)reader_read1, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("read compressed data") },
 	{ "readinto", (PyCFunction)reader_readinto, METH_VARARGS, NULL },
 	{ "readinto1", (PyCFunction)reader_readinto1, METH_VARARGS, NULL },
 	{ "readall", (PyCFunction)reader_readall, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readline", (PyCFunction)reader_readline, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "readlines", (PyCFunction)reader_readlines, METH_NOARGS, PyDoc_STR("Not implemented") },
 	{ "seek", (PyCFunction)reader_seek, METH_VARARGS, PyDoc_STR("Seek the stream") },
 	{ "seekable", (PyCFunction)reader_seekable, METH_NOARGS,
 	PyDoc_STR("Returns True") },
 	{ "tell", (PyCFunction)reader_tell, METH_NOARGS,
 	PyDoc_STR("Returns current number of bytes compressed") },
 	{ "writable", (PyCFunction)reader_writable, METH_NOARGS,
 	PyDoc_STR("Returns False") },
 	{ "write", (PyCFunction)reader_write, METH_VARARGS, PyDoc_STR("unsupported operation") },
 	{ "writelines", (PyCFunction)reader_writelines, METH_VARARGS, PyDoc_STR("unsupported operation") },
 	{ NULL, NULL }
 };
 
 static PyMemberDef reader_members[] = {
 	{ "closed", T_BOOL, offsetof(ZstdDecompressionReader, closed),
 	  READONLY, "whether stream is closed" },
 	{ NULL }
 };
 
 PyTypeObject ZstdDecompressionReaderType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressionReader", /* tp_name */
 	sizeof(ZstdDecompressionReader), /* tp_basicsize */
 	0, /* tp_itemsize */
 	(destructor)reader_dealloc, /* tp_dealloc */
 	0, /* tp_print */
 	0, /* tp_getattr */
 	0, /* tp_setattr */
 	0, /* tp_compare */
 	0, /* tp_repr */
 	0, /* tp_as_number */
 	0, /* tp_as_sequence */
 	0, /* tp_as_mapping */
 	0, /* tp_hash */
 	0, /* tp_call */
 	0, /* tp_str */
 	0, /* tp_getattro */
 	0, /* tp_setattro */
 	0, /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT, /* tp_flags */
 	0, /* tp_doc */
 	0, /* tp_traverse */
 	0, /* tp_clear */
 	0, /* tp_richcompare */
 	0, /* tp_weaklistoffset */
 	reader_iter, /* tp_iter */
 	reader_iternext, /* tp_iternext */
 	reader_methods, /* tp_methods */
 	reader_members, /* tp_members */
 	0, /* tp_getset */
 	0, /* tp_base */
 	0, /* tp_dict */
 	0, /* tp_descr_get */
 	0, /* tp_descr_set */
 	0, /* tp_dictoffset */
 	0, /* tp_init */
 	0, /* tp_alloc */
 	PyType_GenericNew, /* tp_new */
 };
 
 
 void decompressionreader_module_init(PyObject* mod) {
 	/* TODO make reader a sub-class of io.RawIOBase */
 
-	Py_TYPE(&ZstdDecompressionReaderType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdDecompressionReaderType, &PyType_Type);
 	if (PyType_Ready(&ZstdDecompressionReaderType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/decompressionwriter.c

@@ -1,295 +1,295 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdDecompressionWriter__doc,
 """A context manager used for writing decompressed output.\n"
 );
 
 static void ZstdDecompressionWriter_dealloc(ZstdDecompressionWriter* self) {
 	Py_XDECREF(self->decompressor);
 	Py_XDECREF(self->writer);
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdDecompressionWriter_enter(ZstdDecompressionWriter* self) {
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (self->entered) {
 		PyErr_SetString(ZstdError, "cannot __enter__ multiple times");
 		return NULL;
 	}
 
 	self->entered = 1;
 
 	Py_INCREF(self);
 	return (PyObject*)self;
 }
 
 static PyObject* ZstdDecompressionWriter_exit(ZstdDecompressionWriter* self, PyObject* args) {
 	self->entered = 0;
 
 	if (NULL == PyObject_CallMethod((PyObject*)self, "close", NULL)) {
 		return NULL;
 	}
 
 	Py_RETURN_FALSE;
 }
 
 static PyObject* ZstdDecompressionWriter_memory_size(ZstdDecompressionWriter* self) {
 	return PyLong_FromSize_t(ZSTD_sizeof_DCtx(self->decompressor->dctx));
 }
 
 static PyObject* ZstdDecompressionWriter_write(ZstdDecompressionWriter* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	PyObject* result = NULL;
 	Py_buffer source;
 	size_t zresult = 0;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	PyObject* res;
 	Py_ssize_t totalWrite = 0;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:write",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:write",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 	output.size = self->outSize;
 	output.pos = 0;
 
 	input.src = source.buf;
 	input.size = source.len;
 	input.pos = 0;
 
 	while (input.pos < (size_t)source.len) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_decompressStream(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyMem_Free(output.dst);
 			PyErr_Format(ZstdError, "zstd decompress error: %s",
 				ZSTD_getErrorName(zresult));
 			goto finally;
 		}
 
 		if (output.pos) {
 #if PY_MAJOR_VERSION >= 3
 			res = PyObject_CallMethod(self->writer, "write", "y#",
 #else
 			res = PyObject_CallMethod(self->writer, "write", "s#",
 #endif
 				output.dst, output.pos);
 			Py_XDECREF(res);
 			totalWrite += output.pos;
 			output.pos = 0;
 		}
 	}
 
 	PyMem_Free(output.dst);
 
 	if (self->writeReturnRead) {
 		result = PyLong_FromSize_t(input.pos);
 	}
 	else {
 		result = PyLong_FromSsize_t(totalWrite);
 	}
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
 static PyObject* ZstdDecompressionWriter_close(ZstdDecompressionWriter* self) {
 	PyObject* result;
 
 	if (self->closed) {
 		Py_RETURN_NONE;
 	}
 
 	result = PyObject_CallMethod((PyObject*)self, "flush", NULL);
 	self->closed = 1;
 
 	if (NULL == result) {
 		return NULL;
 	}
 
 	/* Call close on underlying stream as well. */
 	if (PyObject_HasAttrString(self->writer, "close")) {
 		return PyObject_CallMethod(self->writer, "close", NULL);
 	}
 
 	Py_RETURN_NONE;
 }
 
 static PyObject* ZstdDecompressionWriter_fileno(ZstdDecompressionWriter* self) {
 	if (PyObject_HasAttrString(self->writer, "fileno")) {
 		return PyObject_CallMethod(self->writer, "fileno", NULL);
 	}
 	else {
 		PyErr_SetString(PyExc_OSError, "fileno not available on underlying writer");
 		return NULL;
 	}
 }
 
 static PyObject* ZstdDecompressionWriter_flush(ZstdDecompressionWriter* self) {
 	if (self->closed) {
 		PyErr_SetString(PyExc_ValueError, "stream is closed");
 		return NULL;
 	}
 
 	if (PyObject_HasAttrString(self->writer, "flush")) {
 		return PyObject_CallMethod(self->writer, "flush", NULL);
 	}
 	else {
 		Py_RETURN_NONE;
 	}
 }
 
 static PyObject* ZstdDecompressionWriter_false(PyObject* self, PyObject* args) {
 	Py_RETURN_FALSE;
 }
 
 static PyObject* ZstdDecompressionWriter_true(PyObject* self, PyObject* args) {
 	Py_RETURN_TRUE;
 }
 
 static PyObject* ZstdDecompressionWriter_unsupported(PyObject* self, PyObject* args, PyObject* kwargs) {
 	PyObject* iomod;
 	PyObject* exc;
 
 	iomod = PyImport_ImportModule("io");
 	if (NULL == iomod) {
 		return NULL;
 	}
 
 	exc = PyObject_GetAttrString(iomod, "UnsupportedOperation");
 	if (NULL == exc) {
 		Py_DECREF(iomod);
 		return NULL;
 	}
 
 	PyErr_SetNone(exc);
 	Py_DECREF(exc);
 	Py_DECREF(iomod);
 
 	return NULL;
 }
 
 static PyMethodDef ZstdDecompressionWriter_methods[] = {
 	{ "__enter__", (PyCFunction)ZstdDecompressionWriter_enter, METH_NOARGS,
 	PyDoc_STR("Enter a decompression context.") },
 	{ "__exit__", (PyCFunction)ZstdDecompressionWriter_exit, METH_VARARGS,
 	PyDoc_STR("Exit a decompression context.") },
 	{ "memory_size", (PyCFunction)ZstdDecompressionWriter_memory_size, METH_NOARGS,
 	PyDoc_STR("Obtain the memory size in bytes of the underlying decompressor.") },
 	{ "close", (PyCFunction)ZstdDecompressionWriter_close, METH_NOARGS, NULL },
 	{ "fileno", (PyCFunction)ZstdDecompressionWriter_fileno, METH_NOARGS, NULL },
 	{ "flush", (PyCFunction)ZstdDecompressionWriter_flush, METH_NOARGS, NULL },
 	{ "isatty", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
 	{ "readable", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
 	{ "readline", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readlines", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "seek", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "seekable", ZstdDecompressionWriter_false, METH_NOARGS, NULL },
 	{ "tell", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "truncate", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "writable", ZstdDecompressionWriter_true, METH_NOARGS, NULL },
 	{ "writelines" , (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "read", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readall", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "readinto", (PyCFunction)ZstdDecompressionWriter_unsupported, METH_VARARGS | METH_KEYWORDS, NULL },
 	{ "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS | METH_KEYWORDS,
 	PyDoc_STR("Compress data") },
 	{ NULL, NULL }
 };
 
 static PyMemberDef ZstdDecompressionWriter_members[] = {
 	{ "closed", T_BOOL, offsetof(ZstdDecompressionWriter, closed), READONLY, NULL },
 	{ NULL }
 };
 
 PyTypeObject ZstdDecompressionWriterType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressionWriter", /* tp_name */
 	sizeof(ZstdDecompressionWriter),/* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)ZstdDecompressionWriter_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdDecompressionWriter__doc,   /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	ZstdDecompressionWriter_methods,/* tp_methods */
 	ZstdDecompressionWriter_members,/* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	0,                              /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void decompressionwriter_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdDecompressionWriterType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdDecompressionWriterType, &PyType_Type);
 	if (PyType_Ready(&ZstdDecompressionWriterType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/decompressobj.c

@@ -1,202 +1,202 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(DecompressionObj__doc__,
 "Perform decompression using a standard library compatible API.\n"
 );
 
 static void DecompressionObj_dealloc(ZstdDecompressionObj* self) {
 	Py_XDECREF(self->decompressor);
 
 	PyObject_Del(self);
 }
 
 static PyObject* DecompressionObj_decompress(ZstdDecompressionObj* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	Py_buffer source;
 	size_t zresult;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	PyObject* result = NULL;
 	Py_ssize_t resultSize = 0;
 
 	output.dst = NULL;
 
 	if (self->finished) {
 		PyErr_SetString(ZstdError, "cannot use a decompressobj multiple times");
 		return NULL;
 	}
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:decompress",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:decompress",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	/* Special case of empty input. Output will always be empty. */
 	if (source.len == 0) {
 		result = PyBytes_FromString("");
 		goto finally;
 	}
 
 	input.src = source.buf;
 	input.size = source.len;
 	input.pos = 0;
 
 	output.dst = PyMem_Malloc(self->outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
 		goto except;
 	}
 	output.size = self->outSize;
 	output.pos = 0;
 
 	while (1) {
 		Py_BEGIN_ALLOW_THREADS
 		zresult = ZSTD_decompressStream(self->decompressor->dctx, &output, &input);
 		Py_END_ALLOW_THREADS
 
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "zstd decompressor error: %s",
 				ZSTD_getErrorName(zresult));
 			goto except;
 		}
 
 		if (0 == zresult) {
 			self->finished = 1;
 		}
 
 		if (output.pos) {
 			if (result) {
 				resultSize = PyBytes_GET_SIZE(result);
 				if (-1 == safe_pybytes_resize(&result, resultSize + output.pos)) {
 					Py_XDECREF(result);
 					goto except;
 				}
 
 				memcpy(PyBytes_AS_STRING(result) + resultSize,
 					output.dst, output.pos);
 			}
 			else {
 				result = PyBytes_FromStringAndSize(output.dst, output.pos);
 				if (!result) {
 					goto except;
 				}
 			}
 		}
 
 		if (zresult == 0 || (input.pos == input.size && output.pos == 0)) {
 			break;
 		}
 
 		output.pos = 0;
 	}
 
 	if (!result) {
 		result = PyBytes_FromString("");
 	}
 
 	goto finally;
 
 except:
 	Py_CLEAR(result);
 
 finally:
 	PyMem_Free(output.dst);
 	PyBuffer_Release(&source);
 
 	return result;
 }
 
 static PyObject* DecompressionObj_flush(ZstdDecompressionObj* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"length",
 		NULL
 	};
 
 	PyObject* length = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O:flush", kwlist, &length)) {
 	return NULL;
 	}
 
 	Py_RETURN_NONE;
 }
 
 static PyMethodDef DecompressionObj_methods[] = {
 	{ "decompress", (PyCFunction)DecompressionObj_decompress,
 	  METH_VARARGS | METH_KEYWORDS, PyDoc_STR("decompress data") },
 	{ "flush", (PyCFunction)DecompressionObj_flush,
 	  METH_VARARGS | METH_KEYWORDS, PyDoc_STR("no-op") },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdDecompressionObjType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressionObj",    /* tp_name */
 	sizeof(ZstdDecompressionObj),   /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)DecompressionObj_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	DecompressionObj__doc__,        /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	DecompressionObj_methods,       /* tp_methods */
 	0,                              /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	0,                              /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void decompressobj_module_init(PyObject* module) {
-	Py_TYPE(&ZstdDecompressionObjType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdDecompressionObjType, &PyType_Type);
 	if (PyType_Ready(&ZstdDecompressionObjType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/decompressor.c

@@ -1,1822 +1,1822 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 #include "pool.h"
 
 extern PyObject* ZstdError;
 
 /**
  * Ensure the ZSTD_DCtx on a decompressor is initiated and ready for a new operation.
  */
 int ensure_dctx(ZstdDecompressor* decompressor, int loadDict) {
 	size_t zresult;
 
 	ZSTD_DCtx_reset(decompressor->dctx, ZSTD_reset_session_only);
 
 	if (decompressor->maxWindowSize) {
 		zresult = ZSTD_DCtx_setMaxWindowSize(decompressor->dctx, decompressor->maxWindowSize);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "unable to set max window size: %s",
 				ZSTD_getErrorName(zresult));
 			return 1;
 		}
 	}
 
 	zresult = ZSTD_DCtx_setFormat(decompressor->dctx, decompressor->format);
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "unable to set decoding format: %s",
 			ZSTD_getErrorName(zresult));
 		return 1;
 	}
 
 	if (loadDict && decompressor->dict) {
 		if (ensure_ddict(decompressor->dict)) {
 			return 1;
 		}
 
 		zresult = ZSTD_DCtx_refDDict(decompressor->dctx, decompressor->dict->ddict);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(ZstdError, "unable to reference prepared dictionary: %s",
 				ZSTD_getErrorName(zresult));
 			return 1;
 		}
 	}
 
 	return 0;
 }
 
 PyDoc_STRVAR(Decompressor__doc__,
 "ZstdDecompressor(dict_data=None)\n"
 "\n"
 "Create an object used to perform Zstandard decompression.\n"
 "\n"
 "An instance can perform multiple decompression operations."
 );
 
 static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"dict_data",
 		"max_window_size",
 		"format",
 		NULL
 	};
 
 	ZstdCompressionDict* dict = NULL;
 	Py_ssize_t maxWindowSize = 0;
 	ZSTD_format_e format = ZSTD_f_zstd1;
 
 	self->dctx = NULL;
 	self->dict = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!nI:ZstdDecompressor", kwlist,
 		&ZstdCompressionDictType, &dict, &maxWindowSize, &format)) {
 		return -1;
 	}
 
 	self->dctx = ZSTD_createDCtx();
 	if (!self->dctx) {
 		PyErr_NoMemory();
 		goto except;
 	}
 
 	self->maxWindowSize = maxWindowSize;
 	self->format = format;
 
 	if (dict) {
 		self->dict = dict;
 		Py_INCREF(dict);
 	}
 
 	if (ensure_dctx(self, 1)) {
 		goto except;
 	}
 
 	return 0;
 
 except:
 	Py_CLEAR(self->dict);
 
 	if (self->dctx) {
 		ZSTD_freeDCtx(self->dctx);
 		self->dctx = NULL;
 	}
 
 	return -1;
 }
 
 static void Decompressor_dealloc(ZstdDecompressor* self) {
 	Py_CLEAR(self->dict);
 
 	if (self->dctx) {
 		ZSTD_freeDCtx(self->dctx);
 		self->dctx = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 PyDoc_STRVAR(Decompressor_memory_size__doc__,
 "memory_size() -- Size of decompression context, in bytes\n"
 );
 
 static PyObject* Decompressor_memory_size(ZstdDecompressor* self) {
 	if (self->dctx) {
 		return PyLong_FromSize_t(ZSTD_sizeof_DCtx(self->dctx));
 	}
 	else {
 		PyErr_SetString(ZstdError, "no decompressor context found; this should never happen");
 		return NULL;
 	}
 }
 
 PyDoc_STRVAR(Decompressor_copy_stream__doc__,
 	"copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n"
 	"\n"
 	"Compressed data will be read from ``ifh``, decompressed, and written to\n"
 	"``ofh``. ``ifh`` must have a ``read(size)`` method. ``ofh`` must have a\n"
 	"``write(data)`` method.\n"
 	"\n"
 	"The optional ``read_size`` and ``write_size`` arguments control the chunk\n"
 	"size of data that is ``read()`` and ``write()`` between streams. They default\n"
 	"to the default input and output sizes of zstd decompressor streams.\n"
 );
 
 static PyObject* Decompressor_copy_stream(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"ifh",
 		"ofh",
 		"read_size",
 		"write_size",
 		NULL
 	};
 
 	PyObject* source;
 	PyObject* dest;
 	size_t inSize = ZSTD_DStreamInSize();
 	size_t outSize = ZSTD_DStreamOutSize();
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_ssize_t totalRead = 0;
 	Py_ssize_t totalWrite = 0;
 	char* readBuffer;
 	Py_ssize_t readSize;
 	PyObject* readResult = NULL;
 	PyObject* res = NULL;
 	size_t zresult = 0;
 	PyObject* writeResult;
 	PyObject* totalReadPy;
 	PyObject* totalWritePy;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk:copy_stream", kwlist,
 		&source, &dest, &inSize, &outSize)) {
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(source, "read")) {
 		PyErr_SetString(PyExc_ValueError, "first argument must have a read() method");
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(dest, "write")) {
 		PyErr_SetString(PyExc_ValueError, "second argument must have a write() method");
 		return NULL;
 	}
 
 	/* Prevent free on uninitialized memory in finally. */
 	output.dst = NULL;
 
 	if (ensure_dctx(self, 1)) {
 		res = NULL;
 		goto finally;
 	}
 
 	output.dst = PyMem_Malloc(outSize);
 	if (!output.dst) {
 		PyErr_NoMemory();
 		res = NULL;
 		goto finally;
 	}
 	output.size = outSize;
 	output.pos = 0;
 
 	/* Read source stream until EOF */
 	while (1) {
 		readResult = PyObject_CallMethod(source, "read", "n", inSize);
 		if (!readResult) {
 			PyErr_SetString(ZstdError, "could not read() from source");
 			goto finally;
 		}
 
 		PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 
 		/* If no data was read, we're at EOF. */
 		if (0 == readSize) {
 			break;
 		}
 
 		totalRead += readSize;
 
 		/* Send data to decompressor */
 		input.src = readBuffer;
 		input.size = readSize;
 		input.pos = 0;
 
 		while (input.pos < input.size) {
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_decompressStream(self->dctx, &output, &input);
 			Py_END_ALLOW_THREADS
 
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "zstd decompressor error: %s",
 					ZSTD_getErrorName(zresult));
 				res = NULL;
 				goto finally;
 			}
 
 			if (output.pos) {
 #if PY_MAJOR_VERSION >= 3
 				writeResult = PyObject_CallMethod(dest, "write", "y#",
 #else
 				writeResult = PyObject_CallMethod(dest, "write", "s#",
 #endif
 					output.dst, output.pos);
 
 				Py_XDECREF(writeResult);
 				totalWrite += output.pos;
 				output.pos = 0;
 			}
 		}
 
 		Py_CLEAR(readResult);
 	}
 
 	/* Source stream is exhausted. Finish up. */
 
 	totalReadPy = PyLong_FromSsize_t(totalRead);
 	totalWritePy = PyLong_FromSsize_t(totalWrite);
 	res = PyTuple_Pack(2, totalReadPy, totalWritePy);
 	Py_DECREF(totalReadPy);
 	Py_DECREF(totalWritePy);
 
 finally:
 	if (output.dst) {
 		PyMem_Free(output.dst);
 	}
 
 	Py_XDECREF(readResult);
 
 	return res;
 }
 
 PyDoc_STRVAR(Decompressor_decompress__doc__,
 "decompress(data[, max_output_size=None]) -- Decompress data in its entirety\n"
 "\n"
 "This method will decompress the entirety of the argument and return the\n"
 "result.\n"
 "\n"
 "The input bytes are expected to contain a full Zstandard frame (something\n"
 "compressed with ``ZstdCompressor.compress()`` or similar). If the input does\n"
 "not contain a full frame, an exception will be raised.\n"
 "\n"
 "If the frame header of the compressed data does not contain the content size\n"
 "``max_output_size`` must be specified or ``ZstdError`` will be raised. An\n"
 "allocation of size ``max_output_size`` will be performed and an attempt will\n"
 "be made to perform decompression into that buffer. If the buffer is too\n"
 "small or cannot be allocated, ``ZstdError`` will be raised. The buffer will\n"
 "be resized if it is too large.\n"
 "\n"
 "Uncompressed data could be much larger than compressed data. As a result,\n"
 "calling this function could result in a very large memory allocation being\n"
 "performed to hold the uncompressed data. Therefore it is **highly**\n"
 "recommended to use a streaming decompression method instead of this one.\n"
 );
 
 PyObject* Decompressor_decompress(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		"max_output_size",
 		NULL
 	};
 
 	Py_buffer source;
 	Py_ssize_t maxOutputSize = 0;
 	unsigned long long decompressedSize;
 	size_t destCapacity;
 	PyObject* result = NULL;
 	size_t zresult;
 	ZSTD_outBuffer outBuffer;
 	ZSTD_inBuffer inBuffer;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|n:decompress",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|n:decompress",
 #endif
 		kwlist, &source, &maxOutputSize)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	if (ensure_dctx(self, 1)) {
 		goto finally;
 	}
 
 	decompressedSize = ZSTD_getFrameContentSize(source.buf, source.len);
 
 	if (ZSTD_CONTENTSIZE_ERROR == decompressedSize) {
 		PyErr_SetString(ZstdError, "error determining content size from frame header");
 		goto finally;
 	}
 	/* Special case of empty frame. */
 	else if (0 == decompressedSize) {
 		result = PyBytes_FromStringAndSize("", 0);
 		goto finally;
 	}
 	/* Missing content size in frame header. */
 	if (ZSTD_CONTENTSIZE_UNKNOWN == decompressedSize) {
 		if (0 == maxOutputSize) {
 			PyErr_SetString(ZstdError, "could not determine content size in frame header");
 			goto finally;
 		}
 
 		result = PyBytes_FromStringAndSize(NULL, maxOutputSize);
 		destCapacity = maxOutputSize;
 		decompressedSize = 0;
 	}
 	/* Size is recorded in frame header. */
 	else {
 		assert(SIZE_MAX >= PY_SSIZE_T_MAX);
 		if (decompressedSize > PY_SSIZE_T_MAX) {
 			PyErr_SetString(ZstdError, "frame is too large to decompress on this platform");
 			goto finally;
 		}
 
 		result = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)decompressedSize);
 		destCapacity = (size_t)decompressedSize;
 	}
 
 	if (!result) {
 		goto finally;
 	}
 
 	outBuffer.dst = PyBytes_AsString(result);
 	outBuffer.size = destCapacity;
 	outBuffer.pos = 0;
 
 	inBuffer.src = source.buf;
 	inBuffer.size = source.len;
 	inBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult));
 		Py_CLEAR(result);
 		goto finally;
 	}
 	else if (zresult) {
 		PyErr_Format(ZstdError, "decompression error: did not decompress full frame");
 		Py_CLEAR(result);
 		goto finally;
 	}
 	else if (decompressedSize && outBuffer.pos != decompressedSize) {
 		PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu",
 			zresult, decompressedSize);
 		Py_CLEAR(result);
 		goto finally;
 	}
 	else if (outBuffer.pos < destCapacity) {
 		if (safe_pybytes_resize(&result, outBuffer.pos)) {
 			Py_CLEAR(result);
 			goto finally;
 		}
 	}
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_decompressobj__doc__,
 "decompressobj([write_size=default])\n"
 "\n"
 "Incrementally feed data into a decompressor.\n"
 "\n"
 "The returned object exposes a ``decompress(data)`` method. This makes it\n"
 "compatible with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor`` so that\n"
 "callers can swap in the zstd decompressor while using the same API.\n"
 );
 
 static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"write_size",
 		NULL
 	};
 
 	ZstdDecompressionObj* result = NULL;
 	size_t outSize = ZSTD_DStreamOutSize();
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|k:decompressobj", kwlist, &outSize)) {
 		return NULL;
 	}
 
 	if (!outSize) {
 		PyErr_SetString(PyExc_ValueError, "write_size must be positive");
 		return NULL;
 	}
 
 	result = (ZstdDecompressionObj*)PyObject_CallObject((PyObject*)&ZstdDecompressionObjType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	if (ensure_dctx(self, 1)) {
 		Py_DECREF(result);
 		return NULL;
 	}
 
 	result->decompressor = self;
 	Py_INCREF(result->decompressor);
 	result->outSize = outSize;
 
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_read_to_iter__doc__,
 "read_to_iter(reader[, read_size=default, write_size=default, skip_bytes=0])\n"
 "Read compressed data and return an iterator\n"
 "\n"
 "Returns an iterator of decompressed data chunks produced from reading from\n"
 "the ``reader``.\n"
 "\n"
 "Compressed data will be obtained from ``reader`` by calling the\n"
 "``read(size)`` method of it. The source data will be streamed into a\n"
 "decompressor. As decompressed data is available, it will be exposed to the\n"
 "returned iterator.\n"
 "\n"
 "Data is ``read()`` in chunks of size ``read_size`` and exposed to the\n"
 "iterator in chunks of size ``write_size``. The default values are the input\n"
 "and output sizes for a zstd streaming decompressor.\n"
 "\n"
 "There is also support for skipping the first ``skip_bytes`` of data from\n"
 "the source.\n"
 );
 
 static ZstdDecompressorIterator* Decompressor_read_to_iter(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"reader",
 		"read_size",
 		"write_size",
 		"skip_bytes",
 		NULL
 	};
 
 	PyObject* reader;
 	size_t inSize = ZSTD_DStreamInSize();
 	size_t outSize = ZSTD_DStreamOutSize();
 	ZstdDecompressorIterator* result;
 	size_t skipBytes = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk:read_to_iter", kwlist,
 		&reader, &inSize, &outSize, &skipBytes)) {
 		return NULL;
 	}
 
 	if (skipBytes >= inSize) {
 		PyErr_SetString(PyExc_ValueError,
 			"skip_bytes must be smaller than read_size");
 		return NULL;
 	}
 
 	result = (ZstdDecompressorIterator*)PyObject_CallObject((PyObject*)&ZstdDecompressorIteratorType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	if (PyObject_HasAttrString(reader, "read")) {
 		result->reader = reader;
 		Py_INCREF(result->reader);
 	}
 	else if (1 == PyObject_CheckBuffer(reader)) {
 		/* Object claims it is a buffer. Try to get a handle to it. */
 		if (0 != PyObject_GetBuffer(reader, &result->buffer, PyBUF_CONTIG_RO)) {
 			goto except;
 		}
 	}
 	else {
 		PyErr_SetString(PyExc_ValueError,
 			"must pass an object with a read() method or conforms to buffer protocol");
 		goto except;
 	}
 
 	result->decompressor = self;
 	Py_INCREF(result->decompressor);
 
 	result->inSize = inSize;
 	result->outSize = outSize;
 	result->skipBytes = skipBytes;
 
 	if (ensure_dctx(self, 1)) {
 		goto except;
 	}
 
 	result->input.src = PyMem_Malloc(inSize);
 	if (!result->input.src) {
 		PyErr_NoMemory();
 		goto except;
 	}
 
 	goto finally;
 
 except:
 	Py_CLEAR(result);
 
 finally:
 
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_stream_reader__doc__,
 "stream_reader(source, [read_size=default, [read_across_frames=False]])\n"
 "\n"
 "Obtain an object that behaves like an I/O stream that can be used for\n"
 "reading decompressed output from an object.\n"
 "\n"
 "The source object can be any object with a ``read(size)`` method or that\n"
 "conforms to the buffer protocol.\n"
 "\n"
 "``read_across_frames`` controls the behavior of ``read()`` when the end\n"
 "of a zstd frame is reached. When ``True``, ``read()`` can potentially\n"
 "return data belonging to multiple zstd frames. When ``False``, ``read()``\n"
 "will return when the end of a frame is reached.\n"
 );
 
 static ZstdDecompressionReader* Decompressor_stream_reader(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"source",
 		"read_size",
 		"read_across_frames",
 		NULL
 	};
 
 	PyObject* source;
 	size_t readSize = ZSTD_DStreamInSize();
 	PyObject* readAcrossFrames = NULL;
 	ZstdDecompressionReader* result;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kO:stream_reader", kwlist,
 		&source, &readSize, &readAcrossFrames)) {
 		return NULL;
 	}
 
 	if (ensure_dctx(self, 1)) {
 		return NULL;
 	}
 
 	result = (ZstdDecompressionReader*)PyObject_CallObject((PyObject*)&ZstdDecompressionReaderType, NULL);
 	if (NULL == result) {
 		return NULL;
 	}
 
 	if (PyObject_HasAttrString(source, "read")) {
 		result->reader = source;
 		Py_INCREF(source);
 		result->readSize = readSize;
 	}
 	else if (1 == PyObject_CheckBuffer(source)) {
 		if (0 != PyObject_GetBuffer(source, &result->buffer, PyBUF_CONTIG_RO)) {
 			Py_CLEAR(result);
 			return NULL;
 		}
 	}
 	else {
 		PyErr_SetString(PyExc_TypeError,
 			"must pass an object with a read() method or that conforms to the buffer protocol");
 		Py_CLEAR(result);
 		return NULL;
 	}
 
 	result->decompressor = self;
 	Py_INCREF(self);
 	result->readAcrossFrames = readAcrossFrames ? PyObject_IsTrue(readAcrossFrames) : 0;
 
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_stream_writer__doc__,
 "Create a context manager to write decompressed data to an object.\n"
 "\n"
 "The passed object must have a ``write()`` method.\n"
 "\n"
 "The caller feeds intput data to the object by calling ``write(data)``.\n"
 "Decompressed data is written to the argument given as it is decompressed.\n"
 "\n"
 "An optional ``write_size`` argument defines the size of chunks to\n"
 "``write()`` to the writer. It defaults to the default output size for a zstd\n"
 "streaming decompressor.\n"
 );
 
 static ZstdDecompressionWriter* Decompressor_stream_writer(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"writer",
 		"write_size",
 		"write_return_read",
 		NULL
 	};
 
 	PyObject* writer;
 	size_t outSize = ZSTD_DStreamOutSize();
 	PyObject* writeReturnRead = NULL;
 	ZstdDecompressionWriter* result;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kO:stream_writer", kwlist,
 		&writer, &outSize, &writeReturnRead)) {
 		return NULL;
 	}
 
 	if (!PyObject_HasAttrString(writer, "write")) {
 		PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method");
 		return NULL;
 	}
 
 	if (ensure_dctx(self, 1)) {
 		return NULL;
 	}
 
 	result = (ZstdDecompressionWriter*)PyObject_CallObject((PyObject*)&ZstdDecompressionWriterType, NULL);
 	if (!result) {
 		return NULL;
 	}
 
 	result->decompressor = self;
 	Py_INCREF(result->decompressor);
 
 	result->writer = writer;
 	Py_INCREF(result->writer);
 
 	result->outSize = outSize;
 	result->writeReturnRead = writeReturnRead ? PyObject_IsTrue(writeReturnRead) : 0;
 
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_decompress_content_dict_chain__doc__,
 "Decompress a series of chunks using the content dictionary chaining technique\n"
 );
 
 static PyObject* Decompressor_decompress_content_dict_chain(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"frames",
 		NULL
 	};
 
 	PyObject* chunks;
 	Py_ssize_t chunksLen;
 	Py_ssize_t chunkIndex;
 	char parity = 0;
 	PyObject* chunk;
 	char* chunkData;
 	Py_ssize_t chunkSize;
 	size_t zresult;
 	ZSTD_frameHeader frameHeader;
 	void* buffer1 = NULL;
 	size_t buffer1Size = 0;
 	size_t buffer1ContentSize = 0;
 	void* buffer2 = NULL;
 	size_t buffer2Size = 0;
 	size_t buffer2ContentSize = 0;
 	void* destBuffer = NULL;
 	PyObject* result = NULL;
 	ZSTD_outBuffer outBuffer;
 	ZSTD_inBuffer inBuffer;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!:decompress_content_dict_chain",
 		kwlist, &PyList_Type, &chunks)) {
 		return NULL;
 	}
 
 	chunksLen = PyList_Size(chunks);
 	if (!chunksLen) {
 		PyErr_SetString(PyExc_ValueError, "empty input chain");
 		return NULL;
 	}
 
 	/* The first chunk should not be using a dictionary. We handle it specially. */
 	chunk = PyList_GetItem(chunks, 0);
 	if (!PyBytes_Check(chunk)) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 must be bytes");
 		return NULL;
 	}
 
 	/* We require that all chunks be zstd frames and that they have content size set. */
 	PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
 	zresult = ZSTD_getFrameHeader(&frameHeader, (void*)chunkData, chunkSize);
 	if (ZSTD_isError(zresult)) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 is not a valid zstd frame");
 		return NULL;
 	}
 	else if (zresult) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 is too small to contain a zstd frame");
 		return NULL;
 	}
 
 	if (ZSTD_CONTENTSIZE_UNKNOWN == frameHeader.frameContentSize) {
 		PyErr_SetString(PyExc_ValueError, "chunk 0 missing content size in frame");
 		return NULL;
 	}
 
 	assert(ZSTD_CONTENTSIZE_ERROR != frameHeader.frameContentSize);
 
 	/* We check against PY_SSIZE_T_MAX here because we ultimately cast the
 	 * result to a Python object and it's length can be no greater than
 	 * Py_ssize_t. In theory, we could have an intermediate frame that is 
 	 * larger. But a) why would this API be used for frames that large b)
 	 * it isn't worth the complexity to support. */
 	assert(SIZE_MAX >= PY_SSIZE_T_MAX);
 	if (frameHeader.frameContentSize > PY_SSIZE_T_MAX) {
 		PyErr_SetString(PyExc_ValueError,
 			"chunk 0 is too large to decompress on this platform");
 		return NULL;
 	}
 
 	if (ensure_dctx(self, 0)) {
 		goto finally;
 	}
 
 	buffer1Size = (size_t)frameHeader.frameContentSize;
 	buffer1 = PyMem_Malloc(buffer1Size);
 	if (!buffer1) {
 		goto finally;
 	}
 
 	outBuffer.dst = buffer1;
 	outBuffer.size = buffer1Size;
 	outBuffer.pos = 0;
 
 	inBuffer.src = chunkData;
 	inBuffer.size = chunkSize;
 	inBuffer.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 	Py_END_ALLOW_THREADS
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "could not decompress chunk 0: %s", ZSTD_getErrorName(zresult));
 		goto finally;
 	}
 	else if (zresult) {
 		PyErr_Format(ZstdError, "chunk 0 did not decompress full frame");
 		goto finally;
 	}
 
 	buffer1ContentSize = outBuffer.pos;
 
 	/* Special case of a simple chain. */
 	if (1 == chunksLen) {
 		result = PyBytes_FromStringAndSize(buffer1, buffer1Size);
 		goto finally;
 	}
 
 	/* This should ideally look at next chunk. But this is slightly simpler. */
 	buffer2Size = (size_t)frameHeader.frameContentSize;
 	buffer2 = PyMem_Malloc(buffer2Size);
 	if (!buffer2) {
 		goto finally;
 	}
 
 	/* For each subsequent chunk, use the previous fulltext as a content dictionary.
 	   Our strategy is to have 2 buffers. One holds the previous fulltext (to be
 	   used as a content dictionary) and the other holds the new fulltext. The
 	   buffers grow when needed but never decrease in size. This limits the
 	   memory allocator overhead.
 	*/
 	for (chunkIndex = 1; chunkIndex < chunksLen; chunkIndex++) {
 		chunk = PyList_GetItem(chunks, chunkIndex);
 		if (!PyBytes_Check(chunk)) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd must be bytes", chunkIndex);
 			goto finally;
 		}
 
 		PyBytes_AsStringAndSize(chunk, &chunkData, &chunkSize);
 		zresult = ZSTD_getFrameHeader(&frameHeader, (void*)chunkData, chunkSize);
 		if (ZSTD_isError(zresult)) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd is not a valid zstd frame", chunkIndex);
 			goto finally;
 		}
 		else if (zresult) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd is too small to contain a zstd frame", chunkIndex);
 			goto finally;
 		}
 
 		if (ZSTD_CONTENTSIZE_UNKNOWN == frameHeader.frameContentSize) {
 			PyErr_Format(PyExc_ValueError, "chunk %zd missing content size in frame", chunkIndex);
 			goto finally;
 		}
 
 		assert(ZSTD_CONTENTSIZE_ERROR != frameHeader.frameContentSize);
 
 		if (frameHeader.frameContentSize > PY_SSIZE_T_MAX) {
 			PyErr_Format(PyExc_ValueError,
 				"chunk %zd is too large to decompress on this platform", chunkIndex);
 			goto finally;
 		}
 
 		inBuffer.src = chunkData;
 		inBuffer.size = chunkSize;
 		inBuffer.pos = 0;
 
 		parity = chunkIndex % 2;
 
 		/* This could definitely be abstracted to reduce code duplication. */
 		if (parity) {
 			/* Resize destination buffer to hold larger content. */
 			if (buffer2Size < frameHeader.frameContentSize) {
 				buffer2Size = (size_t)frameHeader.frameContentSize;
 				destBuffer = PyMem_Realloc(buffer2, buffer2Size);
 				if (!destBuffer) {
 					goto finally;
 				}
 				buffer2 = destBuffer;
 			}
 
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_DCtx_refPrefix_advanced(self->dctx,
 				buffer1, buffer1ContentSize, ZSTD_dct_rawContent);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError,
 					"failed to load prefix dictionary at chunk %zd", chunkIndex);
 				goto finally;
 			}
 
 			outBuffer.dst = buffer2;
 			outBuffer.size = buffer2Size;
 			outBuffer.pos = 0;
 
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
 					chunkIndex, ZSTD_getErrorName(zresult));
 				goto finally;
 			}
 			else if (zresult) {
 				PyErr_Format(ZstdError, "chunk %zd did not decompress full frame",
 					chunkIndex);
 				goto finally;
 			}
 
 			buffer2ContentSize = outBuffer.pos;
 		}
 		else {
 			if (buffer1Size < frameHeader.frameContentSize) {
 				buffer1Size = (size_t)frameHeader.frameContentSize;
 				destBuffer = PyMem_Realloc(buffer1, buffer1Size);
 				if (!destBuffer) {
 					goto finally;
 				}
 				buffer1 = destBuffer;
 			}
 
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_DCtx_refPrefix_advanced(self->dctx,
 				buffer2, buffer2ContentSize, ZSTD_dct_rawContent);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError,
 					"failed to load prefix dictionary at chunk %zd", chunkIndex);
 				goto finally;
 			}
 
 			outBuffer.dst = buffer1;
 			outBuffer.size = buffer1Size;
 			outBuffer.pos = 0;
 
 			Py_BEGIN_ALLOW_THREADS
 			zresult = ZSTD_decompressStream(self->dctx, &outBuffer, &inBuffer);
 			Py_END_ALLOW_THREADS
 			if (ZSTD_isError(zresult)) {
 				PyErr_Format(ZstdError, "could not decompress chunk %zd: %s",
 					chunkIndex, ZSTD_getErrorName(zresult));
 				goto finally;
 			}
 			else if (zresult) {
 				PyErr_Format(ZstdError, "chunk %zd did not decompress full frame",
 					chunkIndex);
 				goto finally;
 			}
 
 			buffer1ContentSize = outBuffer.pos;
 		}
 	}
 
 	result = PyBytes_FromStringAndSize(parity ? buffer2 : buffer1,
 		parity ? buffer2ContentSize : buffer1ContentSize);
 
 finally:
 	if (buffer2) {
 		PyMem_Free(buffer2);
 	}
 	if (buffer1) {
 		PyMem_Free(buffer1);
 	}
 
 	return result;
 }
 
 typedef struct {
 	void* sourceData;
 	size_t sourceSize;
 	size_t destSize;
 } FramePointer;
 
 typedef struct {
 	FramePointer* frames;
 	Py_ssize_t framesSize;
 	unsigned long long compressedSize;
 } FrameSources;
 
 typedef struct {
 	void* dest;
 	Py_ssize_t destSize;
 	BufferSegment* segments;
 	Py_ssize_t segmentsSize;
 } DestBuffer;
 
 typedef enum {
 	WorkerError_none = 0,
 	WorkerError_zstd = 1,
 	WorkerError_memory = 2,
 	WorkerError_sizeMismatch = 3,
 	WorkerError_unknownSize = 4,
 } WorkerError;
 
 typedef struct {
 	/* Source records and length */
 	FramePointer* framePointers;
 	/* Which records to process. */
 	Py_ssize_t startOffset;
 	Py_ssize_t endOffset;
 	unsigned long long totalSourceSize;
 
 	/* Compression state and settings. */
 	ZSTD_DCtx* dctx;
 	int requireOutputSizes;
 
 	/* Output storage. */
 	DestBuffer* destBuffers;
 	Py_ssize_t destCount;
 
 	/* Item that error occurred on. */
 	Py_ssize_t errorOffset;
 	/* If an error occurred. */
 	WorkerError error;
 	/* result from zstd decompression operation */
 	size_t zresult;
 } WorkerState;
 
 static void decompress_worker(WorkerState* state) {
 	size_t allocationSize;
 	DestBuffer* destBuffer;
 	Py_ssize_t frameIndex;
 	Py_ssize_t localOffset = 0;
 	Py_ssize_t currentBufferStartIndex = state->startOffset;
 	Py_ssize_t remainingItems = state->endOffset - state->startOffset + 1;
 	void* tmpBuf;
 	Py_ssize_t destOffset = 0;
 	FramePointer* framePointers = state->framePointers;
 	size_t zresult;
 	unsigned long long totalOutputSize = 0;
 
 	assert(NULL == state->destBuffers);
 	assert(0 == state->destCount);
 	assert(state->endOffset - state->startOffset >= 0);
 
 	/* We could get here due to the way work is allocated. Ideally we wouldn't
 	   get here. But that would require a bit of a refactor in the caller. */
 	if (state->totalSourceSize > SIZE_MAX) {
 		state->error = WorkerError_memory;
 		state->errorOffset = 0;
 		return;
 	}
 
 	/*
 	 * We need to allocate a buffer to hold decompressed data. How we do this
 	 * depends on what we know about the output. The following scenarios are
 	 * possible:
 	 *
 	 * 1. All structs defining frames declare the output size.
 	 * 2. The decompressed size is embedded within the zstd frame.
 	 * 3. The decompressed size is not stored anywhere.
 	 *
 	 * For now, we only support #1 and #2.
 	 */
 
 	/* Resolve ouput segments. */
 	for (frameIndex = state->startOffset; frameIndex <= state->endOffset; frameIndex++) {
 		FramePointer* fp = &framePointers[frameIndex];
 		unsigned long long decompressedSize;
 
 		if (0 == fp->destSize) {
 			decompressedSize = ZSTD_getFrameContentSize(fp->sourceData, fp->sourceSize);
 
 			if (ZSTD_CONTENTSIZE_ERROR == decompressedSize) {
 				state->error = WorkerError_unknownSize;
 				state->errorOffset = frameIndex;
 				return;
 			}
 			else if (ZSTD_CONTENTSIZE_UNKNOWN == decompressedSize) {
 				if (state->requireOutputSizes) {
 					state->error = WorkerError_unknownSize;
 					state->errorOffset = frameIndex;
 					return;
 				}
 
 				/* This will fail the assert for .destSize > 0 below. */
 				decompressedSize = 0;
 			}
 
 			if (decompressedSize > SIZE_MAX) {
 				state->error = WorkerError_memory;
 				state->errorOffset = frameIndex;
 				return;
 			}
 
 			fp->destSize = (size_t)decompressedSize;
 		}
 
 		totalOutputSize += fp->destSize;
 	}
 
 	state->destBuffers = calloc(1, sizeof(DestBuffer));
 	if (NULL == state->destBuffers) {
 		state->error = WorkerError_memory;
 		return;
 	}
 
 	state->destCount = 1;
 
 	destBuffer = &state->destBuffers[state->destCount - 1];
 
 	assert(framePointers[state->startOffset].destSize > 0); /* For now. */
 
 	allocationSize = roundpow2((size_t)state->totalSourceSize);
 
 	if (framePointers[state->startOffset].destSize > allocationSize) {
 		allocationSize = roundpow2(framePointers[state->startOffset].destSize);
 	}
 
 	destBuffer->dest = malloc(allocationSize);
 	if (NULL == destBuffer->dest) {
 		state->error = WorkerError_memory;
 		return;
 	}
 
 	destBuffer->destSize = allocationSize;
 
 	destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
 	if (NULL == destBuffer->segments) {
 		/* Caller will free state->dest as part of cleanup. */
 		state->error = WorkerError_memory;
 		return;
 	}
 
 	destBuffer->segmentsSize = remainingItems;
 
 	for (frameIndex = state->startOffset; frameIndex <= state->endOffset; frameIndex++) {
 		ZSTD_outBuffer outBuffer;
 		ZSTD_inBuffer inBuffer;
 		const void* source = framePointers[frameIndex].sourceData;
 		const size_t sourceSize = framePointers[frameIndex].sourceSize;
 		void* dest;
 		const size_t decompressedSize = framePointers[frameIndex].destSize;
 		size_t destAvailable = destBuffer->destSize - destOffset;
 
 		assert(decompressedSize > 0); /* For now. */
 
 		/*
 		 * Not enough space in current buffer. Finish current before and allocate and
 		 * switch to a new one.
 		 */
 		if (decompressedSize > destAvailable) {
 			/*
 			 * Shrinking the destination buffer is optional. But it should be cheap,
 			 * so we just do it.
 			 */
 			if (destAvailable) {
 				tmpBuf = realloc(destBuffer->dest, destOffset);
 				if (NULL == tmpBuf) {
 					state->error = WorkerError_memory;
 					return;
 				}
 
 				destBuffer->dest = tmpBuf;
 				destBuffer->destSize = destOffset;
 			}
 
 			/* Truncate segments buffer. */
 			tmpBuf = realloc(destBuffer->segments,
 				(frameIndex - currentBufferStartIndex) * sizeof(BufferSegment));
 			if (NULL == tmpBuf) {
 				state->error = WorkerError_memory;
 				return;
 			}
 
 			destBuffer->segments = tmpBuf;
 			destBuffer->segmentsSize = frameIndex - currentBufferStartIndex;
 
 			/* Grow space for new DestBuffer. */
 			tmpBuf = realloc(state->destBuffers, (state->destCount + 1) * sizeof(DestBuffer));
 			if (NULL == tmpBuf) {
 				state->error = WorkerError_memory;
 				return;
 			}
 
 			state->destBuffers = tmpBuf;
 			state->destCount++;
 
 			destBuffer = &state->destBuffers[state->destCount - 1];
 
 			/* Don't take any chances will non-NULL pointers. */
 			memset(destBuffer, 0, sizeof(DestBuffer));
 
 			allocationSize = roundpow2((size_t)state->totalSourceSize);
 
 			if (decompressedSize > allocationSize) {
 				allocationSize = roundpow2(decompressedSize);
 			}
 
 			destBuffer->dest = malloc(allocationSize);
 			if (NULL == destBuffer->dest) {
 				state->error = WorkerError_memory;
 				return;
 			}
 
 			destBuffer->destSize = allocationSize;
 			destAvailable = allocationSize;
 			destOffset = 0;
 			localOffset = 0;
 
 			destBuffer->segments = calloc(remainingItems, sizeof(BufferSegment));
 			if (NULL == destBuffer->segments) {
 				state->error = WorkerError_memory;
 				return;
 			}
 
 			destBuffer->segmentsSize = remainingItems;
 			currentBufferStartIndex = frameIndex;
 		}
 
 		dest = (char*)destBuffer->dest + destOffset;
 
 		outBuffer.dst = dest;
 		outBuffer.size = decompressedSize;
 		outBuffer.pos = 0;
 
 		inBuffer.src = source;
 		inBuffer.size = sourceSize;
 		inBuffer.pos = 0;
 
 		zresult = ZSTD_decompressStream(state->dctx, &outBuffer, &inBuffer);
 		if (ZSTD_isError(zresult)) {
 			state->error = WorkerError_zstd;
 			state->zresult = zresult;
 			state->errorOffset = frameIndex;
 			return;
 		}
 		else if (zresult || outBuffer.pos != decompressedSize) {
 			state->error = WorkerError_sizeMismatch;
 			state->zresult = outBuffer.pos;
 			state->errorOffset = frameIndex;
 			return;
 		}
 
 		destBuffer->segments[localOffset].offset = destOffset;
 		destBuffer->segments[localOffset].length = outBuffer.pos;
 		destOffset += outBuffer.pos;
 		localOffset++;
 		remainingItems--;
 	}
 
 	if (destBuffer->destSize > destOffset) {
 		tmpBuf = realloc(destBuffer->dest, destOffset);
 		if (NULL == tmpBuf) {
 			state->error = WorkerError_memory;
 			return;
 		}
 
 		destBuffer->dest = tmpBuf;
 		destBuffer->destSize = destOffset;
 	}
 }
 
 ZstdBufferWithSegmentsCollection* decompress_from_framesources(ZstdDecompressor* decompressor, FrameSources* frames,
 	Py_ssize_t threadCount) {
 	Py_ssize_t i = 0;
 	int errored = 0;
 	Py_ssize_t segmentsCount;
 	ZstdBufferWithSegments* bws = NULL;
 	PyObject* resultArg = NULL;
 	Py_ssize_t resultIndex;
 	ZstdBufferWithSegmentsCollection* result = NULL;
 	FramePointer* framePointers = frames->frames;
 	unsigned long long workerBytes = 0;
 	Py_ssize_t currentThread = 0;
 	Py_ssize_t workerStartOffset = 0;
 	POOL_ctx* pool = NULL;
 	WorkerState* workerStates = NULL;
 	unsigned long long bytesPerWorker;
 
 	/* Caller should normalize 0 and negative values to 1 or larger. */
 	assert(threadCount >= 1);
 
 	/* More threads than inputs makes no sense under any conditions. */
 	threadCount = frames->framesSize < threadCount ? frames->framesSize
 												   : threadCount;
 
 	/* TODO lower thread count if input size is too small and threads would just
 	   add overhead. */
 
 	if (decompressor->dict) {
 		if (ensure_ddict(decompressor->dict)) {
 			return NULL;
 		}
 	}
 
 	/* If threadCount==1, we don't start a thread pool. But we do leverage the
 	   same API for dispatching work. */
 	workerStates = PyMem_Malloc(threadCount * sizeof(WorkerState));
 	if (NULL == workerStates) {
 		PyErr_NoMemory();
 		goto finally;
 	}
 
 	memset(workerStates, 0, threadCount * sizeof(WorkerState));
 
 	if (threadCount > 1) {
 		pool = POOL_create(threadCount, 1);
 		if (NULL == pool) {
 			PyErr_SetString(ZstdError, "could not initialize zstd thread pool");
 			goto finally;
 		}
 	}
 
 	bytesPerWorker = frames->compressedSize / threadCount;
 
 	if (bytesPerWorker > SIZE_MAX) {
 		PyErr_SetString(ZstdError, "too much data per worker for this platform");
 		goto finally;
 	}
 
 	for (i = 0; i < threadCount; i++) {
 		size_t zresult;
 
 		workerStates[i].dctx = ZSTD_createDCtx();
 		if (NULL == workerStates[i].dctx) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		ZSTD_copyDCtx(workerStates[i].dctx, decompressor->dctx);
 
 		if (decompressor->dict) {
 			zresult = ZSTD_DCtx_refDDict(workerStates[i].dctx, decompressor->dict->ddict);
 			if (zresult) {
 				PyErr_Format(ZstdError, "unable to reference prepared dictionary: %s",
 					ZSTD_getErrorName(zresult));
 				goto finally;
 			}
 		}
 
 		workerStates[i].framePointers = framePointers;
 		workerStates[i].requireOutputSizes = 1;
 	}
 
 	Py_BEGIN_ALLOW_THREADS
 	/* There are many ways to split work among workers.
 
 	   For now, we take a simple approach of splitting work so each worker
 	   gets roughly the same number of input bytes. This will result in more
 	   starvation than running N>threadCount jobs. But it avoids complications
 	   around state tracking, which could involve extra locking.
 	*/
 	for (i = 0; i < frames->framesSize; i++) {
 		workerBytes += frames->frames[i].sourceSize;
 
 		/*
 		 * The last worker/thread needs to handle all remaining work. Don't
 		 * trigger it prematurely. Defer to the block outside of the loop.
 		 * (But still process this loop so workerBytes is correct.
 		 */
 		if (currentThread == threadCount - 1) {
 			continue;
 		}
 
 		if (workerBytes >= bytesPerWorker) {
 			workerStates[currentThread].startOffset = workerStartOffset;
 			workerStates[currentThread].endOffset = i;
 			workerStates[currentThread].totalSourceSize = workerBytes;
 
 			if (threadCount > 1) {
 				POOL_add(pool, (POOL_function)decompress_worker, &workerStates[currentThread]);
 			}
 			else {
 				decompress_worker(&workerStates[currentThread]);
 			}
 			currentThread++;
 			workerStartOffset = i + 1;
 			workerBytes = 0;
 		}
 	}
 
 	if (workerBytes) {
 		workerStates[currentThread].startOffset = workerStartOffset;
 		workerStates[currentThread].endOffset = frames->framesSize - 1;
 		workerStates[currentThread].totalSourceSize = workerBytes;
 
 		if (threadCount > 1) {
 			POOL_add(pool, (POOL_function)decompress_worker, &workerStates[currentThread]);
 		}
 		else {
 			decompress_worker(&workerStates[currentThread]);
 		}
 	}
 
 	if (threadCount > 1) {
 		POOL_free(pool);
 		pool = NULL;
 	}
 	Py_END_ALLOW_THREADS
 
 	for (i = 0; i < threadCount; i++) {
 		switch (workerStates[i].error) {
 		case WorkerError_none:
 			break;
 
 		case WorkerError_zstd:
 			PyErr_Format(ZstdError, "error decompressing item %zd: %s",
 				workerStates[i].errorOffset, ZSTD_getErrorName(workerStates[i].zresult));
 			errored = 1;
 			break;
 
 		case WorkerError_memory:
 			PyErr_NoMemory();
 			errored = 1;
 			break;
 
 		case WorkerError_sizeMismatch:
 			PyErr_Format(ZstdError, "error decompressing item %zd: decompressed %zu bytes; expected %zu",
 				workerStates[i].errorOffset, workerStates[i].zresult,
 				framePointers[workerStates[i].errorOffset].destSize);
 			errored = 1;
 			break;
 
 		case WorkerError_unknownSize:
 			PyErr_Format(PyExc_ValueError, "could not determine decompressed size of item %zd",
 				workerStates[i].errorOffset);
 			errored = 1;
 			break;
 
 		default:
 			PyErr_Format(ZstdError, "unhandled error type: %d; this is a bug",
 				workerStates[i].error);
 			errored = 1;
 			break;
 		}
 
 		if (errored) {
 			break;
 		}
 	}
 
 	if (errored) {
 		goto finally;
 	}
 
 	segmentsCount = 0;
 	for (i = 0; i < threadCount; i++) {
 		segmentsCount += workerStates[i].destCount;
 	}
 
 	resultArg = PyTuple_New(segmentsCount);
 	if (NULL == resultArg) {
 		goto finally;
 	}
 
 	resultIndex = 0;
 
 	for (i = 0; i < threadCount; i++) {
 		Py_ssize_t bufferIndex;
 		WorkerState* state = &workerStates[i];
 
 		for (bufferIndex = 0; bufferIndex < state->destCount; bufferIndex++) {
 			DestBuffer* destBuffer = &state->destBuffers[bufferIndex];
 
 			bws = BufferWithSegments_FromMemory(destBuffer->dest, destBuffer->destSize,
 				destBuffer->segments, destBuffer->segmentsSize);
 			if (NULL == bws) {
 				goto finally;
 			}
 
 			/*
 			* Memory for buffer and segments was allocated using malloc() in worker
 			* and the memory is transferred to the BufferWithSegments instance. So
 			* tell instance to use free() and NULL the reference in the state struct
 			* so it isn't freed below.
 			*/
 			bws->useFree = 1;
 			destBuffer->dest = NULL;
 			destBuffer->segments = NULL;
 
 			PyTuple_SET_ITEM(resultArg, resultIndex++, (PyObject*)bws);
 		}
 	}
 
 	result = (ZstdBufferWithSegmentsCollection*)PyObject_CallObject(
 		(PyObject*)&ZstdBufferWithSegmentsCollectionType, resultArg);
 
 finally:
 	Py_CLEAR(resultArg);
 
 	if (workerStates) {
 		for (i = 0; i < threadCount; i++) {
 			Py_ssize_t bufferIndex;
 			WorkerState* state = &workerStates[i];
 
 			if (state->dctx) {
 				ZSTD_freeDCtx(state->dctx);
 			}
 
 			for (bufferIndex = 0; bufferIndex < state->destCount; bufferIndex++) {
 				if (state->destBuffers) {
 					/*
 					* Will be NULL if memory transfered to a BufferWithSegments.
 					* Otherwise it is left over after an error occurred.
 					*/
 					free(state->destBuffers[bufferIndex].dest);
 					free(state->destBuffers[bufferIndex].segments);
 				}
 			}
 
 			free(state->destBuffers);
 		}
 
 		PyMem_Free(workerStates);
 	}
 
 	POOL_free(pool);
 
 	return result;
 }
 
 PyDoc_STRVAR(Decompressor_multi_decompress_to_buffer__doc__,
 "Decompress multiple frames to output buffers\n"
 "\n"
 "Receives a ``BufferWithSegments``, a ``BufferWithSegmentsCollection`` or a\n"
 "list of bytes-like objects. Each item in the passed collection should be a\n"
 "compressed zstd frame.\n"
 "\n"
 "Unless ``decompressed_sizes`` is specified, the content size *must* be\n"
 "written into the zstd frame header. If ``decompressed_sizes`` is specified,\n"
 "it is an object conforming to the buffer protocol that represents an array\n"
 "of 64-bit unsigned integers in the machine's native format. Specifying\n"
 "``decompressed_sizes`` avoids a pre-scan of each frame to determine its\n"
 "output size.\n"
 "\n"
 "Returns a ``BufferWithSegmentsCollection`` containing the decompressed\n"
 "data. All decompressed data is allocated in a single memory buffer. The\n"
 "``BufferWithSegments`` instance tracks which objects are at which offsets\n"
 "and their respective lengths.\n"
 "\n"
 "The ``threads`` argument controls how many threads to use for operations.\n"
 "Negative values will use the same number of threads as logical CPUs on the\n"
 "machine.\n"
 );
 
 static ZstdBufferWithSegmentsCollection* Decompressor_multi_decompress_to_buffer(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"frames",
 		"decompressed_sizes",
 		"threads",
 		NULL
 	};
 
 	PyObject* frames;
 	Py_buffer frameSizes;
 	int threads = 0;
 	Py_ssize_t frameCount;
 	Py_buffer* frameBuffers = NULL;
 	FramePointer* framePointers = NULL;
 	unsigned long long* frameSizesP = NULL;
 	unsigned long long totalInputSize = 0;
 	FrameSources frameSources;
 	ZstdBufferWithSegmentsCollection* result = NULL;
 	Py_ssize_t i;
 
 	memset(&frameSizes, 0, sizeof(frameSizes));
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|y*i:multi_decompress_to_buffer",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s*i:multi_decompress_to_buffer",
 #endif
 		kwlist, &frames, &frameSizes, &threads)) {
 		return NULL;
 	}
 
 	if (frameSizes.buf) {
 		if (!PyBuffer_IsContiguous(&frameSizes, 'C') || frameSizes.ndim > 1) {
 			PyErr_SetString(PyExc_ValueError, "decompressed_sizes buffer should be contiguous and have a single dimension");
 			goto finally;
 		}
 
 		frameSizesP = (unsigned long long*)frameSizes.buf;
 	}
 
 	if (threads < 0) {
 		threads = cpu_count();
 	}
 
 	if (threads < 2) {
 		threads = 1;
 	}
 
 	if (PyObject_TypeCheck(frames, &ZstdBufferWithSegmentsType)) {
 		ZstdBufferWithSegments* buffer = (ZstdBufferWithSegments*)frames;
 		frameCount = buffer->segmentCount;
 
 		if (frameSizes.buf && frameSizes.len != frameCount * (Py_ssize_t)sizeof(unsigned long long)) {
 			PyErr_Format(PyExc_ValueError, "decompressed_sizes size mismatch; expected %zd, got %zd",
 				frameCount * sizeof(unsigned long long), frameSizes.len);
 			goto finally;
 		}
 
 		framePointers = PyMem_Malloc(frameCount * sizeof(FramePointer));
 		if (!framePointers) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		for (i = 0; i < frameCount; i++) {
 			void* sourceData;
 			unsigned long long sourceSize;
 			unsigned long long decompressedSize = 0;
 
 			if (buffer->segments[i].offset + buffer->segments[i].length > buffer->dataSize) {
 				PyErr_Format(PyExc_ValueError, "item %zd has offset outside memory area", i);
 				goto finally;
 			}
 
 			sourceData = (char*)buffer->data + buffer->segments[i].offset;
 			sourceSize = buffer->segments[i].length;
 			totalInputSize += sourceSize;
 
 			if (frameSizesP) {
 				decompressedSize = frameSizesP[i];
 			}
 
 			if (sourceSize > SIZE_MAX) {
 				PyErr_Format(PyExc_ValueError,
 					"item %zd is too large for this platform", i);
 				goto finally;
 			}
 
 			if (decompressedSize > SIZE_MAX) {
 				PyErr_Format(PyExc_ValueError,
 					"decompressed size of item %zd is too large for this platform", i);
 				goto finally;
 			}
 
 			framePointers[i].sourceData = sourceData;
 			framePointers[i].sourceSize = (size_t)sourceSize;
 			framePointers[i].destSize = (size_t)decompressedSize;
 		}
 	}
 	else if (PyObject_TypeCheck(frames, &ZstdBufferWithSegmentsCollectionType)) {
 		Py_ssize_t offset = 0;
 		ZstdBufferWithSegments* buffer;
 		ZstdBufferWithSegmentsCollection* collection = (ZstdBufferWithSegmentsCollection*)frames;
 
 		frameCount = BufferWithSegmentsCollection_length(collection);
 
 		if (frameSizes.buf && frameSizes.len != frameCount) {
 			PyErr_Format(PyExc_ValueError,
 				"decompressed_sizes size mismatch; expected %zd; got %zd",
 				frameCount * sizeof(unsigned long long), frameSizes.len);
 			goto finally;
 		}
 
 		framePointers = PyMem_Malloc(frameCount * sizeof(FramePointer));
 		if (NULL == framePointers) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		/* Iterate the data structure directly because it is faster. */
 		for (i = 0; i < collection->bufferCount; i++) {
 			Py_ssize_t segmentIndex;
 			buffer = collection->buffers[i];
 
 			for (segmentIndex = 0; segmentIndex < buffer->segmentCount; segmentIndex++) {
 				unsigned long long decompressedSize = frameSizesP ? frameSizesP[offset] : 0;
 
 				if (buffer->segments[segmentIndex].offset + buffer->segments[segmentIndex].length > buffer->dataSize) {
 					PyErr_Format(PyExc_ValueError, "item %zd has offset outside memory area",
 						offset);
 					goto finally;
 				}
 
 				if (buffer->segments[segmentIndex].length > SIZE_MAX) {
 					PyErr_Format(PyExc_ValueError,
 						"item %zd in buffer %zd is too large for this platform",
 						segmentIndex, i);
 					goto finally;
 				}
 
 				if (decompressedSize > SIZE_MAX) {
 					PyErr_Format(PyExc_ValueError,
 						"decompressed size of item %zd in buffer %zd is too large for this platform",
 						segmentIndex, i);
 					goto finally;
 				}
 
 				totalInputSize += buffer->segments[segmentIndex].length;
 
 				framePointers[offset].sourceData = (char*)buffer->data + buffer->segments[segmentIndex].offset;
 				framePointers[offset].sourceSize = (size_t)buffer->segments[segmentIndex].length;
 				framePointers[offset].destSize = (size_t)decompressedSize;
 
 				offset++;
 			}
 		}
 	}
 	else if (PyList_Check(frames)) {
 		frameCount = PyList_GET_SIZE(frames);
 
 		if (frameSizes.buf && frameSizes.len != frameCount * (Py_ssize_t)sizeof(unsigned long long)) {
 			PyErr_Format(PyExc_ValueError, "decompressed_sizes size mismatch; expected %zd, got %zd",
 				frameCount * sizeof(unsigned long long), frameSizes.len);
 			goto finally;
 		}
 
 		framePointers = PyMem_Malloc(frameCount * sizeof(FramePointer));
 		if (!framePointers) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		frameBuffers = PyMem_Malloc(frameCount * sizeof(Py_buffer));
 		if (NULL == frameBuffers) {
 			PyErr_NoMemory();
 			goto finally;
 		}
 
 		memset(frameBuffers, 0, frameCount * sizeof(Py_buffer));
 
 		/* Do a pass to assemble info about our input buffers and output sizes. */
 		for (i = 0; i < frameCount; i++) {
 			unsigned long long decompressedSize = frameSizesP ? frameSizesP[i] : 0;
 
 			if (0 != PyObject_GetBuffer(PyList_GET_ITEM(frames, i),
 				&frameBuffers[i], PyBUF_CONTIG_RO)) {
 				PyErr_Clear();
 				PyErr_Format(PyExc_TypeError, "item %zd not a bytes like object", i);
 				goto finally;
 			}
 
 			if (decompressedSize > SIZE_MAX) {
 				PyErr_Format(PyExc_ValueError,
 					"decompressed size of item %zd is too large for this platform", i);
 				goto finally;
 			}
 
 			totalInputSize += frameBuffers[i].len;
 
 			framePointers[i].sourceData = frameBuffers[i].buf;
 			framePointers[i].sourceSize = frameBuffers[i].len;
 			framePointers[i].destSize = (size_t)decompressedSize;
 		}
 	}
 	else {
 		PyErr_SetString(PyExc_TypeError, "argument must be list or BufferWithSegments");
 		goto finally;
 	}
 
 	/* We now have an array with info about our inputs and outputs. Feed it into
 	   our generic decompression function. */
 	frameSources.frames = framePointers;
 	frameSources.framesSize = frameCount;
 	frameSources.compressedSize = totalInputSize;
 
 	result = decompress_from_framesources(self, &frameSources, threads);
 
 finally:
 	if (frameSizes.buf) {
 		PyBuffer_Release(&frameSizes);
 	}
 	PyMem_Free(framePointers);
 
 	if (frameBuffers) {
 		for (i = 0; i < frameCount; i++) {
 			PyBuffer_Release(&frameBuffers[i]);
 		}
 
 		PyMem_Free(frameBuffers);
 	}
 
 	return result;
 }
 
 static PyMethodDef Decompressor_methods[] = {
 	{ "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_copy_stream__doc__ },
 	{ "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_decompress__doc__ },
 	{ "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_decompressobj__doc__ },
 	{ "read_to_iter", (PyCFunction)Decompressor_read_to_iter, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_read_to_iter__doc__ },
 	/* TODO Remove deprecated API */
 	{ "read_from", (PyCFunction)Decompressor_read_to_iter, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_read_to_iter__doc__ },
 	{ "stream_reader", (PyCFunction)Decompressor_stream_reader,
 	METH_VARARGS | METH_KEYWORDS, Decompressor_stream_reader__doc__ },
 	{ "stream_writer", (PyCFunction)Decompressor_stream_writer, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_stream_writer__doc__ },
 	/* TODO remove deprecated API */
 	{ "write_to", (PyCFunction)Decompressor_stream_writer, METH_VARARGS | METH_KEYWORDS,
 	Decompressor_stream_writer__doc__ },
 	{ "decompress_content_dict_chain", (PyCFunction)Decompressor_decompress_content_dict_chain,
 	  METH_VARARGS | METH_KEYWORDS, Decompressor_decompress_content_dict_chain__doc__ },
 	{ "multi_decompress_to_buffer", (PyCFunction)Decompressor_multi_decompress_to_buffer,
 	  METH_VARARGS | METH_KEYWORDS, Decompressor_multi_decompress_to_buffer__doc__ },
 	{ "memory_size", (PyCFunction)Decompressor_memory_size, METH_NOARGS,
 	Decompressor_memory_size__doc__ },
 	{ NULL, NULL }
 };
 
 PyTypeObject ZstdDecompressorType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressor",        /* tp_name */
 	sizeof(ZstdDecompressor),       /* tp_basicsize */
 	0,                              /* tp_itemsize */
 	(destructor)Decompressor_dealloc, /* tp_dealloc */
 	0,                              /* tp_print */
 	0,                              /* tp_getattr */
 	0,                              /* tp_setattr */
 	0,                              /* tp_compare */
 	0,                              /* tp_repr */
 	0,                              /* tp_as_number */
 	0,                              /* tp_as_sequence */
 	0,                              /* tp_as_mapping */
 	0,                              /* tp_hash */
 	0,                              /* tp_call */
 	0,                              /* tp_str */
 	0,                              /* tp_getattro */
 	0,                              /* tp_setattro */
 	0,                              /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	Decompressor__doc__,            /* tp_doc */
 	0,                              /* tp_traverse */
 	0,                              /* tp_clear */
 	0,                              /* tp_richcompare */
 	0,                              /* tp_weaklistoffset */
 	0,                              /* tp_iter */
 	0,                              /* tp_iternext */
 	Decompressor_methods,           /* tp_methods */
 	0,                              /* tp_members */
 	0,                              /* tp_getset */
 	0,                              /* tp_base */
 	0,                              /* tp_dict */
 	0,                              /* tp_descr_get */
 	0,                              /* tp_descr_set */
 	0,                              /* tp_dictoffset */
 	(initproc)Decompressor_init,    /* tp_init */
 	0,                              /* tp_alloc */
 	PyType_GenericNew,              /* tp_new */
 };
 
 void decompressor_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdDecompressorType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdDecompressorType, &PyType_Type);
 	if (PyType_Ready(&ZstdDecompressorType) < 0) {
 		return;
 	}
 
 	Py_INCREF((PyObject*)&ZstdDecompressorType);
 	PyModule_AddObject(mod, "ZstdDecompressor",
 		(PyObject*)&ZstdDecompressorType);
 }

contrib/python-zstandard/c-ext/decompressoriterator.c

@@ -1,249 +1,249 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 #define min(a, b) (((a) < (b)) ? (a) : (b))
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(ZstdDecompressorIterator__doc__,
 "Represents an iterator of decompressed data.\n"
 );
 
 static void ZstdDecompressorIterator_dealloc(ZstdDecompressorIterator* self) {
 	Py_XDECREF(self->decompressor);
 	Py_XDECREF(self->reader);
 
 	if (self->buffer.buf) {
 		PyBuffer_Release(&self->buffer);
 		memset(&self->buffer, 0, sizeof(self->buffer));
 	}
 
 	if (self->input.src) {
 		PyMem_Free((void*)self->input.src);
 		self->input.src = NULL;
 	}
 
 	PyObject_Del(self);
 }
 
 static PyObject* ZstdDecompressorIterator_iter(PyObject* self) {
 	Py_INCREF(self);
 	return self;
 }
 
 static DecompressorIteratorResult read_decompressor_iterator(ZstdDecompressorIterator* self) {
 	size_t zresult;
 	PyObject* chunk;
 	DecompressorIteratorResult result;
 	size_t oldInputPos = self->input.pos;
 
 	result.chunk = NULL;
 
 	chunk = PyBytes_FromStringAndSize(NULL, self->outSize);
 	if (!chunk) {
 		result.errored = 1;
 		return result;
 	}
 
 	self->output.dst = PyBytes_AsString(chunk);
 	self->output.size = self->outSize;
 	self->output.pos = 0;
 
 	Py_BEGIN_ALLOW_THREADS
 	zresult = ZSTD_decompressStream(self->decompressor->dctx, &self->output, &self->input);
 	Py_END_ALLOW_THREADS
 
 	/* We're done with the pointer. Nullify to prevent anyone from getting a
 	handle on a Python object. */
 	self->output.dst = NULL;
 
 	if (ZSTD_isError(zresult)) {
 		Py_DECREF(chunk);
 		PyErr_Format(ZstdError, "zstd decompress error: %s",
 			ZSTD_getErrorName(zresult));
 		result.errored = 1;
 		return result;
 	}
 
 	self->readCount += self->input.pos - oldInputPos;
 
 	/* Frame is fully decoded. Input exhausted and output sitting in buffer. */
 	if (0 == zresult) {
 		self->finishedInput = 1;
 		self->finishedOutput = 1;
 	}
 
 	/* If it produced output data, return it. */
 	if (self->output.pos) {
 		if (self->output.pos < self->outSize) {
 			if (safe_pybytes_resize(&chunk, self->output.pos)) {
 				Py_XDECREF(chunk);
 				result.errored = 1;
 				return result;
 			}
 		}
 	}
 	else {
 		Py_DECREF(chunk);
 		chunk = NULL;
 	}
 
 	result.errored = 0;
 	result.chunk = chunk;
 
 	return result;
 }
 
 static PyObject* ZstdDecompressorIterator_iternext(ZstdDecompressorIterator* self) {
 	PyObject* readResult = NULL;
 	char* readBuffer;
 	Py_ssize_t readSize;
 	Py_ssize_t bufferRemaining;
 	DecompressorIteratorResult result;
 
 	if (self->finishedOutput) {
 		PyErr_SetString(PyExc_StopIteration, "output flushed");
 		return NULL;
 	}
 
 	/* If we have data left in the input, consume it. */
 	if (self->input.pos < self->input.size) {
 		result = read_decompressor_iterator(self);
 		if (result.chunk || result.errored) {
 			return result.chunk;
 		}
 
 		/* Else fall through to get more data from input. */
 	}
 
 read_from_source:
 
 	if (!self->finishedInput) {
 		if (self->reader) {
 			readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize);
 			if (!readResult) {
 				return NULL;
 			}
 
 			PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize);
 		}
 		else {
 			assert(self->buffer.buf);
 
 			/* Only support contiguous C arrays for now */
 			assert(self->buffer.strides == NULL && self->buffer.suboffsets == NULL);
 			assert(self->buffer.itemsize == 1);
 
 			/* TODO avoid memcpy() below */
 			readBuffer = (char *)self->buffer.buf + self->bufferOffset;
 			bufferRemaining = self->buffer.len - self->bufferOffset;
 			readSize = min(bufferRemaining, (Py_ssize_t)self->inSize);
 			self->bufferOffset += readSize;
 		}
 
 		if (readSize) {
 			if (!self->readCount && self->skipBytes) {
 				assert(self->skipBytes < self->inSize);
 				if ((Py_ssize_t)self->skipBytes >= readSize) {
 					PyErr_SetString(PyExc_ValueError,
 						"skip_bytes larger than first input chunk; "
 						"this scenario is currently unsupported");
 					Py_XDECREF(readResult);
 					return NULL;
 				}
 
 				readBuffer = readBuffer + self->skipBytes;
 				readSize -= self->skipBytes;
 			}
 
 			/* Copy input into previously allocated buffer because it can live longer
 			than a single function call and we don't want to keep a ref to a Python
 			object around. This could be changed... */
 			memcpy((void*)self->input.src, readBuffer, readSize);
 			self->input.size = readSize;
 			self->input.pos = 0;
 		}
 		/* No bytes on first read must mean an empty input stream. */
 		else if (!self->readCount) {
 			self->finishedInput = 1;
 			self->finishedOutput = 1;
 			Py_XDECREF(readResult);
 			PyErr_SetString(PyExc_StopIteration, "empty input");
 			return NULL;
 		}
 		else {
 			self->finishedInput = 1;
 		}
 
 		/* We've copied the data managed by memory. Discard the Python object. */
 		Py_XDECREF(readResult);
 	}
 
 	result = read_decompressor_iterator(self);
 	if (result.errored || result.chunk) {
 		return result.chunk;
 	}
 
 	/* No new output data. Try again unless we know there is no more data. */
 	if (!self->finishedInput) {
 		goto read_from_source;
 	}
 
 	PyErr_SetString(PyExc_StopIteration, "input exhausted");
 	return NULL;
 }
 
 PyTypeObject ZstdDecompressorIteratorType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"zstd.ZstdDecompressorIterator",   /* tp_name */
 	sizeof(ZstdDecompressorIterator),  /* tp_basicsize */
 	0,                                 /* tp_itemsize */
 	(destructor)ZstdDecompressorIterator_dealloc, /* tp_dealloc */
 	0,                                 /* tp_print */
 	0,                                 /* tp_getattr */
 	0,                                 /* tp_setattr */
 	0,                                 /* tp_compare */
 	0,                                 /* tp_repr */
 	0,                                 /* tp_as_number */
 	0,                                 /* tp_as_sequence */
 	0,                                 /* tp_as_mapping */
 	0,                                 /* tp_hash */
 	0,                                 /* tp_call */
 	0,                                 /* tp_str */
 	0,                                 /* tp_getattro */
 	0,                                 /* tp_setattro */
 	0,                                 /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 	ZstdDecompressorIterator__doc__,   /* tp_doc */
 	0,                                 /* tp_traverse */
 	0,                                 /* tp_clear */
 	0,                                 /* tp_richcompare */
 	0,                                 /* tp_weaklistoffset */
 	ZstdDecompressorIterator_iter,     /* tp_iter */
 	(iternextfunc)ZstdDecompressorIterator_iternext, /* tp_iternext */
 	0,                                 /* tp_methods */
 	0,                                 /* tp_members */
 	0,                                 /* tp_getset */
 	0,                                 /* tp_base */
 	0,                                 /* tp_dict */
 	0,                                 /* tp_descr_get */
 	0,                                 /* tp_descr_set */
 	0,                                 /* tp_dictoffset */
 	0,                                 /* tp_init */
 	0,                                 /* tp_alloc */
 	PyType_GenericNew,                 /* tp_new */
 };
 
 void decompressoriterator_module_init(PyObject* mod) {
-	Py_TYPE(&ZstdDecompressorIteratorType) = &PyType_Type;
+	Py_SET_TYPE(&ZstdDecompressorIteratorType, &PyType_Type);
 	if (PyType_Ready(&ZstdDecompressorIteratorType) < 0) {
 		return;
 	}
 }

contrib/python-zstandard/c-ext/frameparams.c

@@ -1,138 +1,138 b''
 /**
 * Copyright (c) 2017-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #include "python-zstandard.h"
 
 extern PyObject* ZstdError;
 
 PyDoc_STRVAR(FrameParameters__doc__,
 	"FrameParameters: information about a zstd frame");
 
 FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs) {
 	static char* kwlist[] = {
 		"data",
 		NULL
 	};
 
 	Py_buffer source;
 	ZSTD_frameHeader header;
 	FrameParametersObject* result = NULL;
 	size_t zresult;
 
 #if PY_MAJOR_VERSION >= 3
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*:get_frame_parameters",
 #else
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*:get_frame_parameters",
 #endif
 		kwlist, &source)) {
 		return NULL;
 	}
 
 	if (!PyBuffer_IsContiguous(&source, 'C') || source.ndim > 1) {
 		PyErr_SetString(PyExc_ValueError,
 			"data buffer should be contiguous and have at most one dimension");
 		goto finally;
 	}
 
 	zresult = ZSTD_getFrameHeader(&header, source.buf, source.len);
 
 	if (ZSTD_isError(zresult)) {
 		PyErr_Format(ZstdError, "cannot get frame parameters: %s", ZSTD_getErrorName(zresult));
 		goto finally;
 	}
 
 	if (zresult) {
 		PyErr_Format(ZstdError, "not enough data for frame parameters; need %zu bytes", zresult);
 		goto finally;
 	}
 
 	result = PyObject_New(FrameParametersObject, &FrameParametersType);
 	if (!result) {
 		goto finally;
 	}
 
 	result->frameContentSize = header.frameContentSize;
 	result->windowSize = header.windowSize;
 	result->dictID = header.dictID;
 	result->checksumFlag = header.checksumFlag ? 1 : 0;
 
 finally:
 	PyBuffer_Release(&source);
 	return result;
 }
 
 static void FrameParameters_dealloc(PyObject* self) {
 	PyObject_Del(self);
 }
 
 static PyMemberDef FrameParameters_members[] = {
 	{ "content_size", T_ULONGLONG,
 	  offsetof(FrameParametersObject, frameContentSize), READONLY,
 	  "frame content size" },
 	{ "window_size", T_ULONGLONG,
 	  offsetof(FrameParametersObject, windowSize), READONLY,
 	  "window size" },
 	{ "dict_id", T_UINT,
 	  offsetof(FrameParametersObject, dictID), READONLY,
 	  "dictionary ID" },
 	{ "has_checksum", T_BOOL,
 	  offsetof(FrameParametersObject, checksumFlag), READONLY,
 	  "checksum flag" },
 	{ NULL }
 };
 
 PyTypeObject FrameParametersType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"FrameParameters",          /* tp_name */
 	sizeof(FrameParametersObject), /* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)FrameParameters_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	0,                         /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	FrameParameters__doc__,    /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	0,                         /* tp_methods */
 	FrameParameters_members,   /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	0,                         /* tp_init */
 	0,                         /* tp_alloc */
 	0,                         /* tp_new */
 };
 
 void frameparams_module_init(PyObject* mod) {
-	Py_TYPE(&FrameParametersType) = &PyType_Type;
+	Py_SET_TYPE(&FrameParametersType, &PyType_Type);
 	if (PyType_Ready(&FrameParametersType) < 0) {
 		return;
 	}
 
 	Py_INCREF(&FrameParametersType);
 	PyModule_AddObject(mod, "FrameParameters", (PyObject*)&FrameParametersType);
 }

contrib/python-zstandard/c-ext/python-zstandard.h

@@ -1,359 +1,360 b''
 /**
 * Copyright (c) 2016-present, Gregory Szorc
 * All rights reserved.
 *
 * This software may be modified and distributed under the terms
 * of the BSD license. See the LICENSE file for details.
 */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include "structmember.h"
+#include <pythoncapi_compat.h>
 
 #define ZSTD_STATIC_LINKING_ONLY
 #define ZDICT_STATIC_LINKING_ONLY
 #include <zstd.h>
 #include <zdict.h>
 
 /* Remember to change the string in zstandard/__init__ as well */
 #define PYTHON_ZSTANDARD_VERSION "0.13.0"
 
 typedef enum {
 	compressorobj_flush_finish,
 	compressorobj_flush_block,
 } CompressorObj_Flush;
 
 /*
    Represents a ZstdCompressionParameters type.
 
    This type holds all the low-level compression parameters that can be set.
 */
 typedef struct {
 	PyObject_HEAD
 	ZSTD_CCtx_params* params;
 } ZstdCompressionParametersObject;
 
 extern PyTypeObject ZstdCompressionParametersType;
 
 /*
    Represents a FrameParameters type.
 
    This type is basically a wrapper around ZSTD_frameParams.
 */
 typedef struct {
 	PyObject_HEAD
 	unsigned long long frameContentSize;
 	unsigned long long windowSize;
 	unsigned dictID;
 	char checksumFlag;
 } FrameParametersObject;
 
 extern PyTypeObject FrameParametersType;
 
 /*
    Represents a ZstdCompressionDict type.
 
    Instances hold data used for a zstd compression dictionary.
 */
 typedef struct {
 	PyObject_HEAD
 
 	/* Pointer to dictionary data. Owned by self. */
 	void* dictData;
 	/* Size of dictionary data. */
 	size_t dictSize;
 	ZSTD_dictContentType_e dictType;
 	/* k parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned k;
 	/* d parameter for cover dictionaries. Only populated by train_cover_dict(). */
 	unsigned d;
 	/* Digested dictionary, suitable for reuse. */
 	ZSTD_CDict* cdict;
 	ZSTD_DDict* ddict;
 } ZstdCompressionDict;
 
 extern PyTypeObject ZstdCompressionDictType;
 
 /*
    Represents a ZstdCompressor type.
 */
 typedef struct {
 	PyObject_HEAD
 
 	/* Number of threads to use for operations. */
 	unsigned int threads;
 	/* Pointer to compression dictionary to use. NULL if not using dictionary
 	   compression. */
 	ZstdCompressionDict* dict;
 	/* Compression context to use. Populated during object construction. */
 	ZSTD_CCtx* cctx;
 	/* Compression parameters in use. */
 	ZSTD_CCtx_params* params;
 } ZstdCompressor;
 
 extern PyTypeObject ZstdCompressorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	ZSTD_outBuffer output;
 	int finished;
 } ZstdCompressionObj;
 
 extern PyTypeObject ZstdCompressionObjType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* writer;
 	ZSTD_outBuffer output;
 	size_t outSize;
 	int entered;
 	int closed;
 	int writeReturnRead;
 	unsigned long long bytesCompressed;
 } ZstdCompressionWriter;
 
 extern PyTypeObject ZstdCompressionWriterType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
 	size_t inSize;
 	size_t outSize;
 
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	int finishedOutput;
 	int finishedInput;
 	PyObject* readResult;
 } ZstdCompressorIterator;
 
 extern PyTypeObject ZstdCompressorIteratorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	size_t readSize;
 
 	int entered;
 	int closed;
 	unsigned long long bytesCompressed;
 
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	int finishedInput;
 	int finishedOutput;
 	PyObject* readResult;
 } ZstdCompressionReader;
 
 extern PyTypeObject ZstdCompressionReaderType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressor* compressor;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_buffer inBuffer;
 	int finished;
 	size_t chunkSize;
 } ZstdCompressionChunker;
 
 extern PyTypeObject ZstdCompressionChunkerType;
 
 typedef enum {
 	compressionchunker_mode_normal,
 	compressionchunker_mode_flush,
 	compressionchunker_mode_finish,
 } CompressionChunkerMode;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdCompressionChunker* chunker;
 	CompressionChunkerMode mode;
 } ZstdCompressionChunkerIterator;
 
 extern PyTypeObject ZstdCompressionChunkerIteratorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZSTD_DCtx* dctx;
 	ZstdCompressionDict* dict;
 	size_t maxWindowSize;
 	ZSTD_format_e format;
 } ZstdDecompressor;
 
 extern PyTypeObject ZstdDecompressorType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	size_t outSize;
 	int finished;
 } ZstdDecompressionObj;
 
 extern PyTypeObject ZstdDecompressionObjType;
 
 typedef struct {
 	PyObject_HEAD
 
 	/* Parent decompressor to which this object is associated. */
 	ZstdDecompressor* decompressor;
 	/* Object to read() from (if reading from a stream). */
 	PyObject* reader;
 	/* Size for read() operations on reader. */
 	size_t readSize;
 	/* Whether a read() can return data spanning multiple zstd frames. */
 	int readAcrossFrames;
 	/* Buffer to read from (if reading from a buffer). */
 	Py_buffer buffer;
 
 	/* Whether the context manager is active. */
 	int entered;
 	/* Whether we've closed the stream. */
 	int closed;
 
 	/* Number of bytes decompressed and returned to user. */
 	unsigned long long bytesDecompressed;
 
 	/* Tracks data going into decompressor. */
 	ZSTD_inBuffer input;
 
 	/* Holds output from read() operation on reader. */
 	PyObject* readResult;
 
 	/* Whether all input has been sent to the decompressor. */
 	int finishedInput;
 	/* Whether all output has been flushed from the decompressor. */
 	int finishedOutput;
 } ZstdDecompressionReader;
 
 extern PyTypeObject ZstdDecompressionReaderType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	PyObject* writer;
 	size_t outSize;
 	int entered;
 	int closed;
 	int writeReturnRead;
 } ZstdDecompressionWriter;
 
 extern PyTypeObject ZstdDecompressionWriterType;
 
 typedef struct {
 	PyObject_HEAD
 
 	ZstdDecompressor* decompressor;
 	PyObject* reader;
 	Py_buffer buffer;
 	Py_ssize_t bufferOffset;
 	size_t inSize;
 	size_t outSize;
 	size_t skipBytes;
 	ZSTD_inBuffer input;
 	ZSTD_outBuffer output;
 	Py_ssize_t readCount;
 	int finishedInput;
 	int finishedOutput;
 } ZstdDecompressorIterator;
 
 extern PyTypeObject ZstdDecompressorIteratorType;
 
 typedef struct {
 	int errored;
 	PyObject* chunk;
 } DecompressorIteratorResult;
 
 typedef struct {
 	/* The public API is that these are 64-bit unsigned integers. So these can't
 	 * be size_t, even though values larger than SIZE_MAX or PY_SSIZE_T_MAX may 
 	 * be nonsensical for this platform. */
 	unsigned long long offset;
 	unsigned long long length;
 } BufferSegment;
 
 typedef struct {
 	PyObject_HEAD
 
 	PyObject* parent;
 	BufferSegment* segments;
 	Py_ssize_t segmentCount;
 } ZstdBufferSegments;
 
 extern PyTypeObject ZstdBufferSegmentsType;
 
 typedef struct {
 	PyObject_HEAD
 
 	PyObject* parent;
 	void* data;
 	Py_ssize_t dataSize;
 	unsigned long long offset;
 } ZstdBufferSegment;
 
 extern PyTypeObject ZstdBufferSegmentType;
 
 typedef struct {
 	PyObject_HEAD
 
 	Py_buffer parent;
 	void* data;
 	unsigned long long dataSize;
 	BufferSegment* segments;
 	Py_ssize_t segmentCount;
 	int useFree;
 } ZstdBufferWithSegments;
 
 extern PyTypeObject ZstdBufferWithSegmentsType;
 
 /**
  * An ordered collection of BufferWithSegments exposed as a squashed collection.
  *
  * This type provides a virtual view spanning multiple BufferWithSegments
  * instances. It allows multiple instances to be "chained" together and
  * exposed as a single collection. e.g. if there are 2 buffers holding
  * 10 segments each, then o[14] will access the 5th segment in the 2nd buffer.
  */
 typedef struct {
 	PyObject_HEAD
 
 	/* An array of buffers that should be exposed through this instance. */
 	ZstdBufferWithSegments** buffers;
 	/* Number of elements in buffers array. */
 	Py_ssize_t bufferCount;
 	/* Array of first offset in each buffer instance. 0th entry corresponds
 	   to number of elements in the 0th buffer. 1st entry corresponds to the
 	   sum of elements in 0th and 1st buffers. */
 	Py_ssize_t* firstElements;
 } ZstdBufferWithSegmentsCollection;
 
 extern PyTypeObject ZstdBufferWithSegmentsCollectionType;
 
 int set_parameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 int set_parameters(ZSTD_CCtx_params* params, ZstdCompressionParametersObject* obj);
 int to_cparams(ZstdCompressionParametersObject* params, ZSTD_compressionParameters* cparams);
 FrameParametersObject* get_frame_parameters(PyObject* self, PyObject* args, PyObject* kwargs);
 int ensure_ddict(ZstdCompressionDict* dict);
 int ensure_dctx(ZstdDecompressor* decompressor, int loadDict);
 ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs);
 ZstdBufferWithSegments* BufferWithSegments_FromMemory(void* data, unsigned long long dataSize, BufferSegment* segments, Py_ssize_t segmentsSize);
 Py_ssize_t BufferWithSegmentsCollection_length(ZstdBufferWithSegmentsCollection*);
 int cpu_count(void);
 size_t roundpow2(size_t);
 int safe_pybytes_resize(PyObject** obj, Py_ssize_t size);

mercurial/cext/osutil.c

@@ -1,1419 +1,1419 b''
 /*
  osutil.c - native operating system services
 
  Copyright 2007 Matt Mackall and others
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 #define _ATFILE_SOURCE
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #ifdef _WIN32
 #include <io.h>
 #include <windows.h>
 #else
 #include <dirent.h>
 #include <signal.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 #ifdef HAVE_LINUX_STATFS
 #include <linux/magic.h>
 #include <sys/vfs.h>
 #endif
 #ifdef HAVE_BSD_STATFS
 #include <sys/mount.h>
 #include <sys/param.h>
 #endif
 #endif
 
 #ifdef __APPLE__
 #include <sys/attr.h>
 #include <sys/vnode.h>
 #endif
 
 #include "util.h"
 
 /* some platforms lack the PATH_MAX definition (eg. GNU/Hurd) */
 #ifndef PATH_MAX
 #define PATH_MAX 4096
 #endif
 
 #ifdef _WIN32
 /*
 stat struct compatible with hg expectations
 Mercurial only uses st_mode, st_size and st_mtime
 the rest is kept to minimize changes between implementations
 */
 struct hg_stat {
 	int st_dev;
 	int st_mode;
 	int st_nlink;
 	__int64 st_size;
 	int st_mtime;
 	int st_ctime;
 };
 struct listdir_stat {
 	PyObject_HEAD
 	struct hg_stat st;
 };
 #else
 struct listdir_stat {
 	PyObject_HEAD
 	struct stat st;
 };
 #endif
 
 #ifdef IS_PY3K
 #define listdir_slot(name) \
 	static PyObject *listdir_stat_##name(PyObject *self, void *x) \
 	{ \
 		return PyLong_FromLong(((struct listdir_stat *)self)->st.name); \
 	}
 #else
 #define listdir_slot(name) \
 	static PyObject *listdir_stat_##name(PyObject *self, void *x) \
 	{ \
 		return PyInt_FromLong(((struct listdir_stat *)self)->st.name); \
 	}
 #endif
 
 listdir_slot(st_dev)
 listdir_slot(st_mode)
 listdir_slot(st_nlink)
 #ifdef _WIN32
 static PyObject *listdir_stat_st_size(PyObject *self, void *x)
 {
 	return PyLong_FromLongLong(
 		(PY_LONG_LONG)((struct listdir_stat *)self)->st.st_size);
 }
 #else
 listdir_slot(st_size)
 #endif
 listdir_slot(st_mtime)
 listdir_slot(st_ctime)
 
 static struct PyGetSetDef listdir_stat_getsets[] = {
 	{"st_dev", listdir_stat_st_dev, 0, 0, 0},
 	{"st_mode", listdir_stat_st_mode, 0, 0, 0},
 	{"st_nlink", listdir_stat_st_nlink, 0, 0, 0},
 	{"st_size", listdir_stat_st_size, 0, 0, 0},
 	{"st_mtime", listdir_stat_st_mtime, 0, 0, 0},
 	{"st_ctime", listdir_stat_st_ctime, 0, 0, 0},
 	{0, 0, 0, 0, 0}
 };
 
 static PyObject *listdir_stat_new(PyTypeObject *t, PyObject *a, PyObject *k)
 {
 	return t->tp_alloc(t, 0);
 }
 
 static void listdir_stat_dealloc(PyObject *o)
 {
-	o->ob_type->tp_free(o);
+	Py_TYPE(o)->tp_free(o);
 }
 
 static PyObject *listdir_stat_getitem(PyObject *self, PyObject *key)
 {
 	long index = PyLong_AsLong(key);
 	if (index == -1 && PyErr_Occurred()) {
 		return NULL;
 	}
 	if (index != 8) {
 		PyErr_Format(PyExc_IndexError, "osutil.stat objects only "
 		                               "support stat.ST_MTIME in "
 		                               "__getitem__");
 		return NULL;
 	}
 	return listdir_stat_st_mtime(self, NULL);
 }
 
 static PyMappingMethods listdir_stat_type_mapping_methods = {
 	(lenfunc)NULL,             /* mp_length */
 	(binaryfunc)listdir_stat_getitem,       /* mp_subscript */
 	(objobjargproc)NULL,    /* mp_ass_subscript */
 };
 
 static PyTypeObject listdir_stat_type = {
 	PyVarObject_HEAD_INIT(NULL, 0) /* header */
 	"osutil.stat",             /*tp_name*/
 	sizeof(struct listdir_stat), /*tp_basicsize*/
 	0,                         /*tp_itemsize*/
 	(destructor)listdir_stat_dealloc, /*tp_dealloc*/
 	0,                         /*tp_print*/
 	0,                         /*tp_getattr*/
 	0,                         /*tp_setattr*/
 	0,                         /*tp_compare*/
 	0,                         /*tp_repr*/
 	0,                         /*tp_as_number*/
 	0,                         /*tp_as_sequence*/
 	&listdir_stat_type_mapping_methods, /*tp_as_mapping*/
 	0,                         /*tp_hash */
 	0,                         /*tp_call*/
 	0,                         /*tp_str*/
 	0,                         /*tp_getattro*/
 	0,                         /*tp_setattro*/
 	0,                         /*tp_as_buffer*/
 	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
 	"stat objects",            /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	0,                         /* tp_methods */
 	0,                         /* tp_members */
 	listdir_stat_getsets,      /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	0,                         /* tp_init */
 	0,                         /* tp_alloc */
 	listdir_stat_new,          /* tp_new */
 };
 
 #ifdef _WIN32
 
 static int to_python_time(const FILETIME *tm)
 {
 	/* number of seconds between epoch and January 1 1601 */
 	const __int64 a0 = (__int64)134774L * (__int64)24L * (__int64)3600L;
 	/* conversion factor from 100ns to 1s */
 	const __int64 a1 = 10000000;
 	/* explicit (int) cast to suspend compiler warnings */
 	return (int)((((__int64)tm->dwHighDateTime << 32)
 			+ tm->dwLowDateTime) / a1 - a0);
 }
 
 static PyObject *make_item(const WIN32_FIND_DATAA *fd, int wantstat)
 {
 	PyObject *py_st;
 	struct hg_stat *stp;
 
 	int kind = (fd->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
 		? _S_IFDIR : _S_IFREG;
 
 	if (!wantstat)
 		return Py_BuildValue(PY23("si", "yi"), fd->cFileName, kind);
 
 	py_st = PyObject_CallObject((PyObject *)&listdir_stat_type, NULL);
 	if (!py_st)
 		return NULL;
 
 	stp = &((struct listdir_stat *)py_st)->st;
 	/*
 	use kind as st_mode
 	rwx bits on Win32 are meaningless
 	and Hg does not use them anyway
 	*/
 	stp->st_mode  = kind;
 	stp->st_mtime = to_python_time(&fd->ftLastWriteTime);
 	stp->st_ctime = to_python_time(&fd->ftCreationTime);
 	if (kind == _S_IFREG)
 		stp->st_size = ((__int64)fd->nFileSizeHigh << 32)
 				+ fd->nFileSizeLow;
 	return Py_BuildValue(PY23("siN", "yiN"), fd->cFileName,
 		kind, py_st);
 }
 
 static PyObject *_listdir(char *path, Py_ssize_t plen, int wantstat, char *skip)
 {
 	PyObject *rval = NULL; /* initialize - return value */
 	PyObject *list;
 	HANDLE fh;
 	WIN32_FIND_DATAA fd;
 	char *pattern;
 
 	/* build the path + \* pattern string */
 	pattern = PyMem_Malloc(plen + 3); /* path + \* + \0 */
 	if (!pattern) {
 		PyErr_NoMemory();
 		goto error_nomem;
 	}
 	memcpy(pattern, path, plen);
 
 	if (plen > 0) {
 		char c = path[plen-1];
 		if (c != ':' && c != '/' && c != '\\')
 			pattern[plen++] = '\\';
 	}
 	pattern[plen++] = '*';
 	pattern[plen] = '\0';
 
 	fh = FindFirstFileA(pattern, &fd);
 	if (fh == INVALID_HANDLE_VALUE) {
 		PyErr_SetFromWindowsErrWithFilename(GetLastError(), path);
 		goto error_file;
 	}
 
 	list = PyList_New(0);
 	if (!list)
 		goto error_list;
 
 	do {
 		PyObject *item;
 
 		if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
 			if (!strcmp(fd.cFileName, ".")
 			|| !strcmp(fd.cFileName, ".."))
 				continue;
 
 			if (skip && !strcmp(fd.cFileName, skip)) {
 				rval = PyList_New(0);
 				goto error;
 			}
 		}
 
 		item = make_item(&fd, wantstat);
 		if (!item)
 			goto error;
 
 		if (PyList_Append(list, item)) {
 			Py_XDECREF(item);
 			goto error;
 		}
 
 		Py_XDECREF(item);
 	} while (FindNextFileA(fh, &fd));
 
 	if (GetLastError() != ERROR_NO_MORE_FILES) {
 		PyErr_SetFromWindowsErrWithFilename(GetLastError(), path);
 		goto error;
 	}
 
 	rval = list;
 	Py_XINCREF(rval);
 error:
 	Py_XDECREF(list);
 error_list:
 	FindClose(fh);
 error_file:
 	PyMem_Free(pattern);
 error_nomem:
 	return rval;
 }
 
 #else
 
 int entkind(struct dirent *ent)
 {
 #ifdef DT_REG
 	switch (ent->d_type) {
 	case DT_REG: return S_IFREG;
 	case DT_DIR: return S_IFDIR;
 	case DT_LNK: return S_IFLNK;
 	case DT_BLK: return S_IFBLK;
 	case DT_CHR: return S_IFCHR;
 	case DT_FIFO: return S_IFIFO;
 	case DT_SOCK: return S_IFSOCK;
 	}
 #endif
 	return -1;
 }
 
 static PyObject *makestat(const struct stat *st)
 {
 	PyObject *stat;
 
 	stat = PyObject_CallObject((PyObject *)&listdir_stat_type, NULL);
 	if (stat)
 		memcpy(&((struct listdir_stat *)stat)->st, st, sizeof(*st));
 	return stat;
 }
 
 static PyObject *_listdir_stat(char *path, int pathlen, int keepstat,
 			       char *skip)
 {
 	PyObject *list, *elem, *ret = NULL;
 	char fullpath[PATH_MAX + 10];
 	int kind, err;
 	struct stat st;
 	struct dirent *ent;
 	DIR *dir;
 #ifdef AT_SYMLINK_NOFOLLOW
 	int dfd = -1;
 #endif
 
 	if (pathlen >= PATH_MAX) {
 		errno = ENAMETOOLONG;
 		PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 		goto error_value;
 	}
 	strncpy(fullpath, path, PATH_MAX);
 	fullpath[pathlen] = '/';
 
 #ifdef AT_SYMLINK_NOFOLLOW
 	dfd = open(path, O_RDONLY);
 	if (dfd == -1) {
 		PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 		goto error_value;
 	}
 	dir = fdopendir(dfd);
 #else
 	dir = opendir(path);
 #endif
 	if (!dir) {
 		PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 		goto error_dir;
 	}
 
 	list = PyList_New(0);
 	if (!list)
 		goto error_list;
 
 	while ((ent = readdir(dir))) {
 		if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, ".."))
 			continue;
 
 		kind = entkind(ent);
 		if (kind == -1 || keepstat) {
 #ifdef AT_SYMLINK_NOFOLLOW
 			err = fstatat(dfd, ent->d_name, &st,
 				      AT_SYMLINK_NOFOLLOW);
 #else
 			strncpy(fullpath + pathlen + 1, ent->d_name,
 				PATH_MAX - pathlen);
 			fullpath[PATH_MAX] = '\0';
 			err = lstat(fullpath, &st);
 #endif
 			if (err == -1) {
 				/* race with file deletion? */
 				if (errno == ENOENT)
 					continue;
 				strncpy(fullpath + pathlen + 1, ent->d_name,
 					PATH_MAX - pathlen);
 				fullpath[PATH_MAX] = 0;
 				PyErr_SetFromErrnoWithFilename(PyExc_OSError,
 							       fullpath);
 				goto error;
 			}
 			kind = st.st_mode & S_IFMT;
 		}
 
 		/* quit early? */
 		if (skip && kind == S_IFDIR && !strcmp(ent->d_name, skip)) {
 			ret = PyList_New(0);
 			goto error;
 		}
 
 		if (keepstat) {
 			PyObject *stat = makestat(&st);
 			if (!stat)
 				goto error;
 			elem = Py_BuildValue(PY23("siN", "yiN"), ent->d_name,
 					     kind, stat);
 		} else
 			elem = Py_BuildValue(PY23("si", "yi"), ent->d_name,
 					     kind);
 		if (!elem)
 			goto error;
 
 		PyList_Append(list, elem);
 		Py_DECREF(elem);
 	}
 
 	ret = list;
 	Py_INCREF(ret);
 
 error:
 	Py_DECREF(list);
 error_list:
 	closedir(dir);
 	/* closedir also closes its dirfd */
 	goto error_value;
 error_dir:
 #ifdef AT_SYMLINK_NOFOLLOW
 	close(dfd);
 #endif
 error_value:
 	return ret;
 }
 
 #ifdef __APPLE__
 
 typedef struct {
 	u_int32_t length;
 	attrreference_t name;
 	fsobj_type_t obj_type;
 	struct timespec mtime;
 #if __LITTLE_ENDIAN__
 	mode_t access_mask;
 	uint16_t padding;
 #else
 	uint16_t padding;
 	mode_t access_mask;
 #endif
 	off_t size;
 } __attribute__((packed)) attrbuf_entry;
 
 int attrkind(attrbuf_entry *entry)
 {
 	switch (entry->obj_type) {
 	case VREG: return S_IFREG;
 	case VDIR: return S_IFDIR;
 	case VLNK: return S_IFLNK;
 	case VBLK: return S_IFBLK;
 	case VCHR: return S_IFCHR;
 	case VFIFO: return S_IFIFO;
 	case VSOCK: return S_IFSOCK;
 	}
 	return -1;
 }
 
 /* get these many entries at a time */
 #define LISTDIR_BATCH_SIZE 50
 
 static PyObject *_listdir_batch(char *path, int pathlen, int keepstat,
 				char *skip, bool *fallback)
 {
 	PyObject *list, *elem, *ret = NULL;
 	int kind, err;
 	unsigned long index;
 	unsigned int count, old_state, new_state;
 	bool state_seen = false;
 	attrbuf_entry *entry;
 	/* from the getattrlist(2) man page: a path can be no longer than
 	   (NAME_MAX * 3 + 1) bytes. Also, "The getattrlist() function will
 	   silently truncate attribute data if attrBufSize is too small." So
 	   pass in a buffer big enough for the worst case. */
 	char attrbuf[LISTDIR_BATCH_SIZE * (sizeof(attrbuf_entry) + NAME_MAX * 3 + 1)];
 	unsigned int basep_unused;
 
 	struct stat st;
 	int dfd = -1;
 
 	/* these must match the attrbuf_entry struct, otherwise you'll end up
 	   with garbage */
 	struct attrlist requested_attr = {0};
 	requested_attr.bitmapcount = ATTR_BIT_MAP_COUNT;
 	requested_attr.commonattr = (ATTR_CMN_NAME | ATTR_CMN_OBJTYPE |
 				     ATTR_CMN_MODTIME | ATTR_CMN_ACCESSMASK);
 	requested_attr.fileattr = ATTR_FILE_DATALENGTH;
 
 	*fallback = false;
 
 	if (pathlen >= PATH_MAX) {
 		errno = ENAMETOOLONG;
 		PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 		goto error_value;
 	}
 
 	dfd = open(path, O_RDONLY);
 	if (dfd == -1) {
 		PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 		goto error_value;
 	}
 
 	list = PyList_New(0);
 	if (!list)
 		goto error_dir;
 
 	do {
 		count = LISTDIR_BATCH_SIZE;
 		err = getdirentriesattr(dfd, &requested_attr, &attrbuf,
 					sizeof(attrbuf), &count, &basep_unused,
 					&new_state, 0);
 		if (err < 0) {
 			if (errno == ENOTSUP) {
 				/* We're on a filesystem that doesn't support
 				   getdirentriesattr. Fall back to the
 				   stat-based implementation. */
 				*fallback = true;
 			} else
 				PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
 			goto error;
 		}
 
 		if (!state_seen) {
 			old_state = new_state;
 			state_seen = true;
 		} else if (old_state != new_state) {
 			/* There's an edge case with getdirentriesattr. Consider
 			   the following initial list of files:
 
 			   a
 			   b
 			   <--
 			   c
 			   d
 
 			   If the iteration is paused at the arrow, and b is
 			   deleted before it is resumed, getdirentriesattr will
 			   not return d at all!  Ordinarily we're expected to
 			   restart the iteration from the beginning. To avoid
 			   getting stuck in a retry loop here, fall back to
 			   stat. */
 			*fallback = true;
 			goto error;
 		}
 
 		entry = (attrbuf_entry *)attrbuf;
 
 		for (index = 0; index < count; index++) {
 			char *filename = ((char *)&entry->name) +
 				entry->name.attr_dataoffset;
 
 			if (!strcmp(filename, ".") || !strcmp(filename, ".."))
 				continue;
 
 			kind = attrkind(entry);
 			if (kind == -1) {
 				PyErr_Format(PyExc_OSError,
 					     "unknown object type %u for file "
 					     "%s%s!",
 					     entry->obj_type, path, filename);
 				goto error;
 			}
 
 			/* quit early? */
 			if (skip && kind == S_IFDIR && !strcmp(filename, skip)) {
 				ret = PyList_New(0);
 				goto error;
 			}
 
 			if (keepstat) {
 				PyObject *stat = NULL;
 				/* from the getattrlist(2) man page: "Only the
 				   permission bits ... are valid". */
 				st.st_mode = (entry->access_mask & ~S_IFMT) | kind;
 				st.st_mtime = entry->mtime.tv_sec;
 				st.st_size = entry->size;
 				stat = makestat(&st);
 				if (!stat)
 					goto error;
 				elem = Py_BuildValue(PY23("siN", "yiN"),
 						     filename, kind, stat);
 			} else
 				elem = Py_BuildValue(PY23("si", "yi"),
 						     filename, kind);
 			if (!elem)
 				goto error;
 
 			PyList_Append(list, elem);
 			Py_DECREF(elem);
 
 			entry = (attrbuf_entry *)((char *)entry + entry->length);
 		}
 	} while (err == 0);
 
 	ret = list;
 	Py_INCREF(ret);
 
 error:
 	Py_DECREF(list);
 error_dir:
 	close(dfd);
 error_value:
 	return ret;
 }
 
 #endif /* __APPLE__ */
 
 static PyObject *_listdir(char *path, int pathlen, int keepstat, char *skip)
 {
 #ifdef __APPLE__
 	PyObject *ret;
 	bool fallback = false;
 
 	ret = _listdir_batch(path, pathlen, keepstat, skip, &fallback);
 	if (ret != NULL || !fallback)
 		return ret;
 #endif
 	return _listdir_stat(path, pathlen, keepstat, skip);
 }
 
 static PyObject *statfiles(PyObject *self, PyObject *args)
 {
 	PyObject *names, *stats;
 	Py_ssize_t i, count;
 
 	if (!PyArg_ParseTuple(args, "O:statfiles", &names))
 		return NULL;
 
 	count = PySequence_Length(names);
 	if (count == -1) {
 		PyErr_SetString(PyExc_TypeError, "not a sequence");
 		return NULL;
 	}
 
 	stats = PyList_New(count);
 	if (stats == NULL)
 		return NULL;
 
 	for (i = 0; i < count; i++) {
 		PyObject *stat, *pypath;
 		struct stat st;
 		int ret, kind;
 		char *path;
 
 		/* With a large file count or on a slow filesystem,
 		   don't block signals for long (issue4878). */
 		if ((i % 1000) == 999 && PyErr_CheckSignals() == -1)
 			goto bail;
 
 		pypath = PySequence_GetItem(names, i);
 		if (!pypath)
 			goto bail;
 		path = PyBytes_AsString(pypath);
 		if (path == NULL) {
 			Py_DECREF(pypath);
 			PyErr_SetString(PyExc_TypeError, "not a string");
 			goto bail;
 		}
 		ret = lstat(path, &st);
 		Py_DECREF(pypath);
 		kind = st.st_mode & S_IFMT;
 		if (ret != -1 && (kind == S_IFREG || kind == S_IFLNK)) {
 			stat = makestat(&st);
 			if (stat == NULL)
 				goto bail;
 			PyList_SET_ITEM(stats, i, stat);
 		} else {
 			Py_INCREF(Py_None);
 			PyList_SET_ITEM(stats, i, Py_None);
 		}
 	}
 
 	return stats;
 
 bail:
 	Py_DECREF(stats);
 	return NULL;
 }
 
 /*
  * recvfds() simply does not release GIL during blocking io operation because
  * command server is known to be single-threaded.
  *
  * Old systems such as Solaris don't provide CMSG_LEN, msg_control, etc.
  * Currently, recvfds() is not supported on these platforms.
  */
 #ifdef CMSG_LEN
 
 static ssize_t recvfdstobuf(int sockfd, int **rfds, void *cbuf, size_t cbufsize)
 {
 	char dummy[1];
 	struct iovec iov = {dummy, sizeof(dummy)};
 	struct msghdr msgh = {0};
 	struct cmsghdr *cmsg;
 
 	msgh.msg_iov = &iov;
 	msgh.msg_iovlen = 1;
 	msgh.msg_control = cbuf;
 	msgh.msg_controllen = (socklen_t)cbufsize;
 	if (recvmsg(sockfd, &msgh, 0) < 0)
 		return -1;
 
 	for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg;
 	     cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
 		if (cmsg->cmsg_level != SOL_SOCKET ||
 		    cmsg->cmsg_type != SCM_RIGHTS)
 			continue;
 		*rfds = (int *)CMSG_DATA(cmsg);
 		return (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int);
 	}
 
 	*rfds = cbuf;
 	return 0;
 }
 
 static PyObject *recvfds(PyObject *self, PyObject *args)
 {
 	int sockfd;
 	int *rfds = NULL;
 	ssize_t rfdscount, i;
 	char cbuf[256];
 	PyObject *rfdslist = NULL;
 
 	if (!PyArg_ParseTuple(args, "i", &sockfd))
 		return NULL;
 
 	rfdscount = recvfdstobuf(sockfd, &rfds, cbuf, sizeof(cbuf));
 	if (rfdscount < 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 
 	rfdslist = PyList_New(rfdscount);
 	if (!rfdslist)
 		goto bail;
 	for (i = 0; i < rfdscount; i++) {
 		PyObject *obj = PyLong_FromLong(rfds[i]);
 		if (!obj)
 			goto bail;
 		PyList_SET_ITEM(rfdslist, i, obj);
 	}
 	return rfdslist;
 
 bail:
 	Py_XDECREF(rfdslist);
 	return NULL;
 }
 
 #endif /* CMSG_LEN */
 
 /* allow disabling setprocname via compiler flags */
 #ifndef SETPROCNAME_USE_NONE
 #if defined(HAVE_SETPROCTITLE)
 /* setproctitle is the first choice - available in FreeBSD */
 #define SETPROCNAME_USE_SETPROCTITLE
 #elif (defined(__linux__) || defined(__APPLE__)) && PY_MAJOR_VERSION == 2
 /* rewrite the argv buffer in place - works in Linux and OS X. Py_GetArgcArgv
  * in Python 3 returns the copied wchar_t **argv, thus unsupported. */
 #define SETPROCNAME_USE_ARGVREWRITE
 #else
 #define SETPROCNAME_USE_NONE
 #endif
 #endif /* ndef SETPROCNAME_USE_NONE */
 
 #ifndef SETPROCNAME_USE_NONE
 static PyObject *setprocname(PyObject *self, PyObject *args)
 {
 	const char *name = NULL;
 	if (!PyArg_ParseTuple(args, PY23("s", "y"), &name))
 		return NULL;
 
 #if defined(SETPROCNAME_USE_SETPROCTITLE)
 	setproctitle("%s", name);
 #elif defined(SETPROCNAME_USE_ARGVREWRITE)
 	{
 		static char *argvstart = NULL;
 		static size_t argvsize = 0;
 		if (argvstart == NULL) {
 			int argc = 0, i;
 			char **argv = NULL;
 			char *argvend;
 			extern void Py_GetArgcArgv(int *argc, char ***argv);
 			Py_GetArgcArgv(&argc, &argv);
 			/* Py_GetArgcArgv may not do much if a custom python
 			 * launcher is used that doesn't record the information
 			 * it needs. Let's handle this gracefully instead of
 			 * segfaulting. */
 			if (argv != NULL)
 				argvend = argvstart = argv[0];
 			else
 				argvend = argvstart = NULL;
 
 			/* Check the memory we can use. Typically, argv[i] and
 			 * argv[i + 1] are continuous. */
 			for (i = 0; i < argc; ++i) {
 				size_t len;
 				if (argv[i] > argvend || argv[i] < argvstart)
 					break; /* not continuous */
 				len = strlen(argv[i]);
 				argvend = argv[i] + len + 1 /* '\0' */;
 			}
 			if (argvend > argvstart) /* sanity check */
 				argvsize = argvend - argvstart;
 		}
 
 		if (argvstart && argvsize > 1) {
 			int n = snprintf(argvstart, argvsize, "%s", name);
 			if (n >= 0 && (size_t)n < argvsize)
 				memset(argvstart + n, 0, argvsize - n);
 		}
 	}
 #endif
 
 	Py_RETURN_NONE;
 }
 #endif /* ndef SETPROCNAME_USE_NONE */
 
 #if defined(HAVE_BSD_STATFS)
 static const char *describefstype(const struct statfs *pbuf)
 {
 	/* BSD or OSX provides a f_fstypename field */
 	return pbuf->f_fstypename;
 }
 #elif defined(HAVE_LINUX_STATFS)
 static const char *describefstype(const struct statfs *pbuf)
 {
 	/* Begin of Linux filesystems */
 #ifdef ADFS_SUPER_MAGIC
 	if (pbuf->f_type == ADFS_SUPER_MAGIC)
 		return "adfs";
 #endif
 #ifdef AFFS_SUPER_MAGIC
 	if (pbuf->f_type == AFFS_SUPER_MAGIC)
 		return "affs";
 #endif
 #ifdef AUTOFS_SUPER_MAGIC
 	if (pbuf->f_type == AUTOFS_SUPER_MAGIC)
 		return "autofs";
 #endif
 #ifdef BDEVFS_MAGIC
 	if (pbuf->f_type == BDEVFS_MAGIC)
 		return "bdevfs";
 #endif
 #ifdef BEFS_SUPER_MAGIC
 	if (pbuf->f_type == BEFS_SUPER_MAGIC)
 		return "befs";
 #endif
 #ifdef BFS_MAGIC
 	if (pbuf->f_type == BFS_MAGIC)
 		return "bfs";
 #endif
 #ifdef BINFMTFS_MAGIC
 	if (pbuf->f_type == BINFMTFS_MAGIC)
 		return "binfmtfs";
 #endif
 #ifdef BTRFS_SUPER_MAGIC
 	if (pbuf->f_type == BTRFS_SUPER_MAGIC)
 		return "btrfs";
 #endif
 #ifdef CGROUP_SUPER_MAGIC
 	if (pbuf->f_type == CGROUP_SUPER_MAGIC)
 		return "cgroup";
 #endif
 #ifdef CIFS_MAGIC_NUMBER
 	if (pbuf->f_type == CIFS_MAGIC_NUMBER)
 		return "cifs";
 #endif
 #ifdef CODA_SUPER_MAGIC
 	if (pbuf->f_type == CODA_SUPER_MAGIC)
 		return "coda";
 #endif
 #ifdef COH_SUPER_MAGIC
 	if (pbuf->f_type == COH_SUPER_MAGIC)
 		return "coh";
 #endif
 #ifdef CRAMFS_MAGIC
 	if (pbuf->f_type == CRAMFS_MAGIC)
 		return "cramfs";
 #endif
 #ifdef DEBUGFS_MAGIC
 	if (pbuf->f_type == DEBUGFS_MAGIC)
 		return "debugfs";
 #endif
 #ifdef DEVFS_SUPER_MAGIC
 	if (pbuf->f_type == DEVFS_SUPER_MAGIC)
 		return "devfs";
 #endif
 #ifdef DEVPTS_SUPER_MAGIC
 	if (pbuf->f_type == DEVPTS_SUPER_MAGIC)
 		return "devpts";
 #endif
 #ifdef EFIVARFS_MAGIC
 	if (pbuf->f_type == EFIVARFS_MAGIC)
 		return "efivarfs";
 #endif
 #ifdef EFS_SUPER_MAGIC
 	if (pbuf->f_type == EFS_SUPER_MAGIC)
 		return "efs";
 #endif
 #ifdef EXT_SUPER_MAGIC
 	if (pbuf->f_type == EXT_SUPER_MAGIC)
 		return "ext";
 #endif
 #ifdef EXT2_OLD_SUPER_MAGIC
 	if (pbuf->f_type == EXT2_OLD_SUPER_MAGIC)
 		return "ext2";
 #endif
 #ifdef EXT2_SUPER_MAGIC
 	if (pbuf->f_type == EXT2_SUPER_MAGIC)
 		return "ext2";
 #endif
 #ifdef EXT3_SUPER_MAGIC
 	if (pbuf->f_type == EXT3_SUPER_MAGIC)
 		return "ext3";
 #endif
 #ifdef EXT4_SUPER_MAGIC
 	if (pbuf->f_type == EXT4_SUPER_MAGIC)
 		return "ext4";
 #endif
 #ifdef F2FS_SUPER_MAGIC
 	if (pbuf->f_type == F2FS_SUPER_MAGIC)
 		return "f2fs";
 #endif
 #ifdef FUSE_SUPER_MAGIC
 	if (pbuf->f_type == FUSE_SUPER_MAGIC)
 		return "fuse";
 #endif
 #ifdef FUTEXFS_SUPER_MAGIC
 	if (pbuf->f_type == FUTEXFS_SUPER_MAGIC)
 		return "futexfs";
 #endif
 #ifdef HFS_SUPER_MAGIC
 	if (pbuf->f_type == HFS_SUPER_MAGIC)
 		return "hfs";
 #endif
 #ifdef HOSTFS_SUPER_MAGIC
 	if (pbuf->f_type == HOSTFS_SUPER_MAGIC)
 		return "hostfs";
 #endif
 #ifdef HPFS_SUPER_MAGIC
 	if (pbuf->f_type == HPFS_SUPER_MAGIC)
 		return "hpfs";
 #endif
 #ifdef HUGETLBFS_MAGIC
 	if (pbuf->f_type == HUGETLBFS_MAGIC)
 		return "hugetlbfs";
 #endif
 #ifdef ISOFS_SUPER_MAGIC
 	if (pbuf->f_type == ISOFS_SUPER_MAGIC)
 		return "isofs";
 #endif
 #ifdef JFFS2_SUPER_MAGIC
 	if (pbuf->f_type == JFFS2_SUPER_MAGIC)
 		return "jffs2";
 #endif
 #ifdef JFS_SUPER_MAGIC
 	if (pbuf->f_type == JFS_SUPER_MAGIC)
 		return "jfs";
 #endif
 #ifdef MINIX_SUPER_MAGIC
 	if (pbuf->f_type == MINIX_SUPER_MAGIC)
 		return "minix";
 #endif
 #ifdef MINIX2_SUPER_MAGIC
 	if (pbuf->f_type == MINIX2_SUPER_MAGIC)
 		return "minix2";
 #endif
 #ifdef MINIX3_SUPER_MAGIC
 	if (pbuf->f_type == MINIX3_SUPER_MAGIC)
 		return "minix3";
 #endif
 #ifdef MQUEUE_MAGIC
 	if (pbuf->f_type == MQUEUE_MAGIC)
 		return "mqueue";
 #endif
 #ifdef MSDOS_SUPER_MAGIC
 	if (pbuf->f_type == MSDOS_SUPER_MAGIC)
 		return "msdos";
 #endif
 #ifdef NCP_SUPER_MAGIC
 	if (pbuf->f_type == NCP_SUPER_MAGIC)
 		return "ncp";
 #endif
 #ifdef NFS_SUPER_MAGIC
 	if (pbuf->f_type == NFS_SUPER_MAGIC)
 		return "nfs";
 #endif
 #ifdef NILFS_SUPER_MAGIC
 	if (pbuf->f_type == NILFS_SUPER_MAGIC)
 		return "nilfs";
 #endif
 #ifdef NTFS_SB_MAGIC
 	if (pbuf->f_type == NTFS_SB_MAGIC)
 		return "ntfs-sb";
 #endif
 #ifdef OCFS2_SUPER_MAGIC
 	if (pbuf->f_type == OCFS2_SUPER_MAGIC)
 		return "ocfs2";
 #endif
 #ifdef OPENPROM_SUPER_MAGIC
 	if (pbuf->f_type == OPENPROM_SUPER_MAGIC)
 		return "openprom";
 #endif
 #ifdef OVERLAYFS_SUPER_MAGIC
 	if (pbuf->f_type == OVERLAYFS_SUPER_MAGIC)
 		return "overlay";
 #endif
 #ifdef PIPEFS_MAGIC
 	if (pbuf->f_type == PIPEFS_MAGIC)
 		return "pipefs";
 #endif
 #ifdef PROC_SUPER_MAGIC
 	if (pbuf->f_type == PROC_SUPER_MAGIC)
 		return "proc";
 #endif
 #ifdef PSTOREFS_MAGIC
 	if (pbuf->f_type == PSTOREFS_MAGIC)
 		return "pstorefs";
 #endif
 #ifdef QNX4_SUPER_MAGIC
 	if (pbuf->f_type == QNX4_SUPER_MAGIC)
 		return "qnx4";
 #endif
 #ifdef QNX6_SUPER_MAGIC
 	if (pbuf->f_type == QNX6_SUPER_MAGIC)
 		return "qnx6";
 #endif
 #ifdef RAMFS_MAGIC
 	if (pbuf->f_type == RAMFS_MAGIC)
 		return "ramfs";
 #endif
 #ifdef REISERFS_SUPER_MAGIC
 	if (pbuf->f_type == REISERFS_SUPER_MAGIC)
 		return "reiserfs";
 #endif
 #ifdef ROMFS_MAGIC
 	if (pbuf->f_type == ROMFS_MAGIC)
 		return "romfs";
 #endif
 #ifdef SECURITYFS_MAGIC
 	if (pbuf->f_type == SECURITYFS_MAGIC)
 		return "securityfs";
 #endif
 #ifdef SELINUX_MAGIC
 	if (pbuf->f_type == SELINUX_MAGIC)
 		return "selinux";
 #endif
 #ifdef SMACK_MAGIC
 	if (pbuf->f_type == SMACK_MAGIC)
 		return "smack";
 #endif
 #ifdef SMB_SUPER_MAGIC
 	if (pbuf->f_type == SMB_SUPER_MAGIC)
 		return "smb";
 #endif
 #ifdef SOCKFS_MAGIC
 	if (pbuf->f_type == SOCKFS_MAGIC)
 		return "sockfs";
 #endif
 #ifdef SQUASHFS_MAGIC
 	if (pbuf->f_type == SQUASHFS_MAGIC)
 		return "squashfs";
 #endif
 #ifdef SYSFS_MAGIC
 	if (pbuf->f_type == SYSFS_MAGIC)
 		return "sysfs";
 #endif
 #ifdef SYSV2_SUPER_MAGIC
 	if (pbuf->f_type == SYSV2_SUPER_MAGIC)
 		return "sysv2";
 #endif
 #ifdef SYSV4_SUPER_MAGIC
 	if (pbuf->f_type == SYSV4_SUPER_MAGIC)
 		return "sysv4";
 #endif
 #ifdef TMPFS_MAGIC
 	if (pbuf->f_type == TMPFS_MAGIC)
 		return "tmpfs";
 #endif
 #ifdef UDF_SUPER_MAGIC
 	if (pbuf->f_type == UDF_SUPER_MAGIC)
 		return "udf";
 #endif
 #ifdef UFS_MAGIC
 	if (pbuf->f_type == UFS_MAGIC)
 		return "ufs";
 #endif
 #ifdef USBDEVICE_SUPER_MAGIC
 	if (pbuf->f_type == USBDEVICE_SUPER_MAGIC)
 		return "usbdevice";
 #endif
 #ifdef V9FS_MAGIC
 	if (pbuf->f_type == V9FS_MAGIC)
 		return "v9fs";
 #endif
 #ifdef VXFS_SUPER_MAGIC
 	if (pbuf->f_type == VXFS_SUPER_MAGIC)
 		return "vxfs";
 #endif
 #ifdef XENFS_SUPER_MAGIC
 	if (pbuf->f_type == XENFS_SUPER_MAGIC)
 		return "xenfs";
 #endif
 #ifdef XENIX_SUPER_MAGIC
 	if (pbuf->f_type == XENIX_SUPER_MAGIC)
 		return "xenix";
 #endif
 #ifdef XFS_SUPER_MAGIC
 	if (pbuf->f_type == XFS_SUPER_MAGIC)
 		return "xfs";
 #endif
 	/* End of Linux filesystems */
 	return NULL;
 }
 #endif /* def HAVE_LINUX_STATFS */
 
 #if defined(HAVE_BSD_STATFS) || defined(HAVE_LINUX_STATFS)
 /* given a directory path, return filesystem type name (best-effort) */
 static PyObject *getfstype(PyObject *self, PyObject *args)
 {
 	const char *path = NULL;
 	struct statfs buf;
 	int r;
 	if (!PyArg_ParseTuple(args, PY23("s", "y"), &path))
 		return NULL;
 
 	memset(&buf, 0, sizeof(buf));
 	r = statfs(path, &buf);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 	return Py_BuildValue(PY23("s", "y"), describefstype(&buf));
 }
 #endif /* defined(HAVE_LINUX_STATFS) || defined(HAVE_BSD_STATFS) */
 
 #if defined(HAVE_BSD_STATFS)
 /* given a directory path, return filesystem mount point (best-effort) */
 static PyObject *getfsmountpoint(PyObject *self, PyObject *args)
 {
 	const char *path = NULL;
 	struct statfs buf;
 	int r;
 	if (!PyArg_ParseTuple(args, PY23("s", "y"), &path))
 		return NULL;
 
 	memset(&buf, 0, sizeof(buf));
 	r = statfs(path, &buf);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 	return Py_BuildValue(PY23("s", "y"), buf.f_mntonname);
 }
 #endif /* defined(HAVE_BSD_STATFS) */
 
 static PyObject *unblocksignal(PyObject *self, PyObject *args)
 {
 	int sig = 0;
 	sigset_t set;
 	int r;
 	if (!PyArg_ParseTuple(args, "i", &sig))
 		return NULL;
 	r = sigemptyset(&set);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 	r = sigaddset(&set, sig);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 	r = sigprocmask(SIG_UNBLOCK, &set, NULL);
 	if (r != 0)
 		return PyErr_SetFromErrno(PyExc_OSError);
 	Py_RETURN_NONE;
 }
 
 #endif /* ndef _WIN32 */
 
 static PyObject *listdir(PyObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *statobj = NULL; /* initialize - optional arg */
 	PyObject *skipobj = NULL; /* initialize - optional arg */
 	char *path, *skip = NULL;
 	Py_ssize_t plen;
 	int wantstat;
 
 	static char *kwlist[] = {"path", "stat", "skip", NULL};
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, PY23("s#|OO:listdir",
 							    "y#|OO:listdir"),
 			kwlist, &path, &plen, &statobj, &skipobj))
 		return NULL;
 
 	wantstat = statobj && PyObject_IsTrue(statobj);
 
 	if (skipobj && skipobj != Py_None) {
 		skip = PyBytes_AsString(skipobj);
 		if (!skip)
 			return NULL;
 	}
 
 	return _listdir(path, plen, wantstat, skip);
 }
 
 #ifdef _WIN32
 static PyObject *posixfile(PyObject *self, PyObject *args, PyObject *kwds)
 {
 	static char *kwlist[] = {"name", "mode", "buffering", NULL};
 	PyObject *file_obj = NULL;
 	char *name = NULL;
 	char *mode = "rb";
 	DWORD access = 0;
 	DWORD creation;
 	HANDLE handle;
 	int fd, flags = 0;
 	int bufsize = -1;
 	char m0, m1, m2;
 	char fpmode[4];
 	int fppos = 0;
 	int plus;
 #ifndef IS_PY3K
 	FILE *fp;
 #endif
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwds, PY23("et|si:posixfile",
 							  "et|yi:posixfile"),
 					 kwlist,
 					 Py_FileSystemDefaultEncoding,
 					 &name, &mode, &bufsize))
 		return NULL;
 
 	m0 = mode[0];
 	m1 = m0 ? mode[1] : '\0';
 	m2 = m1 ? mode[2] : '\0';
 	plus = m1 == '+' || m2 == '+';
 
 	fpmode[fppos++] = m0;
 	if (m1 == 'b' || m2 == 'b') {
 		flags = _O_BINARY;
 		fpmode[fppos++] = 'b';
 	}
 	else
 		flags = _O_TEXT;
 	if (m0 == 'r' && !plus) {
 		flags |= _O_RDONLY;
 		access = GENERIC_READ;
 	} else {
 		/*
 		work around http://support.microsoft.com/kb/899149 and
 		set _O_RDWR for 'w' and 'a', even if mode has no '+'
 		*/
 		flags |= _O_RDWR;
 		access = GENERIC_READ | GENERIC_WRITE;
 		fpmode[fppos++] = '+';
 	}
 	fpmode[fppos++] = '\0';
 
 	switch (m0) {
 	case 'r':
 		creation = OPEN_EXISTING;
 		break;
 	case 'w':
 		creation = CREATE_ALWAYS;
 		break;
 	case 'a':
 		creation = OPEN_ALWAYS;
 		flags |= _O_APPEND;
 		break;
 	default:
 		PyErr_Format(PyExc_ValueError,
 			     "mode string must begin with one of 'r', 'w', "
 			     "or 'a', not '%c'", m0);
 		goto bail;
 	}
 
 	handle = CreateFile(name, access,
 			    FILE_SHARE_READ | FILE_SHARE_WRITE |
 			    FILE_SHARE_DELETE,
 			    NULL,
 			    creation,
 			    FILE_ATTRIBUTE_NORMAL,
 			    0);
 
 	if (handle == INVALID_HANDLE_VALUE) {
 		PyErr_SetFromWindowsErrWithFilename(GetLastError(), name);
 		goto bail;
 	}
 
 	fd = _open_osfhandle((intptr_t)handle, flags);
 
 	if (fd == -1) {
 		CloseHandle(handle);
 		PyErr_SetFromErrnoWithFilename(PyExc_IOError, name);
 		goto bail;
 	}
 #ifndef IS_PY3K
 	fp = _fdopen(fd, fpmode);
 	if (fp == NULL) {
 		_close(fd);
 		PyErr_SetFromErrnoWithFilename(PyExc_IOError, name);
 		goto bail;
 	}
 
 	file_obj = PyFile_FromFile(fp, name, mode, fclose);
 	if (file_obj == NULL) {
 		fclose(fp);
 		goto bail;
 	}
 
 	PyFile_SetBufSize(file_obj, bufsize);
 #else
 	file_obj = PyFile_FromFd(fd, name, mode, bufsize, NULL, NULL, NULL, 1);
 	if (file_obj == NULL)
 		goto bail;
 #endif
 bail:
 	PyMem_Free(name);
 	return file_obj;
 }
 #endif
 
 #ifdef __APPLE__
 #include <ApplicationServices/ApplicationServices.h>
 
 static PyObject *isgui(PyObject *self)
 {
 	CFDictionaryRef dict = CGSessionCopyCurrentDictionary();
 
 	if (dict != NULL) {
 		CFRelease(dict);
 		Py_RETURN_TRUE;
 	} else {
 		Py_RETURN_FALSE;
 	}
 }
 #endif
 
 static char osutil_doc[] = "Native operating system services.";
 
 static PyMethodDef methods[] = {
 	{"listdir", (PyCFunction)listdir, METH_VARARGS | METH_KEYWORDS,
 	 "list a directory\n"},
 #ifdef _WIN32
 	{"posixfile", (PyCFunction)posixfile, METH_VARARGS | METH_KEYWORDS,
 	 "Open a file with POSIX-like semantics.\n"
 "On error, this function may raise either a WindowsError or an IOError."},
 #else
 	{"statfiles", (PyCFunction)statfiles, METH_VARARGS | METH_KEYWORDS,
 	 "stat a series of files or symlinks\n"
 "Returns None for non-existent entries and entries of other types.\n"},
 #ifdef CMSG_LEN
 	{"recvfds", (PyCFunction)recvfds, METH_VARARGS,
 	 "receive list of file descriptors via socket\n"},
 #endif
 #ifndef SETPROCNAME_USE_NONE
 	{"setprocname", (PyCFunction)setprocname, METH_VARARGS,
 	 "set process title (best-effort)\n"},
 #endif
 #if defined(HAVE_BSD_STATFS) || defined(HAVE_LINUX_STATFS)
 	{"getfstype", (PyCFunction)getfstype, METH_VARARGS,
 	 "get filesystem type (best-effort)\n"},
 #endif
 #if defined(HAVE_BSD_STATFS)
 	{"getfsmountpoint", (PyCFunction)getfsmountpoint, METH_VARARGS,
 	 "get filesystem mount point (best-effort)\n"},
 #endif
 	{"unblocksignal", (PyCFunction)unblocksignal, METH_VARARGS,
 	 "change signal mask to unblock a given signal\n"},
 #endif /* ndef _WIN32 */
 #ifdef __APPLE__
 	{
 		"isgui", (PyCFunction)isgui, METH_NOARGS,
 		"Is a CoreGraphics session available?"
 	},
 #endif
 	{NULL, NULL}
 };
 
 static const int version = 4;
 
 #ifdef IS_PY3K
 static struct PyModuleDef osutil_module = {
 	PyModuleDef_HEAD_INIT,
 	"osutil",
 	osutil_doc,
 	-1,
 	methods
 };
 
 PyMODINIT_FUNC PyInit_osutil(void)
 {
 	PyObject *m;
 	if (PyType_Ready(&listdir_stat_type) < 0)
 		return NULL;
 
 	m = PyModule_Create(&osutil_module);
 	PyModule_AddIntConstant(m, "version", version);
 	return m;
 }
 #else
 PyMODINIT_FUNC initosutil(void)
 {
 	PyObject *m;
 	if (PyType_Ready(&listdir_stat_type) == -1)
 		return;
 
 	m = Py_InitModule3("osutil", methods, osutil_doc);
 	PyModule_AddIntConstant(m, "version", version);
 }
 #endif

mercurial/cext/pathencode.c

@@ -1,802 +1,803 b''
 /*
  pathencode.c - efficient path name encoding
 
  Copyright 2012 Facebook
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 /*
  * An implementation of the name encoding scheme used by the fncache
  * store.  The common case is of a path < 120 bytes long, which is
  * handled either in a single pass with no allocations or two passes
  * with a single allocation.  For longer paths, multiple passes are
  * required.
  */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <assert.h>
 #include <ctype.h>
 #include <stdlib.h>
 #include <string.h>
+#include "pythoncapi_compat.h"
 
 #include "util.h"
 
 /* state machine for the fast path */
 enum path_state {
 	START, /* first byte of a path component */
 	A,     /* "AUX" */
 	AU,
 	THIRD, /* third of a 3-byte sequence, e.g. "AUX", "NUL" */
 	C,     /* "CON" or "COMn" */
 	CO,
 	COMLPT, /* "COM" or "LPT" */
 	COMLPTn,
 	L,
 	LP,
 	N,
 	NU,
 	P, /* "PRN" */
 	PR,
 	LDOT, /* leading '.' */
 	DOT,  /* '.' in a non-leading position */
 	H,    /* ".h" */
 	HGDI, /* ".hg", ".d", or ".i" */
 	SPACE,
 	DEFAULT, /* byte of a path component after the first */
 };
 
 /* state machine for dir-encoding */
 enum dir_state {
 	DDOT,
 	DH,
 	DHGDI,
 	DDEFAULT,
 };
 
 static inline int inset(const uint32_t bitset[], char c)
 {
 	return bitset[((uint8_t)c) >> 5] & (1 << (((uint8_t)c) & 31));
 }
 
 static inline void charcopy(char *dest, Py_ssize_t *destlen, size_t destsize,
                             char c)
 {
 	if (dest) {
 		assert(*destlen < destsize);
 		dest[*destlen] = c;
 	}
 	(*destlen)++;
 }
 
 static inline void memcopy(char *dest, Py_ssize_t *destlen, size_t destsize,
                            const void *src, Py_ssize_t len)
 {
 	if (dest) {
 		assert(*destlen + len < destsize);
 		memcpy((void *)&dest[*destlen], src, len);
 	}
 	*destlen += len;
 }
 
 static inline void hexencode(char *dest, Py_ssize_t *destlen, size_t destsize,
                              uint8_t c)
 {
 	static const char hexdigit[] = "0123456789abcdef";
 
 	charcopy(dest, destlen, destsize, hexdigit[c >> 4]);
 	charcopy(dest, destlen, destsize, hexdigit[c & 15]);
 }
 
 /* 3-byte escape: tilde followed by two hex digits */
 static inline void escape3(char *dest, Py_ssize_t *destlen, size_t destsize,
                            char c)
 {
 	charcopy(dest, destlen, destsize, '~');
 	hexencode(dest, destlen, destsize, c);
 }
 
 static Py_ssize_t _encodedir(char *dest, size_t destsize, const char *src,
                              Py_ssize_t len)
 {
 	enum dir_state state = DDEFAULT;
 	Py_ssize_t i = 0, destlen = 0;
 
 	while (i < len) {
 		switch (state) {
 		case DDOT:
 			switch (src[i]) {
 			case 'd':
 			case 'i':
 				state = DHGDI;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'h':
 				state = DH;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			default:
 				state = DDEFAULT;
 				break;
 			}
 			break;
 		case DH:
 			if (src[i] == 'g') {
 				state = DHGDI;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DDEFAULT;
 			}
 			break;
 		case DHGDI:
 			if (src[i] == '/') {
 				memcopy(dest, &destlen, destsize, ".hg", 3);
 				charcopy(dest, &destlen, destsize, src[i++]);
 			}
 			state = DDEFAULT;
 			break;
 		case DDEFAULT:
 			if (src[i] == '.') {
 				state = DDOT;
 			}
 			charcopy(dest, &destlen, destsize, src[i++]);
 			break;
 		}
 	}
 
 	return destlen;
 }
 
 PyObject *encodedir(PyObject *self, PyObject *args)
 {
 	Py_ssize_t len, newlen;
 	PyObject *pathobj, *newobj;
 	char *path;
 
 	if (!PyArg_ParseTuple(args, "O:encodedir", &pathobj)) {
 		return NULL;
 	}
 
 	if (PyBytes_AsStringAndSize(pathobj, &path, &len) == -1) {
 		PyErr_SetString(PyExc_TypeError, "expected a string");
 		return NULL;
 	}
 
 	newlen = len ? _encodedir(NULL, 0, path, len + 1) : 1;
 
 	if (newlen == len + 1) {
 		Py_INCREF(pathobj);
 		return pathobj;
 	}
 
 	newobj = PyBytes_FromStringAndSize(NULL, newlen);
 
 	if (newobj) {
 		assert(PyBytes_Check(newobj));
 		Py_SIZE(newobj)--;
 		_encodedir(PyBytes_AS_STRING(newobj), newlen, path, len + 1);
 	}
 
 	return newobj;
 }
 
 static Py_ssize_t _encode(const uint32_t twobytes[8], const uint32_t onebyte[8],
                           char *dest, Py_ssize_t destlen, size_t destsize,
                           const char *src, Py_ssize_t len, int encodedir)
 {
 	enum path_state state = START;
 	Py_ssize_t i = 0;
 
 	/*
 	 * Python strings end with a zero byte, which we use as a
 	 * terminal token as they are not valid inside path names.
 	 */
 
 	while (i < len) {
 		switch (state) {
 		case START:
 			switch (src[i]) {
 			case '/':
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case '.':
 				state = LDOT;
 				escape3(dest, &destlen, destsize, src[i++]);
 				break;
 			case ' ':
 				state = DEFAULT;
 				escape3(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'a':
 				state = A;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'c':
 				state = C;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'l':
 				state = L;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'n':
 				state = N;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'p':
 				state = P;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			default:
 				state = DEFAULT;
 				break;
 			}
 			break;
 		case A:
 			if (src[i] == 'u') {
 				state = AU;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case AU:
 			if (src[i] == 'x') {
 				state = THIRD;
 				i++;
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case THIRD:
 			state = DEFAULT;
 			switch (src[i]) {
 			case '.':
 			case '/':
 			case '\0':
 				escape3(dest, &destlen, destsize, src[i - 1]);
 				break;
 			default:
 				i--;
 				break;
 			}
 			break;
 		case C:
 			if (src[i] == 'o') {
 				state = CO;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case CO:
 			if (src[i] == 'm') {
 				state = COMLPT;
 				i++;
 			} else if (src[i] == 'n') {
 				state = THIRD;
 				i++;
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case COMLPT:
 			switch (src[i]) {
 			case '1':
 			case '2':
 			case '3':
 			case '4':
 			case '5':
 			case '6':
 			case '7':
 			case '8':
 			case '9':
 				state = COMLPTn;
 				i++;
 				break;
 			default:
 				state = DEFAULT;
 				charcopy(dest, &destlen, destsize, src[i - 1]);
 				break;
 			}
 			break;
 		case COMLPTn:
 			state = DEFAULT;
 			switch (src[i]) {
 			case '.':
 			case '/':
 			case '\0':
 				escape3(dest, &destlen, destsize, src[i - 2]);
 				charcopy(dest, &destlen, destsize, src[i - 1]);
 				break;
 			default:
 				memcopy(dest, &destlen, destsize, &src[i - 2],
 				        2);
 				break;
 			}
 			break;
 		case L:
 			if (src[i] == 'p') {
 				state = LP;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case LP:
 			if (src[i] == 't') {
 				state = COMLPT;
 				i++;
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case N:
 			if (src[i] == 'u') {
 				state = NU;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case NU:
 			if (src[i] == 'l') {
 				state = THIRD;
 				i++;
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case P:
 			if (src[i] == 'r') {
 				state = PR;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case PR:
 			if (src[i] == 'n') {
 				state = THIRD;
 				i++;
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case LDOT:
 			switch (src[i]) {
 			case 'd':
 			case 'i':
 				state = HGDI;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'h':
 				state = H;
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			default:
 				state = DEFAULT;
 				break;
 			}
 			break;
 		case DOT:
 			switch (src[i]) {
 			case '/':
 			case '\0':
 				state = START;
 				memcopy(dest, &destlen, destsize, "~2e", 3);
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'd':
 			case 'i':
 				state = HGDI;
 				charcopy(dest, &destlen, destsize, '.');
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			case 'h':
 				state = H;
 				memcopy(dest, &destlen, destsize, ".h", 2);
 				i++;
 				break;
 			default:
 				state = DEFAULT;
 				charcopy(dest, &destlen, destsize, '.');
 				break;
 			}
 			break;
 		case H:
 			if (src[i] == 'g') {
 				state = HGDI;
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case HGDI:
 			if (src[i] == '/') {
 				state = START;
 				if (encodedir) {
 					memcopy(dest, &destlen, destsize, ".hg",
 					        3);
 				}
 				charcopy(dest, &destlen, destsize, src[i++]);
 			} else {
 				state = DEFAULT;
 			}
 			break;
 		case SPACE:
 			switch (src[i]) {
 			case '/':
 			case '\0':
 				state = START;
 				memcopy(dest, &destlen, destsize, "~20", 3);
 				charcopy(dest, &destlen, destsize, src[i++]);
 				break;
 			default:
 				state = DEFAULT;
 				charcopy(dest, &destlen, destsize, ' ');
 				break;
 			}
 			break;
 		case DEFAULT:
 			while (inset(onebyte, src[i])) {
 				charcopy(dest, &destlen, destsize, src[i++]);
 				if (i == len) {
 					goto done;
 				}
 			}
 			switch (src[i]) {
 			case '.':
 				state = DOT;
 				i++;
 				break;
 			case ' ':
 				state = SPACE;
 				i++;
 				break;
 			case '/':
 				state = START;
 				charcopy(dest, &destlen, destsize, '/');
 				i++;
 				break;
 			default:
 				if (inset(onebyte, src[i])) {
 					do {
 						charcopy(dest, &destlen,
 						         destsize, src[i++]);
 					} while (i < len &&
 					         inset(onebyte, src[i]));
 				} else if (inset(twobytes, src[i])) {
 					char c = src[i++];
 					charcopy(dest, &destlen, destsize, '_');
 					charcopy(dest, &destlen, destsize,
 					         c == '_' ? '_' : c + 32);
 				} else {
 					escape3(dest, &destlen, destsize,
 					        src[i++]);
 				}
 				break;
 			}
 			break;
 		}
 	}
 done:
 	return destlen;
 }
 
 static Py_ssize_t basicencode(char *dest, size_t destsize, const char *src,
                               Py_ssize_t len)
 {
 	static const uint32_t twobytes[8] = {0, 0, 0x87fffffe};
 
 	static const uint32_t onebyte[8] = {
 	    1,
 	    0x2bff3bfa,
 	    0x68000001,
 	    0x2fffffff,
 	};
 
 	Py_ssize_t destlen = 0;
 
 	return _encode(twobytes, onebyte, dest, destlen, destsize, src, len, 1);
 }
 
 static const Py_ssize_t maxstorepathlen = 120;
 
 static Py_ssize_t _lowerencode(char *dest, size_t destsize, const char *src,
                                Py_ssize_t len)
 {
 	static const uint32_t onebyte[8] = {1, 0x2bfffbfb, 0xe8000001,
 	                                    0x2fffffff};
 
 	static const uint32_t lower[8] = {0, 0, 0x7fffffe};
 
 	Py_ssize_t i, destlen = 0;
 
 	for (i = 0; i < len; i++) {
 		if (inset(onebyte, src[i])) {
 			charcopy(dest, &destlen, destsize, src[i]);
 		} else if (inset(lower, src[i])) {
 			charcopy(dest, &destlen, destsize, src[i] + 32);
 		} else {
 			escape3(dest, &destlen, destsize, src[i]);
 		}
 	}
 
 	return destlen;
 }
 
 PyObject *lowerencode(PyObject *self, PyObject *args)
 {
 	char *path;
 	Py_ssize_t len, newlen;
 	PyObject *ret;
 
 	if (!PyArg_ParseTuple(args, PY23("s#:lowerencode", "y#:lowerencode"),
 	                      &path, &len)) {
 		return NULL;
 	}
 
 	newlen = _lowerencode(NULL, 0, path, len);
 	ret = PyBytes_FromStringAndSize(NULL, newlen);
 	if (ret) {
 		_lowerencode(PyBytes_AS_STRING(ret), newlen, path, len);
 	}
 
 	return ret;
 }
 
 /* See store.py:_auxencode for a description. */
 static Py_ssize_t auxencode(char *dest, size_t destsize, const char *src,
                             Py_ssize_t len)
 {
 	static const uint32_t twobytes[8];
 
 	static const uint32_t onebyte[8] = {
 	    ~0U, 0xffff3ffe, ~0U, ~0U, ~0U, ~0U, ~0U, ~0U,
 	};
 
 	return _encode(twobytes, onebyte, dest, 0, destsize, src, len, 0);
 }
 
 static PyObject *hashmangle(const char *src, Py_ssize_t len, const char sha[20])
 {
 	static const Py_ssize_t dirprefixlen = 8;
 	static const Py_ssize_t maxshortdirslen = 68;
 	char *dest;
 	PyObject *ret;
 
 	Py_ssize_t i, d, p, lastslash = len - 1, lastdot = -1;
 	Py_ssize_t destsize, destlen = 0, slop, used;
 
 	while (lastslash >= 0 && src[lastslash] != '/') {
 		if (src[lastslash] == '.' && lastdot == -1) {
 			lastdot = lastslash;
 		}
 		lastslash--;
 	}
 
 #if 0
 	/* All paths should end in a suffix of ".i" or ".d".
            Unfortunately, the file names in test-hybridencode.py
            violate this rule.  */
 	if (lastdot != len - 3) {
 		PyErr_SetString(PyExc_ValueError,
 				"suffix missing or wrong length");
 		return NULL;
 	}
 #endif
 
 	/* If src contains a suffix, we will append it to the end of
 	   the new string, so make room. */
 	destsize = 120;
 	if (lastdot >= 0) {
 		destsize += len - lastdot - 1;
 	}
 
 	ret = PyBytes_FromStringAndSize(NULL, destsize);
 	if (ret == NULL) {
 		return NULL;
 	}
 
 	dest = PyBytes_AS_STRING(ret);
 	memcopy(dest, &destlen, destsize, "dh/", 3);
 
 	/* Copy up to dirprefixlen bytes of each path component, up to
 	   a limit of maxshortdirslen bytes. */
 	for (i = d = p = 0; i < lastslash; i++, p++) {
 		if (src[i] == '/') {
 			char d = dest[destlen - 1];
 			/* After truncation, a directory name may end
 			   in a space or dot, which are unportable. */
 			if (d == '.' || d == ' ') {
 				dest[destlen - 1] = '_';
 				/* The + 3 is to account for "dh/" in the
 				 * beginning */
 			}
 			if (destlen > maxshortdirslen + 3) {
 				break;
 			}
 			charcopy(dest, &destlen, destsize, src[i]);
 			p = -1;
 		} else if (p < dirprefixlen) {
 			charcopy(dest, &destlen, destsize, src[i]);
 		}
 	}
 
 	/* Rewind to just before the last slash copied. */
 	if (destlen > maxshortdirslen + 3) {
 		do {
 			destlen--;
 		} while (destlen > 0 && dest[destlen] != '/');
 	}
 
 	if (destlen > 3) {
 		if (lastslash > 0) {
 			char d = dest[destlen - 1];
 			/* The last directory component may be
 			   truncated, so make it safe. */
 			if (d == '.' || d == ' ') {
 				dest[destlen - 1] = '_';
 			}
 		}
 
 		charcopy(dest, &destlen, destsize, '/');
 	}
 
 	/* Add a prefix of the original file's name. Its length
 	   depends on the number of bytes left after accounting for
 	   hash and suffix. */
 	used = destlen + 40;
 	if (lastdot >= 0) {
 		used += len - lastdot - 1;
 	}
 	slop = maxstorepathlen - used;
 	if (slop > 0) {
 		Py_ssize_t basenamelen =
 		    lastslash >= 0 ? len - lastslash - 2 : len - 1;
 
 		if (basenamelen > slop) {
 			basenamelen = slop;
 		}
 		if (basenamelen > 0) {
 			memcopy(dest, &destlen, destsize, &src[lastslash + 1],
 			        basenamelen);
 		}
 	}
 
 	/* Add hash and suffix. */
 	for (i = 0; i < 20; i++) {
 		hexencode(dest, &destlen, destsize, sha[i]);
 	}
 
 	if (lastdot >= 0) {
 		memcopy(dest, &destlen, destsize, &src[lastdot],
 		        len - lastdot - 1);
 	}
 
 	assert(PyBytes_Check(ret));
-	Py_SIZE(ret) = destlen;
+	Py_SET_SIZE(ret, destlen);
 
 	return ret;
 }
 
 /*
  * Avoiding a trip through Python would improve performance by 50%,
  * but we don't encounter enough long names to be worth the code.
  */
 static int sha1hash(char hash[20], const char *str, Py_ssize_t len)
 {
 	static PyObject *shafunc;
 	PyObject *shaobj, *hashobj;
 
 	if (shafunc == NULL) {
 		PyObject *hashlib = PyImport_ImportModule("hashlib");
 		if (hashlib == NULL) {
 			PyErr_SetString(PyExc_ImportError,
 			                "pathencode failed to find hashlib");
 			return -1;
 		}
 		shafunc = PyObject_GetAttrString(hashlib, "sha1");
 		Py_DECREF(hashlib);
 
 		if (shafunc == NULL) {
 			PyErr_SetString(PyExc_AttributeError,
 			                "module 'hashlib' has no "
 			                "attribute 'sha1' in pathencode");
 			return -1;
 		}
 	}
 
 	shaobj = PyObject_CallFunction(shafunc, PY23("s#", "y#"), str, len);
 
 	if (shaobj == NULL) {
 		return -1;
 	}
 
 	hashobj = PyObject_CallMethod(shaobj, "digest", "");
 	Py_DECREF(shaobj);
 	if (hashobj == NULL) {
 		return -1;
 	}
 
 	if (!PyBytes_Check(hashobj) || PyBytes_GET_SIZE(hashobj) != 20) {
 		PyErr_SetString(PyExc_TypeError,
 		                "result of digest is not a 20-byte hash");
 		Py_DECREF(hashobj);
 		return -1;
 	}
 
 	memcpy(hash, PyBytes_AS_STRING(hashobj), 20);
 	Py_DECREF(hashobj);
 	return 0;
 }
 
 #define MAXENCODE 4096 * 4
 
 static PyObject *hashencode(const char *src, Py_ssize_t len)
 {
 	char dired[MAXENCODE];
 	char lowered[MAXENCODE];
 	char auxed[MAXENCODE];
 	Py_ssize_t dirlen, lowerlen, auxlen, baselen;
 	char sha[20];
 
 	baselen = (len - 5) * 3;
 	if (baselen >= MAXENCODE) {
 		PyErr_SetString(PyExc_ValueError, "string too long");
 		return NULL;
 	}
 
 	dirlen = _encodedir(dired, baselen, src, len);
 	if (sha1hash(sha, dired, dirlen - 1) == -1) {
 		return NULL;
 	}
 	lowerlen = _lowerencode(lowered, baselen, dired + 5, dirlen - 5);
 	auxlen = auxencode(auxed, baselen, lowered, lowerlen);
 	return hashmangle(auxed, auxlen, sha);
 }
 
 PyObject *pathencode(PyObject *self, PyObject *args)
 {
 	Py_ssize_t len, newlen;
 	PyObject *pathobj, *newobj;
 	char *path;
 
 	if (!PyArg_ParseTuple(args, "O:pathencode", &pathobj)) {
 		return NULL;
 	}
 
 	if (PyBytes_AsStringAndSize(pathobj, &path, &len) == -1) {
 		PyErr_SetString(PyExc_TypeError, "expected a string");
 		return NULL;
 	}
 
 	if (len > maxstorepathlen) {
 		newlen = maxstorepathlen + 2;
 	} else {
 		newlen = len ? basicencode(NULL, 0, path, len + 1) : 1;
 	}
 
 	if (newlen <= maxstorepathlen + 1) {
 		if (newlen == len + 1) {
 			Py_INCREF(pathobj);
 			return pathobj;
 		}
 
 		newobj = PyBytes_FromStringAndSize(NULL, newlen);
 
 		if (newobj) {
 			assert(PyBytes_Check(newobj));
 			Py_SIZE(newobj)--;
 			basicencode(PyBytes_AS_STRING(newobj), newlen, path,
 			            len + 1);
 		}
 	} else {
 		newobj = hashencode(path, len + 1);
 	}
 
 	return newobj;
 }

tests/test-check-code.t

@@ -1,95 +1,96 b''
 #require test-repo
 
   $ . "$TESTDIR/helpers-testrepo.sh"
   $ check_code="$TESTDIR"/../contrib/check-code.py
   $ cd "$TESTDIR"/..
 
 New errors are not allowed. Warnings are strongly discouraged.
 (The writing "no-che?k-code" is for not skipping this file when checking.)
 
   $ testrepohg locate \
   > -X contrib/python-zstandard \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/thirdparty \
+  > -X mercurial/pythoncapi_compat.h \
   > | sed 's-\\-/-g' | "$check_code" --warnings --per-file=0 - || false
   Skipping contrib/automation/hgautomation/__init__.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
   Skipping contrib/fuzz/FuzzedDataProvider.h it has no-che?k-code (glob)
   Skipping contrib/fuzz/standalone_fuzz_target_runner.cc it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/py2exe.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/pyoxidizer.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/util.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/wix.py it has no-che?k-code (glob)
   Skipping i18n/polib.py it has no-che?k-code (glob)
   Skipping mercurial/statprof.py it has no-che?k-code (glob)
   Skipping tests/badserverext.py it has no-che?k-code (glob)
 
 @commands in debugcommands.py should be in alphabetical order.
 
   >>> import re
   >>> commands = []
   >>> with open('mercurial/debugcommands.py', 'rb') as fh:
   ...     for line in fh:
   ...         m = re.match(br"^@command\('([a-z]+)", line)
   ...         if m:
   ...             commands.append(m.group(1))
   >>> scommands = list(sorted(commands))
   >>> for i, command in enumerate(scommands):
   ...     if command != commands[i]:
   ...         print('commands in debugcommands.py not sorted; first differing '
   ...               'command is %s; expected %s' % (commands[i], command))
   ...         break
 
 Prevent adding new files in the root directory accidentally.
 
   $ testrepohg files 'glob:*'
   .arcconfig
   .clang-format
   .editorconfig
   .hgignore
   .hgsigs
   .hgtags
   .jshintrc
   CONTRIBUTING
   CONTRIBUTORS
   COPYING
   Makefile
   README.rst
   black.toml
   hg
   hgeditor
   hgweb.cgi
   rustfmt.toml
   setup.py
 
 Prevent adding modules which could be shadowed by ancient .so/.dylib.
 
   $ testrepohg files \
   > mercurial/base85.py \
   > mercurial/bdiff.py \
   > mercurial/diffhelpers.py \
   > mercurial/mpatch.py \
   > mercurial/osutil.py \
   > mercurial/parsers.py \
   > mercurial/zstd.py
   [1]
 
 Keep python3 tests sorted:
   $ sort < contrib/python3-whitelist > $TESTTMP/py3sorted
   $ cmp contrib/python3-whitelist $TESTTMP/py3sorted || echo 'Please sort passing tests!'
 
 Keep Windows line endings in check
 
   $ testrepohg files 'set:eol(dos)'
   contrib/win32/hg.bat
   contrib/win32/mercurial.ini